def test_paging_using_secondary_indexes(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mybool boolean, sometext text, PRIMARY KEY (id, sometext) )") cursor.execute("CREATE INDEX ON paging_test(mybool)") def random_txt(text): return "'{random}'".format(random=uuid.uuid1()) data = """ | id | mybool| sometext | *100| 1 | True | [random] | *300| 2 | False | [random] | *500| 3 | True | [random] | *400| 4 | False | [random] | """ create_rows(data, cursor, 'paging_test', format_funcs=(str, str, random_txt)) stmt = SimpleStatement("select * from paging_test where mybool = true") stmt.setFetchSize(400) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('mybool', 'getBool', str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all_pages(), [400, 200])
def test_page_size_set_multiple_times_before(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, sometext text, PRIMARY KEY (id, sometext) )") def random_txt(text): return "'{random}'".format(random=uuid.uuid1()) data = """ | id | sometext | *2000| 1 | [random] | """ create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt)) stmt = SimpleStatement("select * from paging_test where id = 1") stmt.setFetchSize(1000) stmt.setFetchSize(100) stmt.setFetchSize(500) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('sometext', 'getString', str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 4) self.assertEqual(pf.num_results_all_pages(), [500,500,500,500])
def test_data_delete_removing_remainder(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )") def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | mytext | *500| 1 | [random] | *500| 2 | [random] | """ create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt}) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=500, consistency_level=CL.ALL) ) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # delete the results that would have shown up on page 2 cursor.execute(SimpleStatement("delete from paging_test where id = 2", consistency_level=CL.ALL)) pf.request_all() self.assertEqual(pf.pagecount(), 1) self.assertEqual(pf.num_results_all(), [500])
def test_page_size_after_results_all_retrieved(self): """ Confirm that page size change does nothing after results are exhausted. """ cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, sometext text, PRIMARY KEY (id, sometext) )") def random_txt(text): return "'{random}'".format(random=uuid.uuid1()) data = """ | id | sometext | *2000| 1 | [random] | """ create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt)) stmt = SimpleStatement("select * from paging_test where id = 1") stmt.setFetchSize(500) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('sometext', 'getString', str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 4) self.assertEqual(pf.num_results_all_pages(), [500,500,500,500]) # set statement fetch size and try for more results, should be none stmt.setFetchSize(1000) self.assertEqual(results.one(), None)
def test_node_unavailabe_during_paging(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() cursor = self.cql_connection(node1) self.create_ks(cursor, 'test_paging_size', 1) cursor.execute("CREATE TABLE paging_test ( id uuid, mytext text, PRIMARY KEY (id, mytext) )") def make_uuid(text): return uuid.uuid4() create_rows( """ | id | mytext | *10000| [uuid] | foo | """, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': make_uuid} ) future = cursor.execute_async( SimpleStatement("select * from paging_test where mytext = 'foo' allow filtering", fetch_size=2000, consistency_level=CL.ALL) ) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # stop a node and make sure we get an error trying to page the rest node1.stop() with self.assertRaisesRegexp(RuntimeError, 'Requested pages were not delivered before timeout'): pf.request_all()
def test_with_allow_filtering(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )" ) data = """ |id|value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| |6 |testing | |7 |and more testing| |8 |and more testing| |9 |and more testing| """ create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'value': unicode }) future = cursor.execute_async( SimpleStatement( "select * from paging_test where value = 'and more testing' ALLOW FILTERING", fetch_size=4, consistency_level=CL.ALL)) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [4, 3]) # make sure the allow filtering query matches the expected results (ignoring order) self.assertEqualIgnoreOrder( pf.all_data(), parse_data_into_dicts(""" |id|value | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| |7 |and more testing| |8 |and more testing| |9 |and more testing| """, format_funcs={ 'id': int, 'value': unicode }))
def test_row_TTL_expiry_during_paging(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )" ) def random_txt(text): return unicode(uuid.uuid4()) # create rows with TTL (some of which we'll try to get after expiry) create_rows(""" | id | mytext | *300| 1 | [random] | *400| 2 | [random] | """, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'mytext': random_txt }, postfix='USING TTL 10') # create rows without TTL create_rows(""" | id | mytext | *500| 3 | [random] | """, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'mytext': random_txt }) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2,3)", fetch_size=300, consistency_level=CL.ALL)) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # this page will be partition id=1, it has TTL rows but they are not expired yet # sleep so that the remaining TTL rows from partition id=2 expire time.sleep(15) pf.request_all() self.assertEqual(pf.pagecount(), 3) self.assertEqual(pf.num_results_all(), [300, 300, 200])
def test_with_allow_filtering(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )") data = """ |id|value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| |6 |testing | |7 |and more testing| |8 |and more testing| |9 |and more testing| """ create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str)) stmt = SimpleStatement("select * from paging_test where value = 'and more testing' ALLOW FILTERING") stmt.setFetchSize(4) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all_pages(), [4, 3]) # make sure the allow filtering query matches the expected results (ignoring order) self.assertEqualIgnoreOrder(pf.all_data(), parse_data_into_lists( """ |id|value | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| |7 |and more testing| |8 |and more testing| |9 |and more testing| """, format_funcs=(str, cql_str) ))
def test_paging_across_multi_wide_rows(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )") def random_txt(text): return "'{random}'".format(random=uuid.uuid1()) data = """ | id | value | *5000| 1 | [replaced with random] | *5000| 2 | [replaced with random] | """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt)) stmt = SimpleStatement("select * from paging_test where id in (1,2)") stmt.setFetchSize(3000) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 4) self.assertEqual(pf.num_results_all_pages(), [3000, 3000, 3000, 1000]) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_less_results_than_page_size(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )") data = """ |id| value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str)) stmt = SimpleStatement("select * from paging_test") stmt.setFetchSize(100) results = cursor.execute(stmt) self.assertTrue(results.isFullyFetched()) actual_data = [] for row in results: row_data = [str(row.getInt('id')), cql_str(row.getString('value'))] actual_data.append(row_data) self.assertTrue(row_data in expected_data) self.assertEqual(len(expected_data), len(actual_data)) self.assertTrue(results.isExhausted())
def test_data_change_impacting_earlier_page(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )") def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | mytext | *500| 1 | [random] | *500| 2 | [random] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt}) # get 501 rows so we have definitely got the 1st row of the second partition future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=501, consistency_level=CL.ALL) ) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # we got one page and should be done with the first partition (for id=1) # let's add another row for that first partition (id=1) and make sure it won't sneak into results cursor.execute(SimpleStatement("insert into paging_test (id, mytext) values (1, 'foo')", consistency_level=CL.ALL)) pf.request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [501, 499]) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_equal_results_to_page_size(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )") data = """ |id| value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str)) stmt = SimpleStatement("select * from paging_test") stmt.setFetchSize(5) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 1) self.assertEqual(pf.num_results_all_pages(), [5]) # make sure expected and actual have same data elements (ignoring order) self.assertEqualIgnoreOrder(expected_data, pf.all_data())
def test_with_equal_results_to_page_size(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int PRIMARY KEY, value text )") data = """ |id| value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'value': unicode }) future = cursor.execute_async( SimpleStatement("select * from paging_test", fetch_size=5, consistency_level=CL.ALL)) pf = PageFetcher(future).request_all() self.assertEqual(pf.num_results_all(), [5]) self.assertEqual(pf.pagecount(), 1) # make sure expected and actual have same data elements (ignoring order) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_paging_across_multi_wide_rows(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )") def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | value | *5000| 1 | [replaced with random] | *5000| 2 | [replaced with random] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': random_txt}) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=3000, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 4) self.assertEqual(pf.num_results_all(), [3000, 3000, 3000, 1000]) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_data_change_impacting_later_page(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )") def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | mytext | *500| 1 | [random] | *499| 2 | [random] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt}) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=500, consistency_level=CL.ALL) ) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # we've already paged the first partition, but adding a row for the second (id=2) # should still result in the row being seen on the subsequent pages cursor.execute(SimpleStatement("insert into paging_test (id, mytext) values (2, 'foo')", consistency_level=CL.ALL)) pf.request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [500, 500]) # add the new row to the expected data and then do a compare expected_data.append({u'id': 2, u'mytext': u'foo'}) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_less_results_than_page_size(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int PRIMARY KEY, value text )") data = """ |id| value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'value': unicode }) future = cursor.execute_async( SimpleStatement("select * from paging_test", fetch_size=100, consistency_level=CL.ALL)) pf = PageFetcher(future) pf.request_all() self.assertFalse(pf.has_more_pages) self.assertEqual(len(expected_data), len(pf.all_data()))
def test_with_equal_results_to_page_size(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )") data = """ |id| value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode}) future = cursor.execute_async( SimpleStatement("select * from paging_test", fetch_size=5, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.num_results_all(), [5]) self.assertEqual(pf.pagecount(), 1) # make sure expected and actual have same data elements (ignoring order) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_undefined_page_size_default(self): """ If the page size isn't sent then the default fetch size is used. """ cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id uuid PRIMARY KEY, value text )") def random_txt(text): return uuid.uuid4() data = """ | id |value | *5001| [uuid] |testing | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': random_txt, 'value': unicode}) future = cursor.execute_async( SimpleStatement("select * from paging_test", consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.num_results_all(), [5000, 1]) # make sure expected and actual have same data elements (ignoring order) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_node_unavailabe_during_paging(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 1) cursor.execute("CREATE TABLE paging_test ( id uuid, mytext text, PRIMARY KEY (id, mytext) )") def make_uuid(text): return str(uuid.uuid4()) expected_data = create_rows(""" | id | mytext | *10000| [uuid] | foo | """, cursor, 'paging_test', format_funcs=(make_uuid, cql_str) ) stmt = SimpleStatement("select * from paging_test where mytext = 'foo' allow filtering") stmt.setFetchSize(2000) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getUUID', str), ('mytext', 'getString', cql_str)] ) pf.get_page() # stop a node and make sure we get an error trying to page the rest node1.stop() with self.assertRaisesRegexp(exceptions.UnavailableException, 'Not enough replica available for query at consistency ONE'): pf.get_remaining_pages()
def test_paging_across_multi_wide_rows(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )" ) def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | value | *5000| 1 | [replaced with random] | *5000| 2 | [replaced with random] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'value': random_txt }) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=3000, consistency_level=CL.ALL)) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 4) self.assertEqual(pf.num_results_all(), [3000, 3000, 3000, 1000]) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_undefined_page_size_default(self): """ If the page size isn't sent then the default fetch size is used. """ cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id uuid PRIMARY KEY, value text )") def random_txt(text): return uuid.uuid4() data = """ | id |value | *5001| [uuid] |testing | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': random_txt, 'value': unicode }) future = cursor.execute_async( SimpleStatement("select * from paging_test", consistency_level=CL.ALL)) pf = PageFetcher(future).request_all() self.assertEqual(pf.num_results_all(), [5000, 1]) # make sure expected and actual have same data elements (ignoring order) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_row_TTL_expiry_during_paging(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )") def random_txt(text): return "'{random}'".format(random=uuid.uuid1()) # create rows with TTL (some of which we'll try to get after expiry) create_rows(""" | id | mytext | *300| 1 | [random] | *400| 2 | [random] | """, cursor, 'paging_test', format_funcs=(str, random_txt), postfix = 'USING TTL 10' ) # create rows without TTL create_rows(""" | id | mytext | *500| 3 | [random] | """, cursor, 'paging_test', format_funcs=(str, random_txt) ) stmt = SimpleStatement("select * from paging_test where id in (1,2,3)") stmt.setFetchSize(300) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('mytext', 'getString', cql_str)] ) # this page will be partition id=1, it has TTL rows but they are not expired yet pf.get_page() # sleep so that the remaining TTL rows from partition id=2 expire time.sleep(15) pf.get_remaining_pages() self.assertEqual(pf.pagecount(), 3) self.assertEqual(pf.num_results_all_pages(), [300, 300, 200])
def test_with_allow_filtering(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )") data = """ |id|value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| |6 |testing | |7 |and more testing| |8 |and more testing| |9 |and more testing| """ create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode}) future = cursor.execute_async( SimpleStatement("select * from paging_test where value = 'and more testing' ALLOW FILTERING", fetch_size=4, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [4, 3]) # make sure the allow filtering query matches the expected results (ignoring order) self.assertEqualIgnoreOrder( pf.all_data(), parse_data_into_dicts( """ |id|value | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| |7 |and more testing| |8 |and more testing| |9 |and more testing| """, format_funcs={'id': int, 'value': unicode} ) )
def test_with_order_by_reversed(self): """" Paging over a single partition with ordering and a reversed clustering order. """ cursor = self.prepare() self.create_ks(cursor, 'test_paging', 2) cursor.execute( """ CREATE TABLE paging_test ( id int, value text, value2 text, PRIMARY KEY (id, value) ) WITH CLUSTERING ORDER BY (value DESC) """) data = """ |id|value|value2| |1 |a |a | |1 |b |b | |1 |c |c | |1 |d |d | |1 |e |e | |1 |f |f | |1 |g |g | |1 |h |h | |1 |i |i | |1 |j |j | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode, 'value2': unicode}) future = cursor.execute_async( SimpleStatement("select * from paging_test where id = 1 order by value asc", fetch_size=3, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 4) self.assertEqual(pf.num_results_all(), [3, 3, 3, 1]) # these should be equal (in the same order) self.assertEqual(pf.all_data(), expected_data) # drop the ORDER BY future = cursor.execute_async( SimpleStatement("select * from paging_test where id = 1", fetch_size=3, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 4) self.assertEqual(pf.num_results_all(), [3, 3, 3, 1]) # these should be equal (in the same order) self.assertEqual(pf.all_data(), list(reversed(expected_data)))
def test_with_order_by(self): """" Paging over a single partition with ordering should work. (Spanning multiple partitions won't though, by design. See CASSANDRA-6722). """ cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging', 2) cursor.execute( """ CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) ) WITH CLUSTERING ORDER BY (value ASC) """) data = """ |id|value| |1 |a | |1 |b | |1 |c | |1 |d | |1 |e | |1 |f | |1 |g | |1 |h | |1 |i | |1 |j | """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str)) stmt = SimpleStatement("select * from paging_test where id = 1 order by value asc") stmt.setFetchSize(5) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all_pages(), [5, 5]) # these should be equal (in the same order) self.assertEqual(expected_data, pf.all_data()) # make sure we don't allow paging over multiple partitions with order because that's weird with self.assertRaisesRegexp(exceptions.InvalidQueryException, 'Cannot page queries with both ORDER BY and a IN restriction on the partition key'): stmt = SimpleStatement("select * from paging_test where id in (1,2) order by value asc") cursor.execute(stmt)
def test_data_delete_removing_remainder(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )" ) def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | mytext | *500| 1 | [random] | *500| 2 | [random] | """ create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'mytext': random_txt }) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=500, consistency_level=CL.ALL)) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # delete the results that would have shown up on page 2 cursor.execute( SimpleStatement("delete from paging_test where id = 2", consistency_level=CL.ALL)) pf.request_all() self.assertEqual(pf.pagecount(), 1) self.assertEqual(pf.num_results_all(), [500])
def test_row_TTL_expiry_during_paging(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )") def random_txt(text): return unicode(uuid.uuid4()) # create rows with TTL (some of which we'll try to get after expiry) create_rows( """ | id | mytext | *300| 1 | [random] | *400| 2 | [random] | """, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt}, postfix='USING TTL 10' ) # create rows without TTL create_rows( """ | id | mytext | *500| 3 | [random] | """, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt} ) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2,3)", fetch_size=300, consistency_level=CL.ALL) ) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # this page will be partition id=1, it has TTL rows but they are not expired yet # sleep so that the remaining TTL rows from partition id=2 expire time.sleep(15) pf.request_all() self.assertEqual(pf.pagecount(), 3) self.assertEqual(pf.num_results_all(), [300, 300, 200])
def test_with_limit(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )") def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | value | *5| 1 | [random text] | *5| 2 | [random text] | *10| 3 | [random text] | *10| 4 | [random text] | *20| 5 | [random text] | *30| 6 | [random text] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': random_txt}) scenarios = [ # using equals clause w/single partition {'limit': 10, 'fetch': 20, 'data_size': 30, 'whereclause': 'WHERE id = 6', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # limit < fetch < data {'limit': 10, 'fetch': 30, 'data_size': 20, 'whereclause': 'WHERE id = 5', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # limit < data < fetch {'limit': 20, 'fetch': 10, 'data_size': 30, 'whereclause': 'WHERE id = 6', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10]}, # fetch < limit < data {'limit': 30, 'fetch': 10, 'data_size': 20, 'whereclause': 'WHERE id = 5', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10]}, # fetch < data < limit {'limit': 20, 'fetch': 30, 'data_size': 10, 'whereclause': 'WHERE id = 3', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # data < limit < fetch {'limit': 30, 'fetch': 20, 'data_size': 10, 'whereclause': 'WHERE id = 3', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # data < fetch < limit # using 'in' clause w/multi partitions {'limit': 9, 'fetch': 20, 'data_size': 80, 'whereclause': 'WHERE id in (1,2,3,4,5,6)', 'expect_pgcount': 1, 'expect_pgsizes': [9]}, # limit < fetch < data {'limit': 10, 'fetch': 30, 'data_size': 20, 'whereclause': 'WHERE id in (3,4)', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # limit < data < fetch {'limit': 20, 'fetch': 10, 'data_size': 30, 'whereclause': 'WHERE id in (4,5)', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10]}, # fetch < limit < data {'limit': 30, 'fetch': 10, 'data_size': 20, 'whereclause': 'WHERE id in (3,4)', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10]}, # fetch < data < limit {'limit': 20, 'fetch': 30, 'data_size': 10, 'whereclause': 'WHERE id in (1,2)', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # data < limit < fetch {'limit': 30, 'fetch': 20, 'data_size': 10, 'whereclause': 'WHERE id in (1,2)', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # data < fetch < limit ] def handle_scenario(scenario): future = cursor.execute_async( SimpleStatement( "select * from paging_test {} limit {}".format(scenario['whereclause'], scenario['limit']), fetch_size=scenario['fetch'], consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.num_results_all(), scenario['expect_pgsizes']) self.assertEqual(pf.pagecount(), scenario['expect_pgcount']) # make sure all the data retrieved is a subset of input data self.assertIsSubsetOf(pf.all_data(), expected_data) run_scenarios(scenarios, handle_scenario, deferred_exceptions=(AssertionError,))
def test_with_limit(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )") data = """ |id|value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| |6 |testing | |7 |and more testing| |8 |and more testing| |9 |and more testing| """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str)) stmt = SimpleStatement("select * from paging_test limit 5") stmt.setFetchSize(9) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 1) self.assertEqual(pf.num_results_all_pages(), [5]) # make sure all the data retrieved is a subset of input data self.assertIsSubsetOf(pf.all_data(), expected_data) # let's do another query with a limit larger than one page stmt = SimpleStatement("select * from paging_test limit 8") stmt.setFetchSize(5) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all_pages(), [5, 3]) self.assertIsSubsetOf(pf.all_data(), expected_data)
def test_node_unavailabe_during_paging(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() cursor = self.cql_connection(node1) self.create_ks(cursor, 'test_paging_size', 1) cursor.execute( "CREATE TABLE paging_test ( id uuid, mytext text, PRIMARY KEY (id, mytext) )" ) def make_uuid(text): return uuid.uuid4() create_rows(""" | id | mytext | *10000| [uuid] | foo | """, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': make_uuid}) future = cursor.execute_async( SimpleStatement( "select * from paging_test where mytext = 'foo' allow filtering", fetch_size=2000, consistency_level=CL.ALL)) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # stop a node and make sure we get an error trying to page the rest node1.stop() with self.assertRaisesRegexp( RuntimeError, 'Requested pages were not delivered before timeout'): pf.request_all()
def test_with_order_by(self): """" Paging over a single partition with ordering should work. (Spanning multiple partitions won't though, by design. See CASSANDRA-6722). """ cursor = self.prepare() self.create_ks(cursor, 'test_paging', 2) cursor.execute( """ CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) ) WITH CLUSTERING ORDER BY (value ASC) """) data = """ |id|value| |1 |a | |1 |b | |1 |c | |1 |d | |1 |e | |1 |f | |1 |g | |1 |h | |1 |i | |1 |j | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode}) future = cursor.execute_async( SimpleStatement("select * from paging_test where id = 1 order by value asc", fetch_size=5, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [5, 5]) # these should be equal (in the same order) self.assertEqual(pf.all_data(), expected_data) # make sure we don't allow paging over multiple partitions with order because that's weird with self.assertRaisesRegexp(InvalidRequest, 'Cannot page queries with both ORDER BY and a IN restriction on the partition key'): stmt = SimpleStatement("select * from paging_test where id in (1,2) order by value asc", consistency_level=CL.ALL) cursor.execute(stmt)
def test_data_change_impacting_later_page(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )" ) def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | mytext | *500| 1 | [random] | *499| 2 | [random] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'mytext': random_txt }) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=500, consistency_level=CL.ALL)) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # we've already paged the first partition, but adding a row for the second (id=2) # should still result in the row being seen on the subsequent pages cursor.execute( SimpleStatement( "insert into paging_test (id, mytext) values (2, 'foo')", consistency_level=CL.ALL)) pf.request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [500, 500]) # add the new row to the expected data and then do a compare expected_data.append({u'id': 2, u'mytext': u'foo'}) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_data_change_impacting_earlier_page(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )" ) def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | mytext | *500| 1 | [random] | *500| 2 | [random] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'mytext': random_txt }) # get 501 rows so we have definitely got the 1st row of the second partition future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=501, consistency_level=CL.ALL)) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved # we got one page and should be done with the first partition (for id=1) # let's add another row for that first partition (id=1) and make sure it won't sneak into results cursor.execute( SimpleStatement( "insert into paging_test (id, mytext) values (1, 'foo')", consistency_level=CL.ALL)) pf.request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [501, 499]) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_paging_using_secondary_indexes(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, mybool boolean, sometext text, PRIMARY KEY (id, sometext) )" ) cursor.execute("CREATE INDEX ON paging_test(mybool)") def random_txt(text): return unicode(uuid.uuid4()) def bool_from_str_int(text): return bool(int(text)) data = """ | id | mybool| sometext | *100| 1 | 1 | [random] | *300| 2 | 0 | [random] | *500| 3 | 1 | [random] | *400| 4 | 0 | [random] | """ all_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'mybool': bool_from_str_int, 'sometext': random_txt }) future = cursor.execute_async( SimpleStatement("select * from paging_test where mybool = true", fetch_size=400, consistency_level=CL.ALL)) pf = PageFetcher(future).request_all() # the query only searched for True rows, so let's pare down the expectations for comparison expected_data = filter(lambda x: x.get('mybool') is True, all_data) self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [400, 200]) self.assertEqualIgnoreOrder(expected_data, pf.all_data())
def test_data_change_impacting_later_page(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )") def random_txt(text): return "'{random}'".format(random=uuid.uuid1()) data = """ | id | mytext | *500| 1 | [random] | *499| 2 | [random] | """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt)) stmt = SimpleStatement("select * from paging_test where id in (1,2)") stmt.setFetchSize(500) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('mytext', 'getString', cql_str)] ) pf.get_page() # we've already paged the first partition, but adding a row for the second (id=2) # should still result in the row being seen on the subsequent pages cursor.execute(SimpleStatement("insert into paging_test (id, mytext) values (2, 'foo')")) pf.get_all_pages() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all_pages(), [500, 500]) # add the new row to the expected data and then do a compare expected_data.append([2, "'foo'"]) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_limit(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )") data = """ |id|value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| |6 |testing | |7 |and more testing| |8 |and more testing| |9 |and more testing| """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode}) future = cursor.execute_async( SimpleStatement("select * from paging_test limit 5", fetch_size=9, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 1) self.assertEqual(pf.num_results_all(), [5]) # make sure all the data retrieved is a subset of input data self.assertIsSubsetOf(pf.all_data(), expected_data) # let's do another query with a limit larger than one page future = cursor.execute_async( SimpleStatement("select * from paging_test limit 8", fetch_size=5, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [5, 3]) self.assertIsSubsetOf(pf.all_data(), expected_data)
def test_data_change_impacting_earlier_page(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )") def random_txt(text): return "'{random}'".format(random=uuid.uuid1()) data = """ | id | mytext | *500| 1 | [random] | *500| 2 | [random] | """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt)) stmt = SimpleStatement("select * from paging_test where id in (1,2)") # get 501 rows so we have definitely got the 1st row of the second partition stmt.setFetchSize(501) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('mytext', 'getString', cql_str)] ) pf.get_page() # we got one page and should be done with the first partition (for id=1) # let's add another row for that first partition (id=1) and make sure it won't sneak into results cursor.execute(SimpleStatement("insert into paging_test (id, mytext) values (1, 'foo')")) pf.get_all_pages() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all_pages(), [501, 499]) self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_less_results_than_page_size(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )") data = """ |id| value | |1 |testing | |2 |and more testing| |3 |and more testing| |4 |and more testing| |5 |and more testing| """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode}) future = cursor.execute_async( SimpleStatement("select * from paging_test", fetch_size=100, consistency_level=CL.ALL) ) pf = PageFetcher(future) pf.request_all() self.assertFalse(pf.has_more_pages) self.assertEqual(len(expected_data), len(pf.all_data()))
def test_zero_page_size_ignored(self): """ If the page size <= 0 then the default fetch size is used. """ cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id uuid PRIMARY KEY, value text )") def random_txt(text): return "{random}".format(random=uuid.uuid1()) data = """ | id |value | *5001| [uuid] |testing | """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(random_txt, cql_str)) time.sleep(5) stmt = SimpleStatement("select * from paging_test") stmt.setFetchSize(0) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getUUID', str), ('value', 'getString', cql_str)] ) pf.get_all_pages() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all_pages(), [5000, 1]) # make sure expected and actual have same data elements (ignoring order) self.assertEqualIgnoreOrder(expected_data, pf.all_data())
def test_data_delete_removing_remainder(self): cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() wait_for_node_alive(node1) cursor = self.cql_connection(node1).cursor() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )") def random_txt(text): return "'{random}'".format(random=uuid.uuid1()) data = """ | id | mytext | *500| 1 | [random] | *500| 2 | [random] | """ expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt)) stmt = SimpleStatement("select * from paging_test where id in (1,2)") stmt.setFetchSize(500) results = cursor.execute(stmt) pf = PageFetcher( results, formatters = [('id', 'getInt', str), ('mytext', 'getString', cql_str)] ) pf.get_page() # delete the results that would have shown up on page 2 cursor.execute(SimpleStatement("delete from paging_test where id = 2")) pf.get_all_pages() self.assertEqual(pf.pagecount(), 1) self.assertEqual(pf.num_results_all_pages(), [500])
def test_paging_using_secondary_indexes(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute("CREATE TABLE paging_test ( id int, mybool boolean, sometext text, PRIMARY KEY (id, sometext) )") cursor.execute("CREATE INDEX ON paging_test(mybool)") def random_txt(text): return unicode(uuid.uuid4()) def bool_from_str_int(text): return bool(int(text)) data = """ | id | mybool| sometext | *100| 1 | 1 | [random] | *300| 2 | 0 | [random] | *500| 3 | 1 | [random] | *400| 4 | 0 | [random] | """ all_data = create_rows( data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mybool': bool_from_str_int, 'sometext': random_txt} ) future = cursor.execute_async( SimpleStatement("select * from paging_test where mybool = true", fetch_size=400, consistency_level=CL.ALL) ) pf = PageFetcher(future).request_all() # the query only searched for True rows, so let's pare down the expectations for comparison expected_data = filter(lambda x: x.get('mybool') is True, all_data) self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [400, 200]) self.assertEqualIgnoreOrder(expected_data, pf.all_data())
def test_with_limit(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )" ) def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | value | *5| 1 | [random text] | *5| 2 | [random text] | *10| 3 | [random text] | *10| 4 | [random text] | *20| 5 | [random text] | *30| 6 | [random text] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'value': random_txt }) scenarios = [ # using equals clause w/single partition { 'limit': 10, 'fetch': 20, 'data_size': 30, 'whereclause': 'WHERE id = 6', 'expect_pgcount': 1, 'expect_pgsizes': [10] }, # limit < fetch < data { 'limit': 10, 'fetch': 30, 'data_size': 20, 'whereclause': 'WHERE id = 5', 'expect_pgcount': 1, 'expect_pgsizes': [10] }, # limit < data < fetch { 'limit': 20, 'fetch': 10, 'data_size': 30, 'whereclause': 'WHERE id = 6', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10] }, # fetch < limit < data { 'limit': 30, 'fetch': 10, 'data_size': 20, 'whereclause': 'WHERE id = 5', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10] }, # fetch < data < limit { 'limit': 20, 'fetch': 30, 'data_size': 10, 'whereclause': 'WHERE id = 3', 'expect_pgcount': 1, 'expect_pgsizes': [10] }, # data < limit < fetch { 'limit': 30, 'fetch': 20, 'data_size': 10, 'whereclause': 'WHERE id = 3', 'expect_pgcount': 1, 'expect_pgsizes': [10] }, # data < fetch < limit # using 'in' clause w/multi partitions { 'limit': 9, 'fetch': 20, 'data_size': 80, 'whereclause': 'WHERE id in (1,2,3,4,5,6)', 'expect_pgcount': 1, 'expect_pgsizes': [9] }, # limit < fetch < data { 'limit': 10, 'fetch': 30, 'data_size': 20, 'whereclause': 'WHERE id in (3,4)', 'expect_pgcount': 1, 'expect_pgsizes': [10] }, # limit < data < fetch { 'limit': 20, 'fetch': 10, 'data_size': 30, 'whereclause': 'WHERE id in (4,5)', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10] }, # fetch < limit < data { 'limit': 30, 'fetch': 10, 'data_size': 20, 'whereclause': 'WHERE id in (3,4)', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10] }, # fetch < data < limit { 'limit': 20, 'fetch': 30, 'data_size': 10, 'whereclause': 'WHERE id in (1,2)', 'expect_pgcount': 1, 'expect_pgsizes': [10] }, # data < limit < fetch { 'limit': 30, 'fetch': 20, 'data_size': 10, 'whereclause': 'WHERE id in (1,2)', 'expect_pgcount': 1, 'expect_pgsizes': [10] }, # data < fetch < limit ] def handle_scenario(scenario): future = cursor.execute_async( SimpleStatement("select * from paging_test {} limit {}".format( scenario['whereclause'], scenario['limit']), fetch_size=scenario['fetch'], consistency_level=CL.ALL)) pf = PageFetcher(future).request_all() self.assertEqual(pf.num_results_all(), scenario['expect_pgsizes']) self.assertEqual(pf.pagecount(), scenario['expect_pgcount']) # make sure all the data retrieved is a subset of input data self.assertIsSubsetOf(pf.all_data(), expected_data) run_scenarios(scenarios, handle_scenario, deferred_exceptions=(AssertionError, ))
def test_query_isolation(self): """ Interleave some paged queries and make sure nothing bad happens. """ cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute( "CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )" ) def random_txt(text): return unicode(uuid.uuid4()) data = """ | id | mytext | *5000| 1 | [random] | *5000| 2 | [random] | *5000| 3 | [random] | *5000| 4 | [random] | *5000| 5 | [random] | *5000| 6 | [random] | *5000| 7 | [random] | *5000| 8 | [random] | *5000| 9 | [random] | *5000| 10 | [random] | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'mytext': random_txt }) stmts = [ SimpleStatement("select * from paging_test where id in (1)", fetch_size=500, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (2)", fetch_size=600, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (3)", fetch_size=700, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (4)", fetch_size=800, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (5)", fetch_size=900, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (1)", fetch_size=1000, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (2)", fetch_size=1100, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (3)", fetch_size=1200, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (4)", fetch_size=1300, consistency_level=CL.ALL), SimpleStatement("select * from paging_test where id in (5)", fetch_size=1400, consistency_level=CL.ALL), SimpleStatement( "select * from paging_test where id in (1,2,3,4,5,6,7,8,9,10)", fetch_size=1500, consistency_level=CL.ALL) ] page_fetchers = [] for stmt in stmts: future = cursor.execute_async(stmt) page_fetchers.append(PageFetcher(future)) # first page is auto-retrieved, so no need to request it for pf in page_fetchers: pf.request_one() for pf in page_fetchers: pf.request_one() for pf in page_fetchers: pf.request_all() self.assertEqual(page_fetchers[0].pagecount(), 10) self.assertEqual(page_fetchers[1].pagecount(), 9) self.assertEqual(page_fetchers[2].pagecount(), 8) self.assertEqual(page_fetchers[3].pagecount(), 7) self.assertEqual(page_fetchers[4].pagecount(), 6) self.assertEqual(page_fetchers[5].pagecount(), 5) self.assertEqual(page_fetchers[6].pagecount(), 5) self.assertEqual(page_fetchers[7].pagecount(), 5) self.assertEqual(page_fetchers[8].pagecount(), 4) self.assertEqual(page_fetchers[9].pagecount(), 4) self.assertEqual(page_fetchers[10].pagecount(), 34) self.assertEqualIgnoreOrder(page_fetchers[0].all_data(), expected_data[:5000]) self.assertEqualIgnoreOrder(page_fetchers[1].all_data(), expected_data[5000:10000]) self.assertEqualIgnoreOrder(page_fetchers[2].all_data(), expected_data[10000:15000]) self.assertEqualIgnoreOrder(page_fetchers[3].all_data(), expected_data[15000:20000]) self.assertEqualIgnoreOrder(page_fetchers[4].all_data(), expected_data[20000:25000]) self.assertEqualIgnoreOrder(page_fetchers[5].all_data(), expected_data[:5000]) self.assertEqualIgnoreOrder(page_fetchers[6].all_data(), expected_data[5000:10000]) self.assertEqualIgnoreOrder(page_fetchers[7].all_data(), expected_data[10000:15000]) self.assertEqualIgnoreOrder(page_fetchers[8].all_data(), expected_data[15000:20000]) self.assertEqualIgnoreOrder(page_fetchers[9].all_data(), expected_data[20000:25000]) self.assertEqualIgnoreOrder(page_fetchers[10].all_data(), expected_data[:50000])
def test_with_order_by(self): """" Paging over a single partition with ordering should work. (Spanning multiple partitions won't though, by design. See CASSANDRA-6722). """ cursor = self.prepare() self.create_ks(cursor, 'test_paging', 2) cursor.execute(""" CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) ) WITH CLUSTERING ORDER BY (value ASC) """) data = """ |id|value| |1 |a | |1 |b | |1 |c | |1 |d | |1 |e | |1 |f | |1 |g | |1 |h | |1 |i | |1 |j | """ expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'value': unicode }) future = cursor.execute_async( SimpleStatement( "select * from paging_test where id = 1 order by value asc", fetch_size=5, consistency_level=CL.ALL)) pf = PageFetcher(future).request_all() self.assertEqual(pf.pagecount(), 2) self.assertEqual(pf.num_results_all(), [5, 5]) # these should be equal (in the same order) self.assertEqual(pf.all_data(), expected_data) # make sure we don't allow paging over multiple partitions with order because that's weird with self.assertRaisesRegexp( InvalidRequest, 'Cannot page queries with both ORDER BY and a IN restriction on the partition key' ): stmt = SimpleStatement( "select * from paging_test where id in (1,2) order by value asc", consistency_level=CL.ALL) cursor.execute(stmt)
def test_cell_TTL_expiry_during_paging(self): cursor = self.prepare() self.create_ks(cursor, 'test_paging_size', 2) cursor.execute(""" CREATE TABLE paging_test ( id int, mytext text, somevalue text, anothervalue text, PRIMARY KEY (id, mytext) ) """) def random_txt(text): return unicode(uuid.uuid4()) data = create_rows(""" | id | mytext | somevalue | anothervalue | *500| 1 | [random] | foo | bar | *500| 2 | [random] | foo | bar | *500| 3 | [random] | foo | bar | """, cursor, 'paging_test', cl=CL.ALL, format_funcs={ 'id': int, 'mytext': random_txt }) future = cursor.execute_async( SimpleStatement("select * from paging_test where id in (1,2,3)", fetch_size=500, consistency_level=CL.ALL)) pf = PageFetcher(future) # no need to request page here, because the first page is automatically retrieved page1 = pf.page_data(1) self.assertEqualIgnoreOrder(page1, data[:500]) # set some TTLs for data on page 3 for row in data[1000:1500]: _id, mytext = row['id'], row['mytext'] stmt = SimpleStatement(""" update paging_test using TTL 10 set somevalue='one', anothervalue='two' where id = {id} and mytext = '{mytext}' """.format(id=_id, mytext=mytext), consistency_level=CL.ALL) cursor.execute(stmt) # check page two pf.request_one() page2 = pf.page_data(2) self.assertEqualIgnoreOrder(page2, data[500:1000]) page3expected = [] for row in data[1000:1500]: _id, mytext = row['id'], row['mytext'] page3expected.append({ u'id': _id, u'mytext': mytext, u'somevalue': None, u'anothervalue': None }) time.sleep(15) pf.request_one() page3 = pf.page_data(3) self.assertEqualIgnoreOrder(page3, page3expected)