def test_paging_using_secondary_indexes(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mybool boolean, sometext text, PRIMARY KEY (id, sometext) )")
cursor.execute("CREATE INDEX ON paging_test(mybool)")
def random_txt(text):
return "'{random}'".format(random=uuid.uuid1())
data = """
| id | mybool| sometext |
*100| 1 | True | [random] |
*300| 2 | False | [random] |
*500| 3 | True | [random] |
*400| 4 | False | [random] |
"""
create_rows(data, cursor, 'paging_test', format_funcs=(str, str, random_txt))
stmt = SimpleStatement("select * from paging_test where mybool = true")
stmt.setFetchSize(400)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('mybool', 'getBool', str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all_pages(), [400, 200])
def test_page_size_set_multiple_times_before(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, sometext text, PRIMARY KEY (id, sometext) )")
def random_txt(text):
return "'{random}'".format(random=uuid.uuid1())
data = """
| id | sometext |
*2000| 1 | [random] |
"""
create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt))
stmt = SimpleStatement("select * from paging_test where id = 1")
stmt.setFetchSize(1000)
stmt.setFetchSize(100)
stmt.setFetchSize(500)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('sometext', 'getString', str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 4)
self.assertEqual(pf.num_results_all_pages(), [500,500,500,500])
def test_data_delete_removing_remainder(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )")
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | mytext |
*500| 1 | [random] |
*500| 2 | [random] |
"""
create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=500, consistency_level=CL.ALL)
)
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# delete the results that would have shown up on page 2
cursor.execute(SimpleStatement("delete from paging_test where id = 2", consistency_level=CL.ALL))
pf.request_all()
self.assertEqual(pf.pagecount(), 1)
self.assertEqual(pf.num_results_all(), [500])
def test_page_size_after_results_all_retrieved(self):
"""
Confirm that page size change does nothing after results are exhausted.
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, sometext text, PRIMARY KEY (id, sometext) )")
def random_txt(text):
return "'{random}'".format(random=uuid.uuid1())
data = """
| id | sometext |
*2000| 1 | [random] |
"""
create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt))
stmt = SimpleStatement("select * from paging_test where id = 1")
stmt.setFetchSize(500)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('sometext', 'getString', str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 4)
self.assertEqual(pf.num_results_all_pages(), [500,500,500,500])
# set statement fetch size and try for more results, should be none
stmt.setFetchSize(1000)
self.assertEqual(results.one(), None)
def test_node_unavailabe_during_paging(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
cursor = self.cql_connection(node1)
self.create_ks(cursor, 'test_paging_size', 1)
cursor.execute("CREATE TABLE paging_test ( id uuid, mytext text, PRIMARY KEY (id, mytext) )")
def make_uuid(text):
return uuid.uuid4()
create_rows(
"""
| id | mytext |
*10000| [uuid] | foo |
""",
cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': make_uuid}
)
future = cursor.execute_async(
SimpleStatement("select * from paging_test where mytext = 'foo' allow filtering", fetch_size=2000, consistency_level=CL.ALL)
)
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# stop a node and make sure we get an error trying to page the rest
node1.stop()
with self.assertRaisesRegexp(RuntimeError, 'Requested pages were not delivered before timeout'):
pf.request_all()
def test_with_allow_filtering(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )"
)
data = """
|id|value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
|6 |testing |
|7 |and more testing|
|8 |and more testing|
|9 |and more testing|
"""
create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'value': unicode
})
future = cursor.execute_async(
SimpleStatement(
"select * from paging_test where value = 'and more testing' ALLOW FILTERING",
fetch_size=4,
consistency_level=CL.ALL))
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [4, 3])
# make sure the allow filtering query matches the expected results (ignoring order)
self.assertEqualIgnoreOrder(
pf.all_data(),
parse_data_into_dicts("""
|id|value |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
|7 |and more testing|
|8 |and more testing|
|9 |and more testing|
""",
format_funcs={
'id': int,
'value': unicode
}))
def test_row_TTL_expiry_during_paging(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )"
)
def random_txt(text):
return unicode(uuid.uuid4())
# create rows with TTL (some of which we'll try to get after expiry)
create_rows("""
| id | mytext |
*300| 1 | [random] |
*400| 2 | [random] |
""",
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'mytext': random_txt
},
postfix='USING TTL 10')
# create rows without TTL
create_rows("""
| id | mytext |
*500| 3 | [random] |
""",
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'mytext': random_txt
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2,3)",
fetch_size=300,
consistency_level=CL.ALL))
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# this page will be partition id=1, it has TTL rows but they are not expired yet
# sleep so that the remaining TTL rows from partition id=2 expire
time.sleep(15)
pf.request_all()
self.assertEqual(pf.pagecount(), 3)
self.assertEqual(pf.num_results_all(), [300, 300, 200])
def test_with_allow_filtering(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )")
data = """
|id|value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
|6 |testing |
|7 |and more testing|
|8 |and more testing|
|9 |and more testing|
"""
create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str))
stmt = SimpleStatement("select * from paging_test where value = 'and more testing' ALLOW FILTERING")
stmt.setFetchSize(4)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all_pages(), [4, 3])
# make sure the allow filtering query matches the expected results (ignoring order)
self.assertEqualIgnoreOrder(pf.all_data(),
parse_data_into_lists(
"""
|id|value |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
|7 |and more testing|
|8 |and more testing|
|9 |and more testing|
""", format_funcs=(str, cql_str)
))
def test_paging_across_multi_wide_rows(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )")
def random_txt(text):
return "'{random}'".format(random=uuid.uuid1())
data = """
| id | value |
*5000| 1 | [replaced with random] |
*5000| 2 | [replaced with random] |
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt))
stmt = SimpleStatement("select * from paging_test where id in (1,2)")
stmt.setFetchSize(3000)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 4)
self.assertEqual(pf.num_results_all_pages(), [3000, 3000, 3000, 1000])
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_less_results_than_page_size(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )")
data = """
|id| value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str))
stmt = SimpleStatement("select * from paging_test")
stmt.setFetchSize(100)
results = cursor.execute(stmt)
self.assertTrue(results.isFullyFetched())
actual_data = []
for row in results:
row_data = [str(row.getInt('id')), cql_str(row.getString('value'))]
actual_data.append(row_data)
self.assertTrue(row_data in expected_data)
self.assertEqual(len(expected_data), len(actual_data))
self.assertTrue(results.isExhausted())
def test_data_change_impacting_earlier_page(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )")
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | mytext |
*500| 1 | [random] |
*500| 2 | [random] |
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt})
# get 501 rows so we have definitely got the 1st row of the second partition
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=501, consistency_level=CL.ALL)
)
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# we got one page and should be done with the first partition (for id=1)
# let's add another row for that first partition (id=1) and make sure it won't sneak into results
cursor.execute(SimpleStatement("insert into paging_test (id, mytext) values (1, 'foo')", consistency_level=CL.ALL))
pf.request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [501, 499])
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_equal_results_to_page_size(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )")
data = """
|id| value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str))
stmt = SimpleStatement("select * from paging_test")
stmt.setFetchSize(5)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 1)
self.assertEqual(pf.num_results_all_pages(), [5])
# make sure expected and actual have same data elements (ignoring order)
self.assertEqualIgnoreOrder(expected_data, pf.all_data())
def test_with_equal_results_to_page_size(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int PRIMARY KEY, value text )")
data = """
|id| value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'value': unicode
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test",
fetch_size=5,
consistency_level=CL.ALL))
pf = PageFetcher(future).request_all()
self.assertEqual(pf.num_results_all(), [5])
self.assertEqual(pf.pagecount(), 1)
# make sure expected and actual have same data elements (ignoring order)
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_paging_across_multi_wide_rows(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )")
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | value |
*5000| 1 | [replaced with random] |
*5000| 2 | [replaced with random] |
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': random_txt})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=3000, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 4)
self.assertEqual(pf.num_results_all(), [3000, 3000, 3000, 1000])
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_data_change_impacting_later_page(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )")
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | mytext |
*500| 1 | [random] |
*499| 2 | [random] |
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2)", fetch_size=500, consistency_level=CL.ALL)
)
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# we've already paged the first partition, but adding a row for the second (id=2)
# should still result in the row being seen on the subsequent pages
cursor.execute(SimpleStatement("insert into paging_test (id, mytext) values (2, 'foo')", consistency_level=CL.ALL))
pf.request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [500, 500])
# add the new row to the expected data and then do a compare
expected_data.append({u'id': 2, u'mytext': u'foo'})
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_less_results_than_page_size(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int PRIMARY KEY, value text )")
data = """
|id| value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'value': unicode
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test",
fetch_size=100,
consistency_level=CL.ALL))
pf = PageFetcher(future)
pf.request_all()
self.assertFalse(pf.has_more_pages)
self.assertEqual(len(expected_data), len(pf.all_data()))
def test_with_equal_results_to_page_size(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )")
data = """
|id| value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode})
future = cursor.execute_async(
SimpleStatement("select * from paging_test", fetch_size=5, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.num_results_all(), [5])
self.assertEqual(pf.pagecount(), 1)
# make sure expected and actual have same data elements (ignoring order)
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_undefined_page_size_default(self):
"""
If the page size isn't sent then the default fetch size is used.
"""
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id uuid PRIMARY KEY, value text )")
def random_txt(text):
return uuid.uuid4()
data = """
| id |value |
*5001| [uuid] |testing |
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': random_txt, 'value': unicode})
future = cursor.execute_async(
SimpleStatement("select * from paging_test", consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.num_results_all(), [5000, 1])
# make sure expected and actual have same data elements (ignoring order)
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_node_unavailabe_during_paging(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 1)
cursor.execute("CREATE TABLE paging_test ( id uuid, mytext text, PRIMARY KEY (id, mytext) )")
def make_uuid(text):
return str(uuid.uuid4())
expected_data = create_rows("""
| id | mytext |
*10000| [uuid] | foo |
""",
cursor, 'paging_test', format_funcs=(make_uuid, cql_str)
)
stmt = SimpleStatement("select * from paging_test where mytext = 'foo' allow filtering")
stmt.setFetchSize(2000)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getUUID', str), ('mytext', 'getString', cql_str)]
)
pf.get_page()
# stop a node and make sure we get an error trying to page the rest
node1.stop()
with self.assertRaisesRegexp(exceptions.UnavailableException, 'Not enough replica available for query at consistency ONE'):
pf.get_remaining_pages()
def test_paging_across_multi_wide_rows(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )"
)
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | value |
*5000| 1 | [replaced with random] |
*5000| 2 | [replaced with random] |
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'value': random_txt
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2)",
fetch_size=3000,
consistency_level=CL.ALL))
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 4)
self.assertEqual(pf.num_results_all(), [3000, 3000, 3000, 1000])
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_undefined_page_size_default(self):
"""
If the page size isn't sent then the default fetch size is used.
"""
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id uuid PRIMARY KEY, value text )")
def random_txt(text):
return uuid.uuid4()
data = """
| id |value |
*5001| [uuid] |testing |
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': random_txt,
'value': unicode
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test",
consistency_level=CL.ALL))
pf = PageFetcher(future).request_all()
self.assertEqual(pf.num_results_all(), [5000, 1])
# make sure expected and actual have same data elements (ignoring order)
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_row_TTL_expiry_during_paging(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )")
def random_txt(text):
return "'{random}'".format(random=uuid.uuid1())
# create rows with TTL (some of which we'll try to get after expiry)
create_rows("""
| id | mytext |
*300| 1 | [random] |
*400| 2 | [random] |
""",
cursor, 'paging_test', format_funcs=(str, random_txt), postfix = 'USING TTL 10'
)
# create rows without TTL
create_rows("""
| id | mytext |
*500| 3 | [random] |
""",
cursor, 'paging_test', format_funcs=(str, random_txt)
)
stmt = SimpleStatement("select * from paging_test where id in (1,2,3)")
stmt.setFetchSize(300)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('mytext', 'getString', cql_str)]
)
# this page will be partition id=1, it has TTL rows but they are not expired yet
pf.get_page()
# sleep so that the remaining TTL rows from partition id=2 expire
time.sleep(15)
pf.get_remaining_pages()
self.assertEqual(pf.pagecount(), 3)
self.assertEqual(pf.num_results_all_pages(), [300, 300, 200])
def test_with_allow_filtering(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )")
data = """
|id|value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
|6 |testing |
|7 |and more testing|
|8 |and more testing|
|9 |and more testing|
"""
create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where value = 'and more testing' ALLOW FILTERING", fetch_size=4, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [4, 3])
# make sure the allow filtering query matches the expected results (ignoring order)
self.assertEqualIgnoreOrder(
pf.all_data(),
parse_data_into_dicts(
"""
|id|value |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
|7 |and more testing|
|8 |and more testing|
|9 |and more testing|
""", format_funcs={'id': int, 'value': unicode}
)
)
def test_with_order_by_reversed(self):
""""
Paging over a single partition with ordering and a reversed clustering order.
"""
cursor = self.prepare()
self.create_ks(cursor, 'test_paging', 2)
cursor.execute(
"""
CREATE TABLE paging_test (
id int,
value text,
value2 text,
PRIMARY KEY (id, value)
) WITH CLUSTERING ORDER BY (value DESC)
""")
data = """
|id|value|value2|
|1 |a |a |
|1 |b |b |
|1 |c |c |
|1 |d |d |
|1 |e |e |
|1 |f |f |
|1 |g |g |
|1 |h |h |
|1 |i |i |
|1 |j |j |
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode, 'value2': unicode})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id = 1 order by value asc", fetch_size=3, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 4)
self.assertEqual(pf.num_results_all(), [3, 3, 3, 1])
# these should be equal (in the same order)
self.assertEqual(pf.all_data(), expected_data)
# drop the ORDER BY
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id = 1", fetch_size=3, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 4)
self.assertEqual(pf.num_results_all(), [3, 3, 3, 1])
# these should be equal (in the same order)
self.assertEqual(pf.all_data(), list(reversed(expected_data)))
def test_with_order_by(self):
""""
Paging over a single partition with ordering should work.
(Spanning multiple partitions won't though, by design. See CASSANDRA-6722).
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging', 2)
cursor.execute(
"""
CREATE TABLE paging_test (
id int,
value text,
PRIMARY KEY (id, value)
) WITH CLUSTERING ORDER BY (value ASC)
""")
data = """
|id|value|
|1 |a |
|1 |b |
|1 |c |
|1 |d |
|1 |e |
|1 |f |
|1 |g |
|1 |h |
|1 |i |
|1 |j |
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str))
stmt = SimpleStatement("select * from paging_test where id = 1 order by value asc")
stmt.setFetchSize(5)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all_pages(), [5, 5])
# these should be equal (in the same order)
self.assertEqual(expected_data, pf.all_data())
# make sure we don't allow paging over multiple partitions with order because that's weird
with self.assertRaisesRegexp(exceptions.InvalidQueryException, 'Cannot page queries with both ORDER BY and a IN restriction on the partition key'):
stmt = SimpleStatement("select * from paging_test where id in (1,2) order by value asc")
cursor.execute(stmt)
def test_data_delete_removing_remainder(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )"
)
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | mytext |
*500| 1 | [random] |
*500| 2 | [random] |
"""
create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'mytext': random_txt
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2)",
fetch_size=500,
consistency_level=CL.ALL))
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# delete the results that would have shown up on page 2
cursor.execute(
SimpleStatement("delete from paging_test where id = 2",
consistency_level=CL.ALL))
pf.request_all()
self.assertEqual(pf.pagecount(), 1)
self.assertEqual(pf.num_results_all(), [500])
def test_row_TTL_expiry_during_paging(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )")
def random_txt(text):
return unicode(uuid.uuid4())
# create rows with TTL (some of which we'll try to get after expiry)
create_rows(
"""
| id | mytext |
*300| 1 | [random] |
*400| 2 | [random] |
""",
cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt}, postfix='USING TTL 10'
)
# create rows without TTL
create_rows(
"""
| id | mytext |
*500| 3 | [random] |
""",
cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'mytext': random_txt}
)
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2,3)", fetch_size=300, consistency_level=CL.ALL)
)
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# this page will be partition id=1, it has TTL rows but they are not expired yet
# sleep so that the remaining TTL rows from partition id=2 expire
time.sleep(15)
pf.request_all()
self.assertEqual(pf.pagecount(), 3)
self.assertEqual(pf.num_results_all(), [300, 300, 200])
def test_with_limit(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )")
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | value |
*5| 1 | [random text] |
*5| 2 | [random text] |
*10| 3 | [random text] |
*10| 4 | [random text] |
*20| 5 | [random text] |
*30| 6 | [random text] |
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': random_txt})
scenarios = [
# using equals clause w/single partition
{'limit': 10, 'fetch': 20, 'data_size': 30, 'whereclause': 'WHERE id = 6', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # limit < fetch < data
{'limit': 10, 'fetch': 30, 'data_size': 20, 'whereclause': 'WHERE id = 5', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # limit < data < fetch
{'limit': 20, 'fetch': 10, 'data_size': 30, 'whereclause': 'WHERE id = 6', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10]}, # fetch < limit < data
{'limit': 30, 'fetch': 10, 'data_size': 20, 'whereclause': 'WHERE id = 5', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10]}, # fetch < data < limit
{'limit': 20, 'fetch': 30, 'data_size': 10, 'whereclause': 'WHERE id = 3', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # data < limit < fetch
{'limit': 30, 'fetch': 20, 'data_size': 10, 'whereclause': 'WHERE id = 3', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # data < fetch < limit
# using 'in' clause w/multi partitions
{'limit': 9, 'fetch': 20, 'data_size': 80, 'whereclause': 'WHERE id in (1,2,3,4,5,6)', 'expect_pgcount': 1, 'expect_pgsizes': [9]}, # limit < fetch < data
{'limit': 10, 'fetch': 30, 'data_size': 20, 'whereclause': 'WHERE id in (3,4)', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # limit < data < fetch
{'limit': 20, 'fetch': 10, 'data_size': 30, 'whereclause': 'WHERE id in (4,5)', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10]}, # fetch < limit < data
{'limit': 30, 'fetch': 10, 'data_size': 20, 'whereclause': 'WHERE id in (3,4)', 'expect_pgcount': 2, 'expect_pgsizes': [10, 10]}, # fetch < data < limit
{'limit': 20, 'fetch': 30, 'data_size': 10, 'whereclause': 'WHERE id in (1,2)', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # data < limit < fetch
{'limit': 30, 'fetch': 20, 'data_size': 10, 'whereclause': 'WHERE id in (1,2)', 'expect_pgcount': 1, 'expect_pgsizes': [10]}, # data < fetch < limit
]
def handle_scenario(scenario):
future = cursor.execute_async(
SimpleStatement(
"select * from paging_test {} limit {}".format(scenario['whereclause'], scenario['limit']),
fetch_size=scenario['fetch'], consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.num_results_all(), scenario['expect_pgsizes'])
self.assertEqual(pf.pagecount(), scenario['expect_pgcount'])
# make sure all the data retrieved is a subset of input data
self.assertIsSubsetOf(pf.all_data(), expected_data)
run_scenarios(scenarios, handle_scenario, deferred_exceptions=(AssertionError,))
def test_with_limit(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )")
data = """
|id|value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
|6 |testing |
|7 |and more testing|
|8 |and more testing|
|9 |and more testing|
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, cql_str))
stmt = SimpleStatement("select * from paging_test limit 5")
stmt.setFetchSize(9)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 1)
self.assertEqual(pf.num_results_all_pages(), [5])
# make sure all the data retrieved is a subset of input data
self.assertIsSubsetOf(pf.all_data(), expected_data)
# let's do another query with a limit larger than one page
stmt = SimpleStatement("select * from paging_test limit 8")
stmt.setFetchSize(5)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('value', 'getString', cql_str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all_pages(), [5, 3])
self.assertIsSubsetOf(pf.all_data(), expected_data)
def test_node_unavailabe_during_paging(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
cursor = self.cql_connection(node1)
self.create_ks(cursor, 'test_paging_size', 1)
cursor.execute(
"CREATE TABLE paging_test ( id uuid, mytext text, PRIMARY KEY (id, mytext) )"
)
def make_uuid(text):
return uuid.uuid4()
create_rows("""
| id | mytext |
*10000| [uuid] | foo |
""",
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={'id': make_uuid})
future = cursor.execute_async(
SimpleStatement(
"select * from paging_test where mytext = 'foo' allow filtering",
fetch_size=2000,
consistency_level=CL.ALL))
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# stop a node and make sure we get an error trying to page the rest
node1.stop()
with self.assertRaisesRegexp(
RuntimeError,
'Requested pages were not delivered before timeout'):
pf.request_all()
def test_with_order_by(self):
""""
Paging over a single partition with ordering should work.
(Spanning multiple partitions won't though, by design. See CASSANDRA-6722).
"""
cursor = self.prepare()
self.create_ks(cursor, 'test_paging', 2)
cursor.execute(
"""
CREATE TABLE paging_test (
id int,
value text,
PRIMARY KEY (id, value)
) WITH CLUSTERING ORDER BY (value ASC)
""")
data = """
|id|value|
|1 |a |
|1 |b |
|1 |c |
|1 |d |
|1 |e |
|1 |f |
|1 |g |
|1 |h |
|1 |i |
|1 |j |
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id = 1 order by value asc", fetch_size=5, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [5, 5])
# these should be equal (in the same order)
self.assertEqual(pf.all_data(), expected_data)
# make sure we don't allow paging over multiple partitions with order because that's weird
with self.assertRaisesRegexp(InvalidRequest, 'Cannot page queries with both ORDER BY and a IN restriction on the partition key'):
stmt = SimpleStatement("select * from paging_test where id in (1,2) order by value asc", consistency_level=CL.ALL)
cursor.execute(stmt)
def test_data_change_impacting_later_page(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )"
)
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | mytext |
*500| 1 | [random] |
*499| 2 | [random] |
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'mytext': random_txt
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2)",
fetch_size=500,
consistency_level=CL.ALL))
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# we've already paged the first partition, but adding a row for the second (id=2)
# should still result in the row being seen on the subsequent pages
cursor.execute(
SimpleStatement(
"insert into paging_test (id, mytext) values (2, 'foo')",
consistency_level=CL.ALL))
pf.request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [500, 500])
# add the new row to the expected data and then do a compare
expected_data.append({u'id': 2, u'mytext': u'foo'})
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_data_change_impacting_earlier_page(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )"
)
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | mytext |
*500| 1 | [random] |
*500| 2 | [random] |
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'mytext': random_txt
})
# get 501 rows so we have definitely got the 1st row of the second partition
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2)",
fetch_size=501,
consistency_level=CL.ALL))
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
# we got one page and should be done with the first partition (for id=1)
# let's add another row for that first partition (id=1) and make sure it won't sneak into results
cursor.execute(
SimpleStatement(
"insert into paging_test (id, mytext) values (1, 'foo')",
consistency_level=CL.ALL))
pf.request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [501, 499])
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_paging_using_secondary_indexes(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, mybool boolean, sometext text, PRIMARY KEY (id, sometext) )"
)
cursor.execute("CREATE INDEX ON paging_test(mybool)")
def random_txt(text):
return unicode(uuid.uuid4())
def bool_from_str_int(text):
return bool(int(text))
data = """
| id | mybool| sometext |
*100| 1 | 1 | [random] |
*300| 2 | 0 | [random] |
*500| 3 | 1 | [random] |
*400| 4 | 0 | [random] |
"""
all_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'mybool': bool_from_str_int,
'sometext': random_txt
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where mybool = true",
fetch_size=400,
consistency_level=CL.ALL))
pf = PageFetcher(future).request_all()
# the query only searched for True rows, so let's pare down the expectations for comparison
expected_data = filter(lambda x: x.get('mybool') is True, all_data)
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [400, 200])
self.assertEqualIgnoreOrder(expected_data, pf.all_data())
def test_data_change_impacting_later_page(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )")
def random_txt(text):
return "'{random}'".format(random=uuid.uuid1())
data = """
| id | mytext |
*500| 1 | [random] |
*499| 2 | [random] |
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt))
stmt = SimpleStatement("select * from paging_test where id in (1,2)")
stmt.setFetchSize(500)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('mytext', 'getString', cql_str)]
)
pf.get_page()
# we've already paged the first partition, but adding a row for the second (id=2)
# should still result in the row being seen on the subsequent pages
cursor.execute(SimpleStatement("insert into paging_test (id, mytext) values (2, 'foo')"))
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all_pages(), [500, 500])
# add the new row to the expected data and then do a compare
expected_data.append([2, "'foo'"])
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_limit(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )")
data = """
|id|value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
|6 |testing |
|7 |and more testing|
|8 |and more testing|
|9 |and more testing|
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode})
future = cursor.execute_async(
SimpleStatement("select * from paging_test limit 5", fetch_size=9, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 1)
self.assertEqual(pf.num_results_all(), [5])
# make sure all the data retrieved is a subset of input data
self.assertIsSubsetOf(pf.all_data(), expected_data)
# let's do another query with a limit larger than one page
future = cursor.execute_async(
SimpleStatement("select * from paging_test limit 8", fetch_size=5, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [5, 3])
self.assertIsSubsetOf(pf.all_data(), expected_data)
def test_data_change_impacting_earlier_page(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )")
def random_txt(text):
return "'{random}'".format(random=uuid.uuid1())
data = """
| id | mytext |
*500| 1 | [random] |
*500| 2 | [random] |
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt))
stmt = SimpleStatement("select * from paging_test where id in (1,2)")
# get 501 rows so we have definitely got the 1st row of the second partition
stmt.setFetchSize(501)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('mytext', 'getString', cql_str)]
)
pf.get_page()
# we got one page and should be done with the first partition (for id=1)
# let's add another row for that first partition (id=1) and make sure it won't sneak into results
cursor.execute(SimpleStatement("insert into paging_test (id, mytext) values (1, 'foo')"))
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all_pages(), [501, 499])
self.assertEqualIgnoreOrder(pf.all_data(), expected_data)
def test_with_less_results_than_page_size(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int PRIMARY KEY, value text )")
data = """
|id| value |
|1 |testing |
|2 |and more testing|
|3 |and more testing|
|4 |and more testing|
|5 |and more testing|
"""
expected_data = create_rows(data, cursor, 'paging_test', cl=CL.ALL, format_funcs={'id': int, 'value': unicode})
future = cursor.execute_async(
SimpleStatement("select * from paging_test", fetch_size=100, consistency_level=CL.ALL)
)
pf = PageFetcher(future)
pf.request_all()
self.assertFalse(pf.has_more_pages)
self.assertEqual(len(expected_data), len(pf.all_data()))
def test_zero_page_size_ignored(self):
"""
If the page size <= 0 then the default fetch size is used.
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id uuid PRIMARY KEY, value text )")
def random_txt(text):
return "{random}".format(random=uuid.uuid1())
data = """
| id |value |
*5001| [uuid] |testing |
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(random_txt, cql_str))
time.sleep(5)
stmt = SimpleStatement("select * from paging_test")
stmt.setFetchSize(0)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getUUID', str), ('value', 'getString', cql_str)]
)
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all_pages(), [5000, 1])
# make sure expected and actual have same data elements (ignoring order)
self.assertEqualIgnoreOrder(expected_data, pf.all_data())
def test_data_delete_removing_remainder(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
wait_for_node_alive(node1)
cursor = self.cql_connection(node1).cursor()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )")
def random_txt(text):
return "'{random}'".format(random=uuid.uuid1())
data = """
| id | mytext |
*500| 1 | [random] |
*500| 2 | [random] |
"""
expected_data = create_rows(data, cursor, 'paging_test', format_funcs=(str, random_txt))
stmt = SimpleStatement("select * from paging_test where id in (1,2)")
stmt.setFetchSize(500)
results = cursor.execute(stmt)
pf = PageFetcher(
results, formatters = [('id', 'getInt', str), ('mytext', 'getString', cql_str)]
)
pf.get_page()
# delete the results that would have shown up on page 2
cursor.execute(SimpleStatement("delete from paging_test where id = 2"))
pf.get_all_pages()
self.assertEqual(pf.pagecount(), 1)
self.assertEqual(pf.num_results_all_pages(), [500])
def test_paging_using_secondary_indexes(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("CREATE TABLE paging_test ( id int, mybool boolean, sometext text, PRIMARY KEY (id, sometext) )")
cursor.execute("CREATE INDEX ON paging_test(mybool)")
def random_txt(text):
return unicode(uuid.uuid4())
def bool_from_str_int(text):
return bool(int(text))
data = """
| id | mybool| sometext |
*100| 1 | 1 | [random] |
*300| 2 | 0 | [random] |
*500| 3 | 1 | [random] |
*400| 4 | 0 | [random] |
"""
all_data = create_rows(
data, cursor, 'paging_test', cl=CL.ALL,
format_funcs={'id': int, 'mybool': bool_from_str_int, 'sometext': random_txt}
)
future = cursor.execute_async(
SimpleStatement("select * from paging_test where mybool = true", fetch_size=400, consistency_level=CL.ALL)
)
pf = PageFetcher(future).request_all()
# the query only searched for True rows, so let's pare down the expectations for comparison
expected_data = filter(lambda x: x.get('mybool') is True, all_data)
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [400, 200])
self.assertEqualIgnoreOrder(expected_data, pf.all_data())
def test_with_limit(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, value text, PRIMARY KEY (id, value) )"
)
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | value |
*5| 1 | [random text] |
*5| 2 | [random text] |
*10| 3 | [random text] |
*10| 4 | [random text] |
*20| 5 | [random text] |
*30| 6 | [random text] |
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'value': random_txt
})
scenarios = [
# using equals clause w/single partition
{
'limit': 10,
'fetch': 20,
'data_size': 30,
'whereclause': 'WHERE id = 6',
'expect_pgcount': 1,
'expect_pgsizes': [10]
}, # limit < fetch < data
{
'limit': 10,
'fetch': 30,
'data_size': 20,
'whereclause': 'WHERE id = 5',
'expect_pgcount': 1,
'expect_pgsizes': [10]
}, # limit < data < fetch
{
'limit': 20,
'fetch': 10,
'data_size': 30,
'whereclause': 'WHERE id = 6',
'expect_pgcount': 2,
'expect_pgsizes': [10, 10]
}, # fetch < limit < data
{
'limit': 30,
'fetch': 10,
'data_size': 20,
'whereclause': 'WHERE id = 5',
'expect_pgcount': 2,
'expect_pgsizes': [10, 10]
}, # fetch < data < limit
{
'limit': 20,
'fetch': 30,
'data_size': 10,
'whereclause': 'WHERE id = 3',
'expect_pgcount': 1,
'expect_pgsizes': [10]
}, # data < limit < fetch
{
'limit': 30,
'fetch': 20,
'data_size': 10,
'whereclause': 'WHERE id = 3',
'expect_pgcount': 1,
'expect_pgsizes': [10]
}, # data < fetch < limit
# using 'in' clause w/multi partitions
{
'limit': 9,
'fetch': 20,
'data_size': 80,
'whereclause': 'WHERE id in (1,2,3,4,5,6)',
'expect_pgcount': 1,
'expect_pgsizes': [9]
}, # limit < fetch < data
{
'limit': 10,
'fetch': 30,
'data_size': 20,
'whereclause': 'WHERE id in (3,4)',
'expect_pgcount': 1,
'expect_pgsizes': [10]
}, # limit < data < fetch
{
'limit': 20,
'fetch': 10,
'data_size': 30,
'whereclause': 'WHERE id in (4,5)',
'expect_pgcount': 2,
'expect_pgsizes': [10, 10]
}, # fetch < limit < data
{
'limit': 30,
'fetch': 10,
'data_size': 20,
'whereclause': 'WHERE id in (3,4)',
'expect_pgcount': 2,
'expect_pgsizes': [10, 10]
}, # fetch < data < limit
{
'limit': 20,
'fetch': 30,
'data_size': 10,
'whereclause': 'WHERE id in (1,2)',
'expect_pgcount': 1,
'expect_pgsizes': [10]
}, # data < limit < fetch
{
'limit': 30,
'fetch': 20,
'data_size': 10,
'whereclause': 'WHERE id in (1,2)',
'expect_pgcount': 1,
'expect_pgsizes': [10]
}, # data < fetch < limit
]
def handle_scenario(scenario):
future = cursor.execute_async(
SimpleStatement("select * from paging_test {} limit {}".format(
scenario['whereclause'], scenario['limit']),
fetch_size=scenario['fetch'],
consistency_level=CL.ALL))
pf = PageFetcher(future).request_all()
self.assertEqual(pf.num_results_all(), scenario['expect_pgsizes'])
self.assertEqual(pf.pagecount(), scenario['expect_pgcount'])
# make sure all the data retrieved is a subset of input data
self.assertIsSubsetOf(pf.all_data(), expected_data)
run_scenarios(scenarios,
handle_scenario,
deferred_exceptions=(AssertionError, ))
def test_query_isolation(self):
"""
Interleave some paged queries and make sure nothing bad happens.
"""
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute(
"CREATE TABLE paging_test ( id int, mytext text, PRIMARY KEY (id, mytext) )"
)
def random_txt(text):
return unicode(uuid.uuid4())
data = """
| id | mytext |
*5000| 1 | [random] |
*5000| 2 | [random] |
*5000| 3 | [random] |
*5000| 4 | [random] |
*5000| 5 | [random] |
*5000| 6 | [random] |
*5000| 7 | [random] |
*5000| 8 | [random] |
*5000| 9 | [random] |
*5000| 10 | [random] |
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'mytext': random_txt
})
stmts = [
SimpleStatement("select * from paging_test where id in (1)",
fetch_size=500,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (2)",
fetch_size=600,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (3)",
fetch_size=700,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (4)",
fetch_size=800,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (5)",
fetch_size=900,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (1)",
fetch_size=1000,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (2)",
fetch_size=1100,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (3)",
fetch_size=1200,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (4)",
fetch_size=1300,
consistency_level=CL.ALL),
SimpleStatement("select * from paging_test where id in (5)",
fetch_size=1400,
consistency_level=CL.ALL),
SimpleStatement(
"select * from paging_test where id in (1,2,3,4,5,6,7,8,9,10)",
fetch_size=1500,
consistency_level=CL.ALL)
]
page_fetchers = []
for stmt in stmts:
future = cursor.execute_async(stmt)
page_fetchers.append(PageFetcher(future))
# first page is auto-retrieved, so no need to request it
for pf in page_fetchers:
pf.request_one()
for pf in page_fetchers:
pf.request_one()
for pf in page_fetchers:
pf.request_all()
self.assertEqual(page_fetchers[0].pagecount(), 10)
self.assertEqual(page_fetchers[1].pagecount(), 9)
self.assertEqual(page_fetchers[2].pagecount(), 8)
self.assertEqual(page_fetchers[3].pagecount(), 7)
self.assertEqual(page_fetchers[4].pagecount(), 6)
self.assertEqual(page_fetchers[5].pagecount(), 5)
self.assertEqual(page_fetchers[6].pagecount(), 5)
self.assertEqual(page_fetchers[7].pagecount(), 5)
self.assertEqual(page_fetchers[8].pagecount(), 4)
self.assertEqual(page_fetchers[9].pagecount(), 4)
self.assertEqual(page_fetchers[10].pagecount(), 34)
self.assertEqualIgnoreOrder(page_fetchers[0].all_data(),
expected_data[:5000])
self.assertEqualIgnoreOrder(page_fetchers[1].all_data(),
expected_data[5000:10000])
self.assertEqualIgnoreOrder(page_fetchers[2].all_data(),
expected_data[10000:15000])
self.assertEqualIgnoreOrder(page_fetchers[3].all_data(),
expected_data[15000:20000])
self.assertEqualIgnoreOrder(page_fetchers[4].all_data(),
expected_data[20000:25000])
self.assertEqualIgnoreOrder(page_fetchers[5].all_data(),
expected_data[:5000])
self.assertEqualIgnoreOrder(page_fetchers[6].all_data(),
expected_data[5000:10000])
self.assertEqualIgnoreOrder(page_fetchers[7].all_data(),
expected_data[10000:15000])
self.assertEqualIgnoreOrder(page_fetchers[8].all_data(),
expected_data[15000:20000])
self.assertEqualIgnoreOrder(page_fetchers[9].all_data(),
expected_data[20000:25000])
self.assertEqualIgnoreOrder(page_fetchers[10].all_data(),
expected_data[:50000])
def test_with_order_by(self):
""""
Paging over a single partition with ordering should work.
(Spanning multiple partitions won't though, by design. See CASSANDRA-6722).
"""
cursor = self.prepare()
self.create_ks(cursor, 'test_paging', 2)
cursor.execute("""
CREATE TABLE paging_test (
id int,
value text,
PRIMARY KEY (id, value)
) WITH CLUSTERING ORDER BY (value ASC)
""")
data = """
|id|value|
|1 |a |
|1 |b |
|1 |c |
|1 |d |
|1 |e |
|1 |f |
|1 |g |
|1 |h |
|1 |i |
|1 |j |
"""
expected_data = create_rows(data,
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'value': unicode
})
future = cursor.execute_async(
SimpleStatement(
"select * from paging_test where id = 1 order by value asc",
fetch_size=5,
consistency_level=CL.ALL))
pf = PageFetcher(future).request_all()
self.assertEqual(pf.pagecount(), 2)
self.assertEqual(pf.num_results_all(), [5, 5])
# these should be equal (in the same order)
self.assertEqual(pf.all_data(), expected_data)
# make sure we don't allow paging over multiple partitions with order because that's weird
with self.assertRaisesRegexp(
InvalidRequest,
'Cannot page queries with both ORDER BY and a IN restriction on the partition key'
):
stmt = SimpleStatement(
"select * from paging_test where id in (1,2) order by value asc",
consistency_level=CL.ALL)
cursor.execute(stmt)
def test_cell_TTL_expiry_during_paging(self):
cursor = self.prepare()
self.create_ks(cursor, 'test_paging_size', 2)
cursor.execute("""
CREATE TABLE paging_test (
id int,
mytext text,
somevalue text,
anothervalue text,
PRIMARY KEY (id, mytext) )
""")
def random_txt(text):
return unicode(uuid.uuid4())
data = create_rows("""
| id | mytext | somevalue | anothervalue |
*500| 1 | [random] | foo | bar |
*500| 2 | [random] | foo | bar |
*500| 3 | [random] | foo | bar |
""",
cursor,
'paging_test',
cl=CL.ALL,
format_funcs={
'id': int,
'mytext': random_txt
})
future = cursor.execute_async(
SimpleStatement("select * from paging_test where id in (1,2,3)",
fetch_size=500,
consistency_level=CL.ALL))
pf = PageFetcher(future)
# no need to request page here, because the first page is automatically retrieved
page1 = pf.page_data(1)
self.assertEqualIgnoreOrder(page1, data[:500])
# set some TTLs for data on page 3
for row in data[1000:1500]:
_id, mytext = row['id'], row['mytext']
stmt = SimpleStatement("""
update paging_test using TTL 10
set somevalue='one', anothervalue='two' where id = {id} and mytext = '{mytext}'
""".format(id=_id, mytext=mytext),
consistency_level=CL.ALL)
cursor.execute(stmt)
# check page two
pf.request_one()
page2 = pf.page_data(2)
self.assertEqualIgnoreOrder(page2, data[500:1000])
page3expected = []
for row in data[1000:1500]:
_id, mytext = row['id'], row['mytext']
page3expected.append({
u'id': _id,
u'mytext': mytext,
u'somevalue': None,
u'anothervalue': None
})
time.sleep(15)
pf.request_one()
page3 = pf.page_data(3)
self.assertEqualIgnoreOrder(page3, page3expected)
