def test_always_continue_query(self):
""" Regression test.
If result has no items but does have LastEvaluatedKey, keep querying.
"""
conn = MagicMock()
conn.dynamizer.decode_keys.side_effect = lambda x: x
items = ['a', 'b']
results = [
{
'Items': [],
'LastEvaluatedKey': {
'foo': 1,
'bar': 2
}
},
{
'Items': [],
'LastEvaluatedKey': {
'foo': 1,
'bar': 2
}
},
{
'Items': items
},
]
conn.call.side_effect = lambda *_, **__: results.pop(0)
rs = ResultSet(conn, Limit())
results = list(rs)
self.assertEqual(results, items)
def test_always_continue_query(self):
"""Regression test.
If result has no items but does have LastEvaluatedKey, keep querying.
"""
conn = MagicMock()
conn.dynamizer.decode_keys.side_effect = lambda x: x
items = ["a", "b"]
results = [
{
"Items": [],
"LastEvaluatedKey": {
"foo": 1,
"bar": 2
}
},
{
"Items": [],
"LastEvaluatedKey": {
"foo": 1,
"bar": 2
}
},
{
"Items": items
},
]
conn.call.side_effect = lambda *_, **__: results.pop(0)
rs = ResultSet(conn, Limit())
results = list(rs)
self.assertEqual(results, items)
def _add_limit(self, kwargs):
""" Add the limit kwarg if necessary """
if self.limit is not None or self.scan_limit is not None:
if isinstance(self.limit, Limit):
kwargs['limit'] = self.limit
else:
kwargs['limit'] = Limit(scan_limit=self.scan_limit,
item_limit=self.limit,
strict=True)
def test_limit_complete(self):
""" A limit with item_capacity = 0 is 'complete' """
limit = Limit(item_limit=0)
self.assertTrue(limit.complete)
def _select(self, tree, allow_select_scan):
""" Run a SELECT statement """
tablename = tree.table
desc = self.describe(tablename, require=True)
kwargs = {}
if tree.consistent:
kwargs["consistent"] = True
visitor = Visitor(self.reserved_words)
selection = SelectionExpression.from_selection(tree.attrs)
if selection.is_count:
kwargs["select"] = "COUNT"
if tree.keys_in:
if tree.limit:
raise SyntaxError("Cannot use LIMIT with KEYS IN")
elif tree.using:
raise SyntaxError("Cannot use USING with KEYS IN")
elif tree.order:
raise SyntaxError("Cannot use DESC/ASC with KEYS IN")
elif tree.where:
raise SyntaxError("Cannot use WHERE with KEYS IN")
keys = list(self._iter_where_in(tree))
kwargs["attributes"] = selection.build(visitor)
kwargs["alias"] = visitor.attribute_names
return self.connection.batch_get(tablename, keys=keys, **kwargs)
if tree.limit:
if tree.scan_limit:
kwargs["limit"] = Limit(
scan_limit=resolve(tree.scan_limit[2]),
item_limit=resolve(tree.limit[1]),
strict=True,
)
else:
kwargs["limit"] = Limit(item_limit=resolve(tree.limit[1]),
strict=True)
elif tree.scan_limit:
kwargs["limit"] = Limit(scan_limit=resolve(tree.scan_limit[2]))
(action, query_kwargs, index) = self._build_query(desc, tree, visitor)
if action == "scan" and not allow_select_scan:
raise SyntaxError(
"No index found for query. Please use a SCAN query, or "
"set allow_select_scan=True\nopt allow_select_scan true")
order_by = None
if tree.order_by:
order_by = tree.order_by[0]
reverse = tree.order == "DESC"
if tree.order:
if action == "scan" and not tree.order_by:
raise SyntaxError("No index found for query, "
"cannot use ASC or DESC without "
"ORDER BY <field>")
if action == "query":
if order_by is None or order_by == index.range_key:
kwargs["desc"] = reverse
kwargs.update(query_kwargs)
# This is a special case for when we're querying an index and selecting
# fields that aren't projected into the index.
# We will change the query to only fetch the primary keys, and then
# fill in the selected attributes after the fact.
fetch_attrs_after = False
if index is not None and not index.projects_all_attributes(
selection.all_fields):
kwargs["attributes"] = [
visitor.get_field(a) for a in desc.primary_key_attributes
]
fetch_attrs_after = True
else:
kwargs["attributes"] = selection.build(visitor)
kwargs["expr_values"] = visitor.expression_values
kwargs["alias"] = visitor.attribute_names
method = getattr(self.connection, action + "2")
result = method(tablename, **kwargs)
# If the queried index didn't project the selected attributes, we need
# to do a BatchGetItem to fetch all the data.
if fetch_attrs_after:
if not isinstance(result, list):
result = list(result)
# If no results, no need to batch_get
if not result:
return result
visitor = Visitor(self.reserved_words)
kwargs = {"keys": [desc.primary_key(item) for item in result]}
kwargs["attributes"] = selection.build(visitor)
kwargs["alias"] = visitor.attribute_names
result = self.connection.batch_get(tablename, **kwargs)
def order(items):
""" Sort the items by the specified keys """
if order_by is None:
return items
if index is None or order_by != index.range_key:
if not isinstance(items, list):
items = list(items)
items.sort(key=lambda x: x.get(order_by), reverse=reverse)
return items
# Save the data to a file
if tree.save_file:
if selection.is_count:
raise Exception("Cannot use count(*) with SAVE")
count = 0
result = order(selection.convert(item, True) for item in result)
filename = tree.save_file[0]
if filename[0] in ['"', "'"]:
filename = unwrap(filename)
# If it's still an iterator, convert to a list so we can iterate
# multiple times.
if not isinstance(result, list):
result = list(result)
remainder, ext = os.path.splitext(filename)
if ext.lower() in [".gz", ".gzip"]:
ext = os.path.splitext(remainder)[1]
opened = gzip.open(filename, "wb")
else:
opened = open(filename, "wb")
if ext.lower() == ".csv":
if selection.all_keys:
headers = selection.all_keys
else:
# Have to do this to get all the headers :(
result = list(result)
all_headers = set()
for item in result:
all_headers.update(item.keys())
headers = list(all_headers)
with opened as ofile:
writer = csv.DictWriter(ofile,
fieldnames=headers,
extrasaction="ignore")
writer.writeheader()
for item in result:
count += 1
writer.writerow(item)
elif ext.lower() == ".json":
with opened as ofile:
for item in result:
count += 1
ofile.write(self._encoder.encode(item))
ofile.write("\n")
else:
with opened as ofile:
for item in result:
count += 1
pickle.dump(item, ofile)
return count
elif not selection.is_count:
result = order(selection.convert(item) for item in result)
return result