def get_unicity_rules(collection_id, parent_id, record, unique_fields,
id_field, for_creation):
"""Build filter to target existing records that violate the resource
unicity rules on fields.
:returns: a list of list of filters
"""
rules = []
for field in set(unique_fields):
value = record.get(field)
# None values cannot be considered unique.
if value is None:
continue
filters = [Filter(field, value, COMPARISON.EQ)]
if not for_creation:
object_id = record[id_field]
exclude = Filter(id_field, object_id, COMPARISON.NOT)
filters.append(exclude)
rules.append(filters)
return rules
def test_paginated_fetches_next_page(self):
objects = self.sample_objects
objects.reverse()
def list_all_mock(*args, **kwargs):
this_objects = objects[:3]
del objects[:3]
return this_objects
self.storage.list_all.side_effect = list_all_mock
list(paginated(self.storage, sorting=[Sort("id", -1)]))
assert self.storage.list_all.call_args_list == [
mock.call(sorting=[Sort("id", -1)],
limit=25,
pagination_rules=None),
mock.call(
sorting=[Sort("id", -1)],
limit=25,
pagination_rules=[[Filter("id", "object-03", COMPARISON.LT)]],
),
mock.call(
sorting=[Sort("id", -1)],
limit=25,
pagination_rules=[[Filter("id", "object-01", COMPARISON.LT)]],
),
]
def test_paginated_fetches_next_page(self):
records = self.sample_records
records.reverse()
def get_all_mock(*args, **kwargs):
this_records = records[:3]
del records[:3]
return this_records, len(this_records)
self.storage.get_all.side_effect = get_all_mock
list(paginated(self.storage, sorting=[Sort('id', -1)]))
assert self.storage.get_all.call_args_list == [
mock.call(sorting=[Sort('id', -1)],
limit=25,
pagination_rules=None),
mock.call(
sorting=[Sort('id', -1)],
limit=25,
pagination_rules=[[Filter('id', 'record-03', COMPARISON.LT)]]),
mock.call(
sorting=[Sort('id', -1)],
limit=25,
pagination_rules=[[Filter('id', 'record-01', COMPARISON.LT)]]),
]
def _build_pagination_rules(self, sorting, last_record, rules=None):
"""Return the list of rules for a given sorting attribute and
last_record.
"""
if rules is None:
rules = []
rule = []
next_sorting = sorting[:-1]
for field, _ in next_sorting:
rule.append(Filter(field, last_record.get(field), COMPARISON.EQ))
field, direction = sorting[-1]
if direction == -1:
rule.append(Filter(field, last_record.get(field), COMPARISON.LT))
else:
rule.append(Filter(field, last_record.get(field), COMPARISON.GT))
rules.append(rule)
if len(next_sorting) == 0:
return rules
return self._build_pagination_rules(next_sorting, last_record, rules)
def test_get_all_can_filter_with_none_values(self):
self.create_record({"name": "Alexis"})
self.create_record({"title": "haha"})
self.create_record({"name": "Mathieu"})
filters = [Filter("name", "Fanny", utils.COMPARISON.GT)]
records, _ = self.storage.get_all(filters=filters, **self.storage_kw)
self.assertEqual(len(records), 1) # None is not greater than "Fanny"
self.assertEqual(records[0]["name"], "Mathieu")
filters = [Filter("name", "Fanny", utils.COMPARISON.LT)]
records, _ = self.storage.get_all(filters=filters, **self.storage_kw)
self.assertEqual(len(records), 2) # None is less than "Fanny"
def test_get_all_can_filter_with_numeric_id(self):
for l in [0, 42]:
self.create_record({'id': str(l)})
filters = [Filter('id', 0, utils.COMPARISON.EQ)]
records, _ = self.storage.get_all(filters=filters,
**self.storage_kw)
self.assertEqual(len(records), 1)
filters = [Filter('id', 42, utils.COMPARISON.EQ)]
records, _ = self.storage.get_all(filters=filters,
**self.storage_kw)
self.assertEqual(len(records), 1)
def test_get_all_handle_all_pagination_rules(self):
for x in range(10):
record = dict(self.record)
record["number"] = x % 3
last_record = self.create_record(record)
records, total_records = self.storage.get_all(
limit=5, pagination_rules=[
[Filter('number', 1, utils.COMPARISON.GT)],
[Filter('id', last_record['id'], utils.COMPARISON.EQ)],
], **self.storage_kw)
self.assertEqual(total_records, 10)
self.assertEqual(len(records), 4)
def test_get_all_can_filter_with_numeric_strings(self):
for l in ["0566199093", "0781566199"]:
self.create_record({'phone': l})
filters = [Filter('phone', "0566199093", utils.COMPARISON.EQ)]
records, _ = self.storage.get_all(filters=filters,
**self.storage_kw)
self.assertEqual(len(records), 1)
def test_get_all_can_filter_with_float_values(self):
for l in [10, 11.5, 8.5, 6, 7.5]:
self.create_record({'note': l})
filters = [Filter('note', 9.5, utils.COMPARISON.LT)]
records, _ = self.storage.get_all(filters=filters,
**self.storage_kw)
self.assertEqual(len(records), 3)
def test_get_all_can_filter_by_subobjects_values(self):
for l in ['a', 'b', 'c']:
self.create_record({'code': {'sub': l}})
filters = [Filter('code.sub', 'a', utils.COMPARISON.EQ)]
records, _ = self.storage.get_all(filters=filters,
**self.storage_kw)
self.assertEqual(len(records), 1)
def paginated(storage, *args, sorting, batch_size=BATCH_SIZE, **kwargs):
"""A generator used to access paginated results from storage.get_all.
:param kwargs: Passed through unchanged to get_all.
"""
if len(sorting) > 1:
raise NotImplementedError("FIXME: only supports one-length sorting") # pragma: nocover
pagination_direction = COMPARISON.GT if sorting[0].direction > 0 else COMPARISON.LT
record_pagination = None
while True:
(records, _) = storage.get_all(
sorting=sorting, limit=batch_size, pagination_rules=record_pagination, **kwargs
)
if not records:
break
for record in records:
yield record
record_pagination = [
# FIXME: support more than one-length sorting
[Filter(sorting[0].field, record[sorting[0].field], pagination_direction)]
]
def _get_records(self, rc, last_modified=None):
# If last_modified was specified, only retrieve items since then.
storage_kwargs = {}
if last_modified is not None:
gt_last_modified = Filter(FIELD_LAST_MODIFIED, last_modified,
COMPARISON.GT)
storage_kwargs['filters'] = [
gt_last_modified,
]
storage_kwargs['sorting'] = [Sort(FIELD_LAST_MODIFIED, 1)]
parent_id = "/buckets/{bucket}/collections/{collection}".format(**rc)
records, count = self.storage.get_all(parent_id=parent_id,
collection_id='record',
include_deleted=True,
**storage_kwargs)
if len(records) == count == 0:
# When the collection empty (no records and no tombstones)
collection_timestamp = None
else:
collection_timestamp = self.storage.collection_timestamp(
parent_id=parent_id, collection_id='record')
return records, collection_timestamp
def test_get_all_raises_if_missing_on_strange_query(self):
with self.assertRaises(ValueError):
self.storage.get_all(
"some-collection",
"some-parent",
filters=[Filter("author", MISSING, COMPARISON.HAS)],
)
def put(self):
"""Record ``PUT`` endpoint: create or replace the provided record and
return it.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and record modified
in the iterim.
.. note::
If ``If-None-Match: *`` request header is provided, the
``PUT`` will succeed only if no record exists with this id.
.. seealso::
Add custom behaviour by overriding
:meth:`kinto.core.resource.UserResource.process_record`.
"""
self._raise_400_if_invalid_id(self.record_id)
id_field = self.model.id_field
existing = None
tombstones = None
try:
existing = self._get_record_or_404(self.record_id)
except HTTPNotFound:
# Look if this record used to exist (for preconditions check).
filter_by_id = Filter(id_field, self.record_id, COMPARISON.EQ)
tombstones, _ = self.model.get_records(filters=[filter_by_id],
include_deleted=True)
if len(tombstones) > 0:
existing = tombstones[0]
finally:
if existing:
self._raise_412_if_modified(existing)
post_record = self.request.validated['data']
record_id = post_record.setdefault(id_field, self.record_id)
self._raise_400_if_id_mismatch(record_id, self.record_id)
new_record = self.process_record(post_record, old=existing)
try:
unique = self.mapping.get_option('unique_fields')
if existing and not tombstones:
record = self.model.update_record(new_record,
unique_fields=unique)
else:
record = self.model.create_record(new_record,
unique_fields=unique)
self.request.response.status_code = 201
except storage_exceptions.UnicityError as e:
self._raise_conflict(e)
timestamp = record[self.model.modified_field]
self._add_timestamp_header(self.request.response, timestamp=timestamp)
action = existing and ACTIONS.UPDATE or ACTIONS.CREATE
return self.postprocess(record, action=action, old=existing)
def test_delete_all_can_delete_partially(self):
self.create_record({'foo': 'po'})
self.create_record()
filters = [Filter('foo', 'bar', utils.COMPARISON.EQ)]
self.storage.delete_all(filters=filters, **self.storage_kw)
_, count = self.storage.get_all(**self.storage_kw)
self.assertEqual(count, 1)
def test_get_all_can_filter_with_list_of_excluded_values(self):
for l in ['a', 'b', 'c']:
self.create_record({'code': l})
filters = [Filter('code', ('a', 'b'), utils.COMPARISON.EXCLUDE)]
records, _ = self.storage.get_all(filters=filters,
**self.storage_kw)
self.assertEqual(len(records), 1)
def test_delete_all_supports_pagination_rules(self):
for i in range(6):
self.create_record({'foo': i})
pagination_rules = [[Filter('foo', 3, utils.COMPARISON.GT)]]
deleted = self.storage.delete_all(limit=4, pagination_rules=pagination_rules,
**self.storage_kw)
self.assertEqual(len(deleted), 2)
def test_records_filtered_when_searched_by_string_field(self):
self.create_record({'name': 'foo'})
self.create_record({'name': 'bar'})
self.create_record({'name': 'FOOBAR'})
filters = [Filter('name', 'FoO', utils.COMPARISON.LIKE)]
results, count = self.storage.get_all(filters=filters, **self.storage_kw)
self.assertEqual(len(results), 2)
def test_get_all_can_filter_with_list_of_values_on_id(self):
record1 = self.create_record({'code': 'a'})
record2 = self.create_record({'code': 'b'})
filters = [Filter('id', [record1['id'], record2['id']],
utils.COMPARISON.IN)]
records, _ = self.storage.get_all(filters=filters,
**self.storage_kw)
self.assertEqual(len(records), 2)
def test_return_empty_set_if_filtering_on_deleted_without_include(self):
self.create_record()
self.create_and_delete_record()
filters = [Filter('deleted', True, utils.COMPARISON.EQ)]
records, count = self.storage.get_all(filters=filters,
**self.storage_kw)
self.assertEqual(len(records), 0)
self.assertEqual(count, 0)
def _extract_filters(self):
filters = super()._extract_filters()
filters_str_id = []
for filt in filters:
if filt.field in ("record_id", "collection_id", "bucket_id"):
if isinstance(filt.value, int):
filt = Filter(filt.field, str(filt.value), filt.operator)
filters_str_id.append(filt)
return filters_str_id
def _extract_filters(self, queryparams=None):
filters = super(History, self)._extract_filters(queryparams)
filters_str_id = []
for filt in filters:
if filt.field in ('record_id', 'collection_id', 'bucket_id'):
if isinstance(filt.value, int):
filt = Filter(filt.field, str(filt.value), filt.operator)
filters_str_id.append(filt)
return filters_str_id
def test_get_all_can_filter_with_strings(self):
for l in ["Rémy", "Alexis", "Marie"]:
self.create_record({'name': l})
sorting = [Sort('name', 1)]
filters = [Filter('name', "Mathieu", utils.COMPARISON.LT)]
records, _ = self.storage.get_all(sorting=sorting, filters=filters,
**self.storage_kw)
self.assertEqual(records[0]['name'], "Alexis")
self.assertEqual(records[1]['name'], "Marie")
self.assertEqual(len(records), 2)
def test_filtering_on_arbitrary_field_excludes_deleted_records(self):
filters = self._get_last_modified_filters()
self.create_record({'status': 0})
self.create_and_delete_record({'status': 0})
filters += [Filter('status', 0, utils.COMPARISON.EQ)]
records, count = self.storage.get_all(filters=filters,
include_deleted=True,
**self.storage_kw)
self.assertEqual(len(records), 1)
self.assertEqual(count, 1)
def get_records(request, prefix, collection):
resources = request.registry.amo_resources
parent_id = PARENT_PATTERN.format(**resources[prefix][collection])
cid = "record"
records, count = request.registry.storage.get_all(
collection_id=cid,
parent_id=parent_id,
filters=[Filter('enabled', True, utils.COMPARISON.EQ)],
sorting=[Sort('last_modified', 1)])
last_modified = records[-1]['last_modified'] if count > 1 else 0
return records, last_modified
def test_get_all_can_filter_with_numeric_values(self):
self.create_record({'missing': 'code'})
for l in [1, 10, 6, 46]:
self.create_record({'code': l})
sorting = [Sort('code', 1)]
filters = [Filter('code', 10, utils.COMPARISON.MAX)]
records, _ = self.storage.get_all(sorting=sorting, filters=filters,
**self.storage_kw)
self.assertEqual(records[0]['code'], 1)
self.assertEqual(records[1]['code'], 6)
self.assertEqual(records[2]['code'], 10)
self.assertEqual(len(records), 3)
filters = [Filter('code', 10, utils.COMPARISON.LT)]
records, _ = self.storage.get_all(sorting=sorting, filters=filters,
**self.storage_kw)
self.assertEqual(records[0]['code'], 1)
self.assertEqual(records[1]['code'], 6)
self.assertEqual(len(records), 2)
def test_return_empty_set_if_filtering_on_deleted_false(self):
filters = self._get_last_modified_filters()
self.create_record()
self.create_and_delete_record()
filters += [Filter('deleted', False, utils.COMPARISON.EQ)]
records, count = self.storage.get_all(filters=filters,
include_deleted=True,
**self.storage_kw)
self.assertEqual(len(records), 0)
self.assertEqual(count, 0)
def test_support_filtering_out_on_deleted_field(self):
filters = self._get_last_modified_filters()
self.create_record()
self.create_and_delete_record()
filters += [Filter('deleted', True, utils.COMPARISON.NOT)]
records, count = self.storage.get_all(filters=filters,
include_deleted=True,
**self.storage_kw)
self.assertEqual(count, 1)
self.assertNotIn('deleted', records[0])
self.assertEqual(len(records), 1)
def test_get_source_records_asks_storage_for_last_modified_records(self):
records = []
count = mock.sentinel.count
self.storage.get_all.return_value = (records, count)
self.updater.get_source_records(1234)
self.storage.get_all.assert_called_with(
collection_id='record',
parent_id='/buckets/sourcebucket/collections/sourcecollection',
include_deleted=True,
filters=[Filter('last_modified', 1234, COMPARISON.GT)],
sorting=[Sort('last_modified', 1)])
def test_number_of_fetched_records_is_per_page(self):
for i in range(10):
self.create_record({'number': i})
settings = {**self.settings, 'storage_max_fetch_size': 2}
config = self._get_config(settings=settings)
backend = self.backend.load_from_config(config)
results, count = backend.get_all(
pagination_rules=[[Filter('number', 1, COMPARISON.GT)]],
**self.storage_kw)
self.assertEqual(count, 10)
self.assertEqual(len(results), 2)
