def deserialize(self, cstruct=colander.null):
"""
Deserialize and validate the QuerySchema fields and try to deserialize and
get the native value of additional filds (field filters) that may be present
on the cstruct.
e.g:: ?exclude_id=a,b&deleted=true -> {'exclude_id': ['a', 'b'], deleted: True}
"""
values = {}
schema_values = super().deserialize(cstruct)
if schema_values is colander.drop:
return schema_values
# Deserialize querystring field filters (see docstring e.g)
for k, v in cstruct.items():
# Deserialize lists used on contains_ and contains_any_ filters
if k.startswith('contains_'):
as_list = native_value(v)
if not isinstance(as_list, list):
values[k] = [as_list]
else:
values[k] = as_list
# Deserialize lists used on in_ and exclude_ filters
elif k.startswith('in_') or k.startswith('exclude_'):
as_list = FieldList().deserialize(v)
values[k] = [native_value(v) for v in as_list]
else:
values[k] = native_value(v)
values.update(schema_values)
return values
def deserialize(self, cstruct=colander.null):
"""
Deserialize and validate the QuerySchema fields and try to deserialize and
get the native value of additional filds (field filters) that may be present
on the cstruct.
e.g:: ?exclude_id=a,b&deleted=true -> {'exclude_id': ['a', 'b'], deleted: True}
"""
values = {}
schema_values = super().deserialize(cstruct)
if schema_values is colander.drop:
return schema_values
# Deserialize querystring field filters (see docstring e.g)
for k, v in cstruct.items():
# Deserialize lists used on contains_ and contains_any_ filters
if k.startswith("contains_"):
as_list = native_value(v)
if not isinstance(as_list, list):
values[k] = [as_list]
else:
values[k] = as_list
# Deserialize lists used on in_ and exclude_ filters
elif k.startswith("in_") or k.startswith("exclude_"):
as_list = FieldList().deserialize(v)
values[k] = [native_value(v) for v in as_list]
else:
values[k] = native_value(v)
values.update(schema_values)
return values
def delete(self):
"""Record ``DELETE`` endpoint: delete a record and return it.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if
the record is not found.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and record modified
in the iterim.
"""
self._raise_400_if_invalid_id(self.record_id)
record = self._get_record_or_404(self.record_id)
self._raise_412_if_modified(record)
# Retreive the last_modified information from a querystring if present.
last_modified = self.request.GET.get('last_modified')
if last_modified:
last_modified = native_value(last_modified.strip('"'))
if not isinstance(last_modified, six.integer_types):
error_details = {
'name': 'last_modified',
'location': 'querystring',
'description': 'Invalid value for %s' % last_modified
}
raise_invalid(self.request, **error_details)
# If less or equal than current record. Ignore it.
if last_modified <= record[self.model.modified_field]:
last_modified = None
deleted = self.model.delete_record(record, last_modified=last_modified)
return self.postprocess(deleted, action=ACTIONS.DELETE, old=record)
def delete(self):
"""Record ``DELETE`` endpoint: delete a record and return it.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if
the record is not found.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and record modified
in the iterim.
"""
self._raise_400_if_invalid_id(self.record_id)
record = self._get_record_or_404(self.record_id)
self._raise_412_if_modified(record)
# Retreive the last_modified information from a querystring if present.
last_modified = self.request.GET.get('last_modified')
if last_modified:
last_modified = native_value(last_modified.strip('"'))
if not isinstance(last_modified, six.integer_types):
error_details = {
'name': 'last_modified',
'location': 'querystring',
'description': 'Invalid value for %s' % last_modified
}
raise_invalid(self.request, **error_details)
# If less or equal than current record. Ignore it.
if last_modified <= record[self.model.modified_field]:
last_modified = None
deleted = self.model.delete_record(record, last_modified=last_modified)
return self.postprocess(deleted, action=ACTIONS.DELETE)
def load_default_settings(config, default_settings):
"""Read settings provided in Paste ini file, set default values and
replace if defined as environment variable.
"""
settings = config.get_settings()
settings_prefix = settings["settings_prefix"]
def _prefixed_keys(key):
unprefixed = key
if key.startswith(settings_prefix + "."):
unprefixed = key.split(".", 1)[1]
project_prefix = f"{settings_prefix}.{unprefixed}"
return unprefixed, project_prefix
# Fill settings with default values if not defined.
for key, default_value in sorted(default_settings.items()):
unprefixed, project_prefix = keys = _prefixed_keys(key)
is_defined = len(set(settings.keys()).intersection(set(keys))) > 0
if not is_defined:
settings[unprefixed] = default_value
for key, value in sorted(settings.items()):
value = utils.native_value(value)
unprefixed, project_prefix = keys = _prefixed_keys(key)
# Fail if not only one is defined.
defined = set(settings.keys()).intersection(set(keys))
distinct_values = set([str(settings[d]) for d in defined])
if len(defined) > 1 and len(distinct_values) > 1:
names = "', '".join(defined)
raise ValueError(f"Settings '{names}' are in conflict.")
# Override settings from OS env values.
# e.g. HTTP_PORT, READINGLIST_HTTP_PORT, KINTO_HTTP_PORT
from_env = utils.read_env(unprefixed, value)
from_env = utils.read_env(project_prefix, from_env)
settings[unprefixed] = from_env
config.add_settings(settings)
def test_non_string_values(self):
self.assertEqual(native_value(7), 7)
self.assertEqual(native_value(True), True)
def test_zero_and_one_coerce_to_integers(self):
self.assertEqual(native_value('1'), 1)
self.assertEqual(native_value('0'), 0)
def test_true_values(self):
true_strings = ['True', 'on', 'true', 'yes']
true_values = [native_value(s) for s in true_strings]
self.assertTrue(all(true_values))
def test_infinite_strings(self):
self.assertEqual(native_value("803E5708"), "803E5708")
self.assertEqual(native_value("Infinity"), "Infinity")
def test_false_values(self):
false_strings = ["false"]
false_values = [native_value(s) for s in false_strings]
self.assertFalse(any(false_values))
def test_zero_and_one_coerce_to_integers(self):
self.assertEqual(native_value("1"), 1)
self.assertEqual(native_value("0"), 0)
def test_null_value(self):
self.assertEqual(native_value('null'), None)
def test_false_values(self):
false_strings = ['False', 'off', 'false', 'no']
false_values = [native_value(s) for s in false_strings]
self.assertFalse(any(false_values))
def test_simple_string(self):
self.assertEqual(native_value('value'), 'value')
def test_bad_string_values(self):
self.assertEqual(native_value("\u0000"), "\x00")
def test_true_values(self):
true_strings = ['True', 'on', 'true', 'yes']
true_values = [native_value(s) for s in true_strings]
self.assertTrue(all(true_values))
def test_defined_simple_quote_string_as_text_value(self):
self.assertEqual(native_value("'7'"), "'7'")
def test_defined_integer_as_text_value(self):
self.assertEqual(native_value('"7"'), '7')
def test_defined_null_as_text_value(self):
self.assertEqual(native_value('"null"'), 'null')
def test_simple_string(self):
self.assertEqual(native_value("value"), "value")
def test_true_values(self):
true_strings = ["true"]
true_values = [native_value(s) for s in true_strings]
self.assertTrue(all(true_values))
def test_defined_simple_quote_string_as_text_value(self):
self.assertEqual(native_value("'7'"), "'7'")
def test_non_string_values(self):
self.assertEqual(native_value(7), 7)
self.assertEqual(native_value(True), True)
def test_float(self):
self.assertEqual(native_value("3.14"), 3.14)
def test_defined_string(self):
self.assertEqual(native_value('"value"'), 'value')
def test_false_values(self):
false_strings = ["false"]
false_values = [native_value(s) for s in false_strings]
self.assertFalse(any(false_values))
def _extract_filters(self, queryparams=None):
"""Extracts filters from QueryString parameters."""
if not queryparams:
queryparams = self.request.GET
filters = []
for param, paramvalue in queryparams.items():
param = param.strip()
error_details = {
'name': param,
'location': 'querystring',
'description': 'Invalid value for %s' % param
}
# Ignore specific fields
if param.startswith('_') and param not in ('_since',
'_to',
'_before'):
continue
# Handle the _since specific filter.
if param in ('_since', '_to', '_before'):
value = native_value(paramvalue.strip('"'))
if not isinstance(value, six.integer_types):
raise_invalid(self.request, **error_details)
if param == '_since':
operator = COMPARISON.GT
else:
if param == '_to':
message = ('_to is now deprecated, '
'you should use _before instead')
url = ('https://kinto.readthedocs.io/en/2.4.0/api/'
'resource.html#list-of-available-url-'
'parameters')
send_alert(self.request, message, url)
operator = COMPARISON.LT
filters.append(
Filter(self.model.modified_field, value, operator)
)
continue
allKeywords = '|'.join([i.name.lower() for i in COMPARISON])
m = re.match(r'^('+allKeywords+')_([\w\.]+)$', param)
if m:
keyword, field = m.groups()
operator = getattr(COMPARISON, keyword.upper())
else:
operator, field = COMPARISON.EQ, param
if not self.is_known_field(field):
error_msg = "Unknown filter field '{0}'".format(param)
error_details['description'] = error_msg
raise_invalid(self.request, **error_details)
value = native_value(paramvalue)
if operator in (COMPARISON.IN, COMPARISON.EXCLUDE):
value = set([native_value(v) for v in paramvalue.split(',')])
all_integers = all([isinstance(v, six.integer_types)
for v in value])
all_strings = all([isinstance(v, six.text_type)
for v in value])
has_invalid_value = (
(field == self.model.id_field and not all_strings) or
(field == self.model.modified_field and not all_integers)
)
if has_invalid_value:
raise_invalid(self.request, **error_details)
filters.append(Filter(field, value, operator))
return filters
def test_bad_string_values(self):
self.assertEqual(native_value("\u0000"), "\x00")
def test_simple_string(self):
self.assertEqual(native_value('value'), 'value')
def _extract_filters(self, queryparams=None):
"""Extracts filters from QueryString parameters."""
if not queryparams:
queryparams = self.request.GET
filters = []
for param, paramvalue in queryparams.items():
param = param.strip()
error_details = {
'name': param,
'location': 'querystring',
'description': 'Invalid value for %s' % param
}
# Ignore specific fields
if param.startswith('_') and param not in ('_since',
'_to',
'_before'):
continue
# Handle the _since specific filter.
if param in ('_since', '_to', '_before'):
value = native_value(paramvalue.strip('"'))
if not isinstance(value, six.integer_types):
raise_invalid(self.request, **error_details)
if param == '_since':
operator = COMPARISON.GT
else:
if param == '_to':
message = ('_to is now deprecated, '
'you should use _before instead')
url = ('https://kinto.readthedocs.io/en/2.4.0/api/'
'resource.html#list-of-available-url-'
'parameters')
send_alert(self.request, message, url)
operator = COMPARISON.LT
filters.append(
Filter(self.model.modified_field, value, operator)
)
continue
m = re.match(r'^(min|max|not|lt|gt|in|exclude)_(\w+)$', param)
if m:
keyword, field = m.groups()
operator = getattr(COMPARISON, keyword.upper())
else:
operator, field = COMPARISON.EQ, param
if not self.is_known_field(field):
error_msg = "Unknown filter field '{0}'".format(param)
error_details['description'] = error_msg
raise_invalid(self.request, **error_details)
value = native_value(paramvalue)
if operator in (COMPARISON.IN, COMPARISON.EXCLUDE):
value = set([native_value(v) for v in paramvalue.split(',')])
all_integers = all([isinstance(v, six.integer_types)
for v in value])
all_strings = all([isinstance(v, six.text_type)
for v in value])
has_invalid_value = (
(field == self.model.id_field and not all_strings) or
(field == self.model.modified_field and not all_integers)
)
if has_invalid_value:
raise_invalid(self.request, **error_details)
filters.append(Filter(field, value, operator))
return filters
def test_true_values(self):
true_strings = ["true"]
true_values = [native_value(s) for s in true_strings]
self.assertTrue(all(true_values))
def deserialize(self, cstruct=colander.null):
if isinstance(cstruct, str):
cstruct = native_value(cstruct)
return super(QueryField, self).deserialize(cstruct)
def test_simple_string(self):
self.assertEqual(native_value("value"), "value")
def deserialize(self, cstruct=colander.null):
if isinstance(cstruct, str):
cstruct = native_value(cstruct)
return super(QueryField, self).deserialize(cstruct)
def test_defined_string(self):
self.assertEqual(native_value('"value"'), "value")
def test_integer(self):
self.assertEqual(native_value('7'), 7)
def test_null_value(self):
self.assertEqual(native_value("null"), None)
def test_float(self):
self.assertEqual(native_value('3.14'), 3.14)
def test_defined_null_as_text_value(self):
self.assertEqual(native_value('"null"'), "null")
def test_false_values(self):
false_strings = ['False', 'off', 'false', 'no']
false_values = [native_value(s) for s in false_strings]
self.assertFalse(any(false_values))
def test_integer(self):
self.assertEqual(native_value("7"), 7)
def test_defined_integer_as_text_value(self):
self.assertEqual(native_value('"7"'), "7")
def _extract_filters(self, queryparams=None):
"""Extracts filters from QueryString parameters."""
if not queryparams:
queryparams = self.request.GET
filters = []
for param, paramvalue in queryparams.items():
param = param.strip()
error_details = {"name": param, "location": "querystring", "description": "Invalid value for %s" % param}
# Ignore specific fields
if param.startswith("_") and param not in ("_since", "_to", "_before"):
continue
# Handle the _since specific filter.
if param in ("_since", "_to", "_before"):
value = native_value(paramvalue.strip('"'))
if not isinstance(value, six.integer_types):
raise_invalid(self.request, **error_details)
if param == "_since":
operator = COMPARISON.GT
else:
if param == "_to":
message = "_to is now deprecated, " "you should use _before instead"
url = (
"https://kinto.readthedocs.io/en/2.4.0/api/"
"resource.html#list-of-available-url-"
"parameters"
)
send_alert(self.request, message, url)
operator = COMPARISON.LT
filters.append(Filter(self.model.modified_field, value, operator))
continue
allKeywords = "|".join([i.name.lower() for i in COMPARISON])
m = re.match(r"^(" + allKeywords + ")_([\w\.]+)$", param)
if m:
keyword, field = m.groups()
operator = getattr(COMPARISON, keyword.upper())
else:
operator, field = COMPARISON.EQ, param
if not self.is_known_field(field):
error_msg = "Unknown filter field '{0}'".format(param)
error_details["description"] = error_msg
raise_invalid(self.request, **error_details)
value = native_value(paramvalue)
if operator in (COMPARISON.IN, COMPARISON.EXCLUDE):
value = set([native_value(v) for v in paramvalue.split(",")])
all_integers = all([isinstance(v, six.integer_types) for v in value])
all_strings = all([isinstance(v, six.text_type) for v in value])
has_invalid_value = (field == self.model.id_field and not all_strings) or (
field == self.model.modified_field and not all_integers
)
if has_invalid_value:
raise_invalid(self.request, **error_details)
filters.append(Filter(field, value, operator))
return filters