def test_no_sorting_default_to_modified_field(self):
token = self.resource._build_pagination_token([('last_modified', -1)],
self.record,
42)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertDictEqual(tokeninfo['last_record'],
{"last_modified": 1234})
def _extract_pagination_rules_from_token(self, limit, sorting):
"""Get pagination params."""
token = self.request.validated['querystring'].get('_token', None)
filters = []
offset = 0
if token:
error_msg = None
try:
tokeninfo = json.loads(decode64(token))
if not isinstance(tokeninfo, dict):
raise ValueError()
last_record = tokeninfo['last_record']
offset = tokeninfo['offset']
nonce = tokeninfo['nonce']
except (ValueError, KeyError, TypeError):
error_msg = '_token has invalid content'
# We don't want pagination tokens to be reused several times (#1171).
# The cache backend is used to keep track of "nonces".
if self.request.method.lower() == 'delete' and error_msg is None:
registry = self.request.registry
deleted = registry.cache.delete(nonce)
if deleted is None:
error_msg = '_token was already used or has expired.'
if error_msg:
error_details = {
'location': 'querystring',
'description': error_msg
}
raise_invalid(self.request, **error_details)
filters = self._build_pagination_rules(sorting, last_record)
return filters, offset
def test_disambiguate_fieldname_containing_dots(self):
token = self.resource._build_pagination_token([
('nested.other.subvalue', -1),
('title', 1),
], self.record, 88)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertEqual(tokeninfo['last_record'],
{'title': 'Title', 'nested.other.subvalue': 43})
def test_sorting_on_nested_field(self):
token = self.resource._build_pagination_token([
('nested.subvalue', -1),
('title', 1),
], self.record, 88)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertEqual(tokeninfo['last_record'],
{'title': 'Title', 'nested.subvalue': 42})
def test_strip_malformed_sort_field(self):
token = self.resource._build_pagination_token([
('non.existent', -1),
('title', 1),
], self.record, 88)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertEqual(tokeninfo['last_record'],
{'title': 'Title'})
def test_multiple_sorting_keep_all(self):
token = self.resource._build_pagination_token(
[("status", 1), ("title", -1), ("last_modified", -1)], self.record, 31
)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(
tokeninfo["last_record"], {"last_modified": 1234, "status": 2, "title": "Title"}
)
def test_sorting_handle_ordering_direction(self):
token = self.resource._build_pagination_token([
('status', 1),
('last_modified', 1)
], self.record, 32)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertEqual(tokeninfo['last_record'],
{"last_modified": 1234, "status": 2})
def delete(self, key):
query = 'DELETE FROM cache WHERE key = :key RETURNING value;'
with self.client.connect() as conn:
result = conn.execute(query, dict(key=self.prefix + key))
if result.rowcount > 0:
value = result.fetchone()['value']
return json.loads(value)
return None
def get(self, key):
purge = "DELETE FROM cache WHERE ttl IS NOT NULL AND now() > ttl;"
query = "SELECT value FROM cache WHERE key = :key;"
with self.client.connect() as conn:
conn.execute(purge)
result = conn.execute(query, dict(key=self.prefix + key))
if result.rowcount > 0:
value = result.fetchone()['value']
return json.loads(value)
def test_can_build_while_sorting_on_missing_field(self):
token = self.resource._build_pagination_token([
('unknown', 1),
('title', -1),
('last_modified', -1)
], self.record, 31)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertEqual(tokeninfo['last_record'],
{"last_modified": 1234, "title": "Title"})
def test_multiple_sorting_keep_all(self):
token = self.resource._build_pagination_token([
('status', 1),
('title', -1),
('last_modified', -1)
], self.record, 31)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertEqual(tokeninfo['last_record'],
{"last_modified": 1234, "status": 2,
'title': 'Title'})
def test_standard_entries_are_filled(self):
with mock.patch('kinto.core.utils.msec_time', return_value=12):
value = self.renderer(self.logger, 'debug', {})
log = json.loads(value)
self.assertDictEqual(log, {
'EnvVersion': '2.0',
'Hostname': os.uname()[1],
'Logger': '',
'Pid': os.getpid(),
'Severity': 7,
'Timestamp': 12000000,
'Type': '',
'Fields': {}
})
def get(self, key):
purge = """
DELETE FROM cache c
USING (
SELECT key
FROM cache
WHERE ttl IS NOT NULL AND now() > ttl
ORDER BY key ASC
FOR UPDATE
) del
WHERE del.key = c.key;"""
query = 'SELECT value FROM cache WHERE key = :key AND now() < ttl;'
with self.client.connect() as conn:
conn.execute(purge)
result = conn.execute(query, dict(key=self.prefix + key))
if result.rowcount > 0:
value = result.fetchone()['value']
return json.loads(value)
def test_returns_alert_if_eos_in_the_future(self):
# Set an end of service date in the future.
tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
with mock.patch.dict(self.app.app.registry.settings, [
('eos', tomorrow.isoformat()),
('eos_url', 'http://eos-url'),
('eos_message',
'This service will soon be decommissioned')]):
response = self.app.get('/')
# Requests should work as usual and contain an
# Alert header, with the service end of life information.
self.assertIn('Alert', response.headers)
self.assertDictEqual(json.loads(response.headers['Alert']), {
'code': 'soft-eol',
'url': 'http://eos-url',
'message': 'This service will soon be decommissioned'
})
def _extract_pagination_rules_from_token(self, limit, sorting):
"""Get pagination params."""
queryparams = self.request.GET
token = queryparams.get("_token", None)
filters = []
offset = 0
if token:
try:
tokeninfo = json.loads(decode64(token))
if not isinstance(tokeninfo, dict):
raise ValueError()
last_record = tokeninfo["last_record"]
offset = tokeninfo["offset"]
except (ValueError, KeyError, TypeError):
error_msg = "_token has invalid content"
error_details = {"location": "querystring", "description": error_msg}
raise_invalid(self.request, **error_details)
filters = self._build_pagination_rules(sorting, last_record)
return filters, offset
def test_returns_410_if_eos_in_the_past(self):
# Set an end of service date in the past.
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
with mock.patch.dict(self.app.app.registry.settings, [
('eos', yesterday.isoformat()),
('eos_url', 'http://eos-url'),
('eos_message',
'This service had been decommissioned')]):
response = self.app.get('/', status=410)
self.assertIn('Alert', response.headers)
self.assertDictEqual(json.loads(response.headers['Alert']), {
'code': 'hard-eol',
'url': 'http://eos-url',
'message': 'This service had been decommissioned'
})
self.assertDictEqual(response.json, {
"errno": ERRORS.SERVICE_DEPRECATED.value,
"message": "The service you are trying to connect no longer "
"exists at this location.",
"code": 410,
"error": "Gone"
})
def test_type_comes_from_structlog_event(self):
value = self.renderer(self.logger, 'debug', {'event': 'booh'})
log = json.loads(value)
self.assertEqual(log['Type'], 'booh')
def test_sorting_handle_both_rules(self):
token = self.resource._build_pagination_token(
[("status", -1), ("last_modified", -1)], self.record, 34
)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertDictEqual(tokeninfo["last_record"], {"last_modified": 1234, "status": 2})
def get(self, key):
value = self._client.get(self.prefix + key)
if value:
value = value.decode('utf-8')
return json.loads(value)
def test_list_of_string_values_are_not_serialized(self):
list_values = ['life', 'of', 'pi']
logged = self.renderer(self.logger, 'info', {'params': list_values})
log = json.loads(logged)
self.assertEqual(log['Fields']['params'], list_values)
def test_objects_values_are_serialized_as_string(self):
querystring = {'_sort': 'name'}
logged = self.renderer(self.logger, 'info', {'params': querystring})
log = json.loads(logged)
self.assertEqual(log['Fields']['params'], json.dumps(querystring))
def test_sorting_on_nested_field(self):
token = self.resource._build_pagination_token(
[("nested.subvalue", -1), ("title", 1)], self.record, 88
)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["last_record"], {"title": "Title", "nested.subvalue": 42})
def test_list_of_string_values_are_not_serialized(self):
list_values = ['life', 'of', 'pi']
logged = self.renderer(self.logger, 'info', {'params': list_values})
log = json.loads(logged)
self.assertEqual(log['Fields']['params'], list_values)
def test_list_of_homogeneous_values_are_serialized_as_string(self):
list_values = ['life', 'of', 'pi', 3.14]
logged = self.renderer(self.logger, 'info', {'params': list_values})
log = json.loads(logged)
self.assertEqual(log['Fields']['params'], json.dumps(list_values))
def test_objects_values_are_serialized_as_string(self):
querystring = {'_sort': 'name'}
logged = self.renderer(self.logger, 'info', {'params': querystring})
log = json.loads(logged)
self.assertEqual(log['Fields']['params'], json.dumps(querystring))
def test_fields_can_be_provided_directly(self):
value = self.renderer(self.logger, 'critical', {'Fields': {'win': 11}})
log = json.loads(value)
self.assertEqual(log['Fields'], {'win': 11})
def test_unknown_fields_are_moved_to_fields_entry(self):
value = self.renderer(self.logger, 'critical', {'win': 11})
log = json.loads(value)
self.assertEqual(log['Fields'], {'win': 11})
def test_severity_comes_from_logger_name(self):
value = self.renderer(self.logger, 'critical', {})
log = json.loads(value)
self.assertEqual(log['Severity'], 0)
def test_unknown_fields_are_moved_to_fields_entry(self):
value = self.renderer(self.logger, 'critical', {'win': 11})
log = json.loads(value)
self.assertEqual(log['Fields'], {'win': 11})
def test_fields_can_be_provided_directly(self):
value = self.renderer(self.logger, 'critical', {'Fields': {'win': 11}})
log = json.loads(value)
self.assertEqual(log['Fields'], {'win': 11})
def test_strip_malformed_sort_field(self):
token = self.resource._build_pagination_token(
[("non.existent", -1), ("title", 1)], self.record, 88
)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["last_record"], {"title": "Title"})
def test_list_of_homogeneous_values_are_serialized_as_string(self):
list_values = ['life', 'of', 'pi', 3.14]
logged = self.renderer(self.logger, 'info', {'params': list_values})
log = json.loads(logged)
self.assertEqual(log['Fields']['params'], json.dumps(list_values))
def test_no_sorting_default_to_modified_field(self):
token = self.resource._build_pagination_token([('last_modified', -1)],
self.record, 42)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertDictEqual(tokeninfo['last_record'], {"last_modified": 1234})
def _get(self, key):
value = self._client.get(self.prefix + key)
if not value:
return None, 0
data = json.loads(value)
return data["value"], data["ttl"]
def test_strip_malformed_sort_field(self):
token = self.resource._build_pagination_token([("non.existent", -1),
("title", 1)],
self.record, 88)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["last_record"], {"title": "Title"})
def _get(self, key):
value = self._client.get(self.prefix + key)
if not value:
return None, 0
data = json.loads(value)
return data['value'], data['ttl']
def test_sorting_on_nested_field(self):
token = self.resource._build_pagination_token(
[("nested.subvalue", -1), ("title", 1)], self.obj, 88
)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["last_object"], {"title": "Title", "nested.subvalue": 42})
def test_token_contains_current_offset(self):
token = self.resource._build_pagination_token([('last_modified', -1)],
self.record, 42)
tokeninfo = json.loads(b64decode(token).decode('ascii'))
self.assertEqual(tokeninfo['offset'], 42)
def test_type_comes_from_structlog_event(self):
value = self.renderer(self.logger, 'debug', {'event': 'booh'})
log = json.loads(value)
self.assertEqual(log['Type'], 'booh')
def test_token_contains_current_offset(self):
token = self.resource._build_pagination_token([("last_modified", -1)], self.record, 42)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["offset"], 42)
def test_severity_comes_from_logger_name(self):
value = self.renderer(self.logger, 'critical', {})
log = json.loads(value)
self.assertEqual(log['Severity'], 0)
def test_sorting_handle_ordering_direction(self):
token = self.resource._build_pagination_token(
[("status", 1), ("last_modified", 1)], self.obj, 32
)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["last_object"], {"last_modified": 1234, "status": 2})
def test_standard_entries_are_not_overwritten(self):
value = self.renderer(self.logger, 'debug', {'Hostname': 'her'})
log = json.loads(value)
self.assertEqual(log['Hostname'], 'her')
def test_disambiguate_fieldname_containing_dots(self):
token = self.resource._build_pagination_token(
[("nested.other.subvalue", -1), ("title", 1)], self.record, 88
)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["last_record"], {"title": "Title", "nested.other.subvalue": 43})
def test_token_contains_current_offset(self):
token = self.resource._build_pagination_token([("last_modified", -1)], self.obj, 42)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["offset"], 42)
def test_can_build_while_sorting_on_missing_field(self):
token = self.resource._build_pagination_token(
[("unknown", 1), ("title", -1), ("last_modified", -1)], self.record, 31
)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertEqual(tokeninfo["last_record"], {"last_modified": 1234, "title": "Title"})
def test_no_sorting_default_to_modified_field(self):
token = self.resource._build_pagination_token([("last_modified", -1)], self.obj, 42)
tokeninfo = json.loads(b64decode(token).decode("ascii"))
self.assertDictEqual(tokeninfo["last_object"], {"last_modified": 1234})