def get(self):
"""Record ``GET`` endpoint: retrieve a record.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if
the record is not found.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if
``If-None-Match`` header is provided and record not
modified in the interim.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and record modified
in the iterim.
"""
self._raise_400_if_invalid_id(self.record_id)
record = self._get_record_or_404(self.record_id)
timestamp = record[self.model.modified_field]
self._add_timestamp_header(self.request.response, timestamp=timestamp)
self._add_cache_header(self.request.response)
self._raise_304_if_not_modified(record)
self._raise_412_if_modified(record)
partial_fields = self._extract_partial_fields()
if partial_fields:
record = dict_subset(record, partial_fields)
return self.postprocess(record)
def get(self):
"""Object ``GET`` endpoint: retrieve an object.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if
the object is not found.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if
``If-None-Match`` header is provided and object not
modified in the interim.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and object modified
in the iterim.
"""
self._raise_400_if_invalid_id(self.object_id)
obj = self._get_object_or_404(self.object_id)
timestamp = obj[self.model.modified_field]
self._add_timestamp_header(self.request.response, timestamp=timestamp)
self._add_cache_header(self.request.response)
self._raise_304_if_not_modified(obj)
self._raise_412_if_modified(obj)
partial_fields = self._extract_partial_fields()
if partial_fields:
obj = dict_subset(obj, partial_fields)
return self.postprocess(obj)
def get(self):
"""Record ``GET`` endpoint: retrieve a record.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if
the record is not found.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if
``If-None-Match`` header is provided and record not
modified in the interim.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and record modified
in the iterim.
"""
self._raise_400_if_invalid_id(self.record_id)
record = self._get_record_or_404(self.record_id)
timestamp = record[self.model.modified_field]
self._add_timestamp_header(self.request.response, timestamp=timestamp)
self._add_cache_header(self.request.response)
self._raise_304_if_not_modified(record)
self._raise_412_if_modified(record)
partial_fields = self._extract_partial_fields()
if partial_fields:
record = dict_subset(record, partial_fields)
return self.postprocess(record)
def collection_get(self):
"""Model ``GET`` endpoint: retrieve multiple records.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if
``If-None-Match`` header is provided and collection not
modified in the interim.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and collection modified
in the iterim.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPBadRequest`
if filters or sorting are invalid.
"""
self._add_timestamp_header(self.request.response)
self._add_cache_header(self.request.response)
self._raise_304_if_not_modified()
self._raise_412_if_modified()
headers = self.request.response.headers
filters = self._extract_filters()
limit = self._extract_limit()
sorting = self._extract_sorting(limit)
partial_fields = self._extract_partial_fields()
filter_fields = [f.field for f in filters]
include_deleted = self.model.modified_field in filter_fields
pagination_rules, offset = self._extract_pagination_rules_from_token(
limit, sorting)
records, total_records = self.model.get_records(
filters=filters,
sorting=sorting,
limit=limit,
pagination_rules=pagination_rules,
include_deleted=include_deleted)
offset = offset + len(records)
next_page = None
if limit and len(records) == limit and offset < total_records:
lastrecord = records[-1]
next_page = self._next_page_url(sorting, limit, lastrecord, offset)
headers['Next-Page'] = encode_header(next_page)
if partial_fields:
records = [
dict_subset(record, partial_fields)
for record in records
]
# Bind metric about response size.
logger.bind(nb_records=len(records), limit=limit)
headers['Total-Records'] = encode_header('%s' % total_records)
return self.postprocess(records)
def collection_get(self):
"""Model ``GET`` endpoint: retrieve multiple records.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if
``If-None-Match`` header is provided and collection not
modified in the interim.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and collection modified
in the iterim.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPBadRequest`
if filters or sorting are invalid.
"""
self._add_timestamp_header(self.request.response)
self._add_cache_header(self.request.response)
self._raise_304_if_not_modified()
self._raise_412_if_modified()
headers = self.request.response.headers
filters = self._extract_filters()
limit = self._extract_limit()
sorting = self._extract_sorting(limit)
partial_fields = self._extract_partial_fields()
filter_fields = [f.field for f in filters]
include_deleted = self.model.modified_field in filter_fields
pagination_rules, offset = self._extract_pagination_rules_from_token(
limit, sorting)
records, total_records = self.model.get_records(
filters=filters,
sorting=sorting,
limit=limit,
pagination_rules=pagination_rules,
include_deleted=include_deleted)
offset = offset + len(records)
next_page = None
if limit and len(records) == limit and offset < total_records:
lastrecord = records[-1]
next_page = self._next_page_url(sorting, limit, lastrecord, offset)
headers['Next-Page'] = encode_header(next_page)
if partial_fields:
records = [
dict_subset(record, partial_fields)
for record in records
]
# Bind metric about response size.
logger.bind(nb_records=len(records), limit=limit)
headers['Total-Records'] = encode_header('%s' % total_records)
return self.postprocess(records)
def _plural_get(self, head_request):
self._add_timestamp_header(self.request.response)
self._add_cache_header(self.request.response)
self._raise_304_if_not_modified()
# Plural endpoints are considered resources that always exist
self._raise_412_if_modified(obj={})
headers = self.request.response.headers
filters = self._extract_filters()
limit = self._extract_limit()
sorting = self._extract_sorting(limit)
partial_fields = self._extract_partial_fields()
filter_fields = [f.field for f in filters]
include_deleted = self.model.modified_field in filter_fields
pagination_rules, offset = self._extract_pagination_rules_from_token(limit, sorting)
# The reason why we call self.model.get_objects() with `limit=limit + 1` is to avoid
# having to count the total number of objects in the database just to be able
# to *decide* whether or not to have a `Next-Page` header.
# This way, we can quickly depend on the number of objects returned and compare that
# with what the client requested.
# For example, if there are 100 objects in the database and the client used limit=100,
# it would, internally, ask for 101 objects. So if you retrieved 100 objects
# it means we got less than we asked for and thus there is not another page.
# Equally, if there are 200 objects in the database and the client used
# limit=100 it would, internally, ask for 101 objects and actually get that. Then,
# you know there is another page.
if head_request:
count = self.model.count_objects(filters=filters)
headers["Total-Objects"] = headers["Total-Records"] = str(count)
return self.postprocess([])
objects = self.model.get_objects(
filters=filters,
sorting=sorting,
limit=limit + 1, # See bigger explanation above.
pagination_rules=pagination_rules,
include_deleted=include_deleted,
)
offset = offset + len(objects)
if limit and len(objects) == limit + 1:
lastobject = objects[-2]
next_page = self._next_page_url(sorting, limit, lastobject, offset)
headers["Next-Page"] = next_page
if partial_fields:
objects = [dict_subset(obj, partial_fields) for obj in objects]
# See bigger explanation above about the use of limits. The need for slicing
# here is because we might have asked for 1 more object just to see if there's
# a next page. But we have to honor the limit in our returned response.
return self.postprocess(objects[:limit])
def plural_get(self):
"""Model ``GET`` endpoint: retrieve multiple objects.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if
``If-None-Match`` header is provided and the objects not
modified in the interim.
:raises:
:exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if
``If-Match`` header is provided and the objects modified
in the iterim.
:raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPBadRequest`
if filters or sorting are invalid.
"""
self._add_timestamp_header(self.request.response)
self._add_cache_header(self.request.response)
self._raise_304_if_not_modified()
# Plural endpoints are considered resources that always exist
self._raise_412_if_modified(obj={})
headers = self.request.response.headers
filters = self._extract_filters()
limit = self._extract_limit()
sorting = self._extract_sorting(limit)
partial_fields = self._extract_partial_fields()
filter_fields = [f.field for f in filters]
include_deleted = self.model.modified_field in filter_fields
pagination_rules, offset = self._extract_pagination_rules_from_token(
limit, sorting)
objects, total_objects = self.model.get_objects(
filters=filters,
sorting=sorting,
limit=limit,
pagination_rules=pagination_rules,
include_deleted=include_deleted,
)
offset = offset + len(objects)
if limit and len(objects) == limit and offset < total_objects:
lastobject = objects[-1]
next_page = self._next_page_url(sorting, limit, lastobject, offset)
headers["Next-Page"] = next_page
if partial_fields:
objects = [dict_subset(obj, partial_fields) for obj in objects]
headers["Total-Objects"] = str(total_objects)
# Clients backward compatibility.
headers["Total-Records"] = headers["Total-Objects"]
return self.postprocess(objects)
def test_can_filter_subobjects_keys(self):
input = dict(a=1, b=dict(c=2, d=3, e=4))
obtained = dict_subset(input, ["a", "b.d", "b.e"])
expected = dict(a=1, b=dict(d=3, e=4))
self.assertEqual(obtained, expected)
def test_can_filter_subobjects_recursively(self):
input = dict(a=1, b=dict(c=2, d=dict(e=4, f=5)))
obtained = dict_subset(input, ["a", "b.d.e"])
expected = dict(a=1, b=dict(d=dict(e=4)))
self.assertEqual(obtained, expected)
def test_ignores_duplicated_keys(self):
obtained = dict_subset(dict(a=1, b=2), ["a", "a"])
expected = dict(a=1)
self.assertEqual(obtained, expected)
def test_can_filter_subobjects(self):
obtained = dict_subset(dict(a=1, b=dict(c=2, d=3)), ["a", "b.c"])
expected = dict(a=1, b=dict(c=2))
self.assertEqual(obtained, expected)
def test_is_noop_if_no_keys(self):
obtained = dict_subset(dict(a=1, b=2), [])
expected = dict()
self.assertEqual(obtained, expected)
def test_ignores_unknown_keys(self):
obtained = dict_subset(dict(a=1, b=2), ["a", "c"])
expected = dict(a=1)
self.assertEqual(obtained, expected)
def test_extract_by_keys(self):
obtained = dict_subset(dict(a=1, b=2), ["b"])
expected = dict(b=2)
self.assertEqual(obtained, expected)
def test_ignores_duplicated_keys(self):
obtained = dict_subset(dict(a=1, b=2), ["a", "a"])
expected = dict(a=1)
self.assertEqual(obtained, expected)
def test_is_noop_if_no_keys(self):
obtained = dict_subset(dict(a=1, b=2), [])
expected = dict()
self.assertEqual(obtained, expected)
def includeme(config):
# We import stuff here, so that kinto-signer can be installed with `--no-deps`
# and used without having this Pyramid ecosystem installed.
import transaction
from kinto.core.events import ACTIONS, ResourceChanged
from pyramid.events import NewRequest
from pyramid.settings import asbool
from . import listeners, utils
from .backends import heartbeat
# Register heartbeat to check signer integration.
config.registry.heartbeats["signer"] = heartbeat
# Load settings from KINTO_SIGNER_* environment variables.
settings = config.get_settings()
for setting, default_value in DEFAULT_SETTINGS.items():
settings[f"signer.{setting}"] = utils.get_first_matching_setting(
setting_name=setting,
settings=settings,
prefixes=["signer."],
default=default_value,
)
# Check source and destination resources are configured.
resources = utils.parse_resources(settings["signer.resources"])
# Expand the resources with the ones that come from per-bucket resources
# and have specific settings.
# For example, consider the case where resource is ``/buckets/dev -> /buckets/prod``
# and there is a setting ``signer.dev.recipes.signer_backend = foo``
output_resources = resources.copy()
for key, resource in resources.items():
# If collection is not None, there is nothing to expand :)
if resource["source"]["collection"] is not None:
continue
bid = resource["source"]["bucket"]
# Match setting names like signer.stage.specific.autograph.hawk_id
matches = [(v, re.search(rf"signer\.{bid}\.([^\.]+)\.(.+)", k))
for k, v in settings.items()]
found = [(v, m.group(1), m.group(2)) for (v, m) in matches if m]
# Expand the list of resources with the ones that contain collection
# specific settings.
for setting_value, cid, setting_name in found:
signer_key = f"/buckets/{bid}/collections/{cid}"
if signer_key not in output_resources:
specific = copy.deepcopy(resource)
specific["source"]["collection"] = cid
specific["destination"]["collection"] = cid
if "preview" in specific:
specific["preview"]["collection"] = cid
output_resources[signer_key] = specific
output_resources[signer_key][setting_name] = setting_value
resources = output_resources
# Determine which are the settings that apply to all buckets/collections.
global_settings = {
"editors_group": "{collection_id}-editors",
"reviewers_group": "{collection_id}-reviewers",
"to_review_enabled": asbool(settings["signer.to_review_enabled"]),
}
# For each resource that is configured, we determine what signer is
# configured and what are the review settings.
# Note: the `resource` values are mutated in place.
config.registry.signers = {}
for signer_key, resource in resources.items():
bid = resource["source"]["bucket"]
server_wide = "signer."
bucket_wide = f"signer.{bid}."
prefixes = [bucket_wide, server_wide]
per_bucket_config = resource["source"]["collection"] is None
if not per_bucket_config:
cid = resource["source"]["collection"]
collection_wide = f"signer.{bid}.{cid}."
deprecated = f"signer.{bid}_{cid}."
prefixes = [collection_wide, deprecated] + prefixes
# Instantiates the signers associated to this resource.
dotted_location = utils.get_first_matching_setting(
"signer_backend",
settings,
prefixes,
default=DEFAULT_SETTINGS["signer_backend"],
)
signer_module = config.maybe_dotted(dotted_location)
backend = signer_module.load_from_settings(settings, prefixes=prefixes)
config.registry.signers[signer_key] = backend
# Check if review enabled/disabled for this particular resources.
resource_to_review_enabled = asbool(
utils.get_first_matching_setting(
"to_review_enabled",
settings,
prefixes,
default=global_settings["to_review_enabled"],
))
# Keep the `to_review_enabled` field in the resource object
# only if it was overriden. In other words, this will be exposed in
# the capabilities if the resource's review setting is different from
# the global server setting.
if resource_to_review_enabled != global_settings["to_review_enabled"]:
resource["to_review_enabled"] = resource_to_review_enabled
else:
resource.pop("to_review_enabled", None)
# Expose the capabilities in the root endpoint.
exposed_resources = [
core_utils.dict_subset(
r, ["source", "destination", "preview", "to_review_enabled"])
for r in resources.values()
]
message = "Digital signatures for integrity and authenticity of records."
docs = "https://github.com/Kinto/kinto-signer#kinto-signer"
config.add_api_capability(
"signer",
message,
docs,
version=__version__,
resources=exposed_resources,
# Backward compatibility with < v26
group_check_enabled=True,
**global_settings,
)
config.add_subscriber(on_review_approved, ReviewApproved)
config.add_subscriber(
functools.partial(listeners.set_work_in_progress_status,
resources=resources),
ResourceChanged,
for_resources=("record", ),
)
config.add_subscriber(
functools.partial(listeners.check_collection_status,
resources=resources,
**global_settings),
ResourceChanged,
for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE),
for_resources=("collection", ),
)
config.add_subscriber(
functools.partial(listeners.check_collection_tracking,
resources=resources),
ResourceChanged,
for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE),
for_resources=("collection", ),
)
config.add_subscriber(
functools.partial(
listeners.create_editors_reviewers_groups,
resources=resources,
editors_group=global_settings["editors_group"],
reviewers_group=global_settings["reviewers_group"],
),
ResourceChanged,
for_actions=(ACTIONS.CREATE, ),
for_resources=("collection", ),
)
config.add_subscriber(
functools.partial(listeners.cleanup_preview_destination,
resources=resources),
ResourceChanged,
for_actions=(ACTIONS.DELETE, ),
for_resources=("collection", ),
)
config.add_subscriber(
functools.partial(listeners.prevent_collection_delete,
resources=resources),
ResourceChanged,
for_actions=(ACTIONS.DELETE, ),
for_resources=("collection", ),
)
if not asbool(settings["signer.allow_floats"]):
config.add_subscriber(
functools.partial(listeners.prevent_float_value,
resources=resources),
ResourceChanged,
for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE),
for_resources=("record", ),
)
sign_data_listener = functools.partial(listeners.sign_collection_data,
resources=resources,
**global_settings)
# If StatsD is enabled, monitor execution time of listener.
if config.registry.statsd:
# Due to https://github.com/jsocol/pystatsd/issues/85
for attr in ("__module__", "__name__"):
origin = getattr(listeners.sign_collection_data, attr)
setattr(sign_data_listener, attr, origin)
statsd_client = config.registry.statsd
key = "plugins.signer"
sign_data_listener = statsd_client.timer(key)(sign_data_listener)
config.add_subscriber(
sign_data_listener,
ResourceChanged,
for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE),
for_resources=("collection", ),
)
def on_new_request(event):
"""Send the kinto-signer events in the before commit hook.
This allows database operations done in subscribers to be automatically
committed or rolledback.
"""
# Since there is one transaction per batch, ignore subrequests.
if hasattr(event.request, "parent"):
return
current = transaction.get()
current.addBeforeCommitHook(listeners.send_signer_events,
args=(event, ))
config.add_subscriber(on_new_request, NewRequest)
try:
from kinto_emailer import send_notification
config.add_subscriber(send_notification, ReviewRequested)
config.add_subscriber(send_notification, ReviewApproved)
config.add_subscriber(send_notification, ReviewRejected)
except ImportError:
pass
# Automatically create resources on startup if option is enabled.
def auto_create_resources(event, resources):
storage = event.app.registry.storage
permission = event.app.registry.permission
write_principals = aslist(
event.app.registry.
settings["signer.auto_create_resources_principals"])
for resource in resources.values():
perms = {"write": write_principals}
bucket = resource["source"]["bucket"]
collection = resource["source"]["collection"]
bucket_uri = f"/buckets/{bucket}"
storage_create_raw(
storage_backend=storage,
permission_backend=permission,
resource_name="bucket",
parent_id="",
object_uri=bucket_uri,
object_id=bucket,
permissions=perms,
)
# If resource is configured for specific collection, create it too.
if collection:
collection_uri = f"{bucket_uri}/collections/{collection}"
storage_create_raw(
storage_backend=storage,
permission_backend=permission,
resource_name="collection",
parent_id=bucket_uri,
object_uri=collection_uri,
object_id=collection,
permissions=perms,
)
# Create resources on startup (except when executing `migrate`).
if (asbool(settings.get("signer.auto_create_resources", False))
and "migrate" not in sys.argv):
config.add_subscriber(
functools.partial(
auto_create_resources,
resources=resources,
),
ApplicationCreated,
)
def test_can_filter_subobjects_recursively(self):
input = dict(b=dict(c=2, d=dict(e=4, f=5, g=6)))
obtained = dict_subset(input, ["b.d.e", "b.d.f"])
expected = dict(b=dict(d=dict(e=4, f=5)))
self.assertEqual(obtained, expected)
def test_can_filter_subobjects_keys(self):
input = dict(a=1, b=dict(c=2, d=3, e=4))
obtained = dict_subset(input, ["a", "b.d", "b.e"])
expected = dict(a=1, b=dict(d=3, e=4))
self.assertEqual(obtained, expected)
def test_can_filter_subobjects(self):
obtained = dict_subset(dict(a=1, b=dict(c=2, d=3)), ["a", "b.c"])
expected = dict(a=1, b=dict(c=2))
self.assertEqual(obtained, expected)
def test_ignores_if_subobject_is_not_dict(self):
input = dict(a=1, b=dict(c=2, d=3))
obtained = dict_subset(input, ["a", "b.c.d", "b.d"])
expected = dict(a=1, b=dict(c=2, d=3))
self.assertEqual(obtained, expected)
def test_extract_by_keys(self):
obtained = dict_subset(dict(a=1, b=2), ["b"])
expected = dict(b=2)
self.assertEqual(obtained, expected)
def test_ignores_if_subobject_is_not_dict(self):
input = dict(a=1, b=dict(c=2, d=3))
obtained = dict_subset(input, ["a", "b.c.d", "b.d"])
expected = dict(a=1, b=dict(c=2, d=3))
self.assertEqual(obtained, expected)
def test_ignores_unknown_keys(self):
obtained = dict_subset(dict(a=1, b=2), ["a", "a.b", "d.b", "c"])
expected = dict(a=1)
self.assertEqual(obtained, expected)
