def get(self): """Record ``GET`` endpoint: retrieve a record. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if the record is not found. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if ``If-None-Match`` header is provided and record not modified in the interim. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if ``If-Match`` header is provided and record modified in the iterim. """ self._raise_400_if_invalid_id(self.record_id) record = self._get_record_or_404(self.record_id) timestamp = record[self.model.modified_field] self._add_timestamp_header(self.request.response, timestamp=timestamp) self._add_cache_header(self.request.response) self._raise_304_if_not_modified(record) self._raise_412_if_modified(record) partial_fields = self._extract_partial_fields() if partial_fields: record = dict_subset(record, partial_fields) return self.postprocess(record)
def get(self): """Object ``GET`` endpoint: retrieve an object. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotFound` if the object is not found. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if ``If-None-Match`` header is provided and object not modified in the interim. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if ``If-Match`` header is provided and object modified in the iterim. """ self._raise_400_if_invalid_id(self.object_id) obj = self._get_object_or_404(self.object_id) timestamp = obj[self.model.modified_field] self._add_timestamp_header(self.request.response, timestamp=timestamp) self._add_cache_header(self.request.response) self._raise_304_if_not_modified(obj) self._raise_412_if_modified(obj) partial_fields = self._extract_partial_fields() if partial_fields: obj = dict_subset(obj, partial_fields) return self.postprocess(obj)
def collection_get(self): """Model ``GET`` endpoint: retrieve multiple records. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if ``If-None-Match`` header is provided and collection not modified in the interim. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if ``If-Match`` header is provided and collection modified in the iterim. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPBadRequest` if filters or sorting are invalid. """ self._add_timestamp_header(self.request.response) self._add_cache_header(self.request.response) self._raise_304_if_not_modified() self._raise_412_if_modified() headers = self.request.response.headers filters = self._extract_filters() limit = self._extract_limit() sorting = self._extract_sorting(limit) partial_fields = self._extract_partial_fields() filter_fields = [f.field for f in filters] include_deleted = self.model.modified_field in filter_fields pagination_rules, offset = self._extract_pagination_rules_from_token( limit, sorting) records, total_records = self.model.get_records( filters=filters, sorting=sorting, limit=limit, pagination_rules=pagination_rules, include_deleted=include_deleted) offset = offset + len(records) next_page = None if limit and len(records) == limit and offset < total_records: lastrecord = records[-1] next_page = self._next_page_url(sorting, limit, lastrecord, offset) headers['Next-Page'] = encode_header(next_page) if partial_fields: records = [ dict_subset(record, partial_fields) for record in records ] # Bind metric about response size. logger.bind(nb_records=len(records), limit=limit) headers['Total-Records'] = encode_header('%s' % total_records) return self.postprocess(records)
def _plural_get(self, head_request): self._add_timestamp_header(self.request.response) self._add_cache_header(self.request.response) self._raise_304_if_not_modified() # Plural endpoints are considered resources that always exist self._raise_412_if_modified(obj={}) headers = self.request.response.headers filters = self._extract_filters() limit = self._extract_limit() sorting = self._extract_sorting(limit) partial_fields = self._extract_partial_fields() filter_fields = [f.field for f in filters] include_deleted = self.model.modified_field in filter_fields pagination_rules, offset = self._extract_pagination_rules_from_token(limit, sorting) # The reason why we call self.model.get_objects() with `limit=limit + 1` is to avoid # having to count the total number of objects in the database just to be able # to *decide* whether or not to have a `Next-Page` header. # This way, we can quickly depend on the number of objects returned and compare that # with what the client requested. # For example, if there are 100 objects in the database and the client used limit=100, # it would, internally, ask for 101 objects. So if you retrieved 100 objects # it means we got less than we asked for and thus there is not another page. # Equally, if there are 200 objects in the database and the client used # limit=100 it would, internally, ask for 101 objects and actually get that. Then, # you know there is another page. if head_request: count = self.model.count_objects(filters=filters) headers["Total-Objects"] = headers["Total-Records"] = str(count) return self.postprocess([]) objects = self.model.get_objects( filters=filters, sorting=sorting, limit=limit + 1, # See bigger explanation above. pagination_rules=pagination_rules, include_deleted=include_deleted, ) offset = offset + len(objects) if limit and len(objects) == limit + 1: lastobject = objects[-2] next_page = self._next_page_url(sorting, limit, lastobject, offset) headers["Next-Page"] = next_page if partial_fields: objects = [dict_subset(obj, partial_fields) for obj in objects] # See bigger explanation above about the use of limits. The need for slicing # here is because we might have asked for 1 more object just to see if there's # a next page. But we have to honor the limit in our returned response. return self.postprocess(objects[:limit])
def plural_get(self): """Model ``GET`` endpoint: retrieve multiple objects. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPNotModified` if ``If-None-Match`` header is provided and the objects not modified in the interim. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPPreconditionFailed` if ``If-Match`` header is provided and the objects modified in the iterim. :raises: :exc:`~pyramid:pyramid.httpexceptions.HTTPBadRequest` if filters or sorting are invalid. """ self._add_timestamp_header(self.request.response) self._add_cache_header(self.request.response) self._raise_304_if_not_modified() # Plural endpoints are considered resources that always exist self._raise_412_if_modified(obj={}) headers = self.request.response.headers filters = self._extract_filters() limit = self._extract_limit() sorting = self._extract_sorting(limit) partial_fields = self._extract_partial_fields() filter_fields = [f.field for f in filters] include_deleted = self.model.modified_field in filter_fields pagination_rules, offset = self._extract_pagination_rules_from_token( limit, sorting) objects, total_objects = self.model.get_objects( filters=filters, sorting=sorting, limit=limit, pagination_rules=pagination_rules, include_deleted=include_deleted, ) offset = offset + len(objects) if limit and len(objects) == limit and offset < total_objects: lastobject = objects[-1] next_page = self._next_page_url(sorting, limit, lastobject, offset) headers["Next-Page"] = next_page if partial_fields: objects = [dict_subset(obj, partial_fields) for obj in objects] headers["Total-Objects"] = str(total_objects) # Clients backward compatibility. headers["Total-Records"] = headers["Total-Objects"] return self.postprocess(objects)
def test_can_filter_subobjects_keys(self): input = dict(a=1, b=dict(c=2, d=3, e=4)) obtained = dict_subset(input, ["a", "b.d", "b.e"]) expected = dict(a=1, b=dict(d=3, e=4)) self.assertEqual(obtained, expected)
def test_can_filter_subobjects_recursively(self): input = dict(a=1, b=dict(c=2, d=dict(e=4, f=5))) obtained = dict_subset(input, ["a", "b.d.e"]) expected = dict(a=1, b=dict(d=dict(e=4))) self.assertEqual(obtained, expected)
def test_ignores_duplicated_keys(self): obtained = dict_subset(dict(a=1, b=2), ["a", "a"]) expected = dict(a=1) self.assertEqual(obtained, expected)
def test_can_filter_subobjects(self): obtained = dict_subset(dict(a=1, b=dict(c=2, d=3)), ["a", "b.c"]) expected = dict(a=1, b=dict(c=2)) self.assertEqual(obtained, expected)
def test_is_noop_if_no_keys(self): obtained = dict_subset(dict(a=1, b=2), []) expected = dict() self.assertEqual(obtained, expected)
def test_ignores_unknown_keys(self): obtained = dict_subset(dict(a=1, b=2), ["a", "c"]) expected = dict(a=1) self.assertEqual(obtained, expected)
def test_extract_by_keys(self): obtained = dict_subset(dict(a=1, b=2), ["b"]) expected = dict(b=2) self.assertEqual(obtained, expected)
def includeme(config): # We import stuff here, so that kinto-signer can be installed with `--no-deps` # and used without having this Pyramid ecosystem installed. import transaction from kinto.core.events import ACTIONS, ResourceChanged from pyramid.events import NewRequest from pyramid.settings import asbool from . import listeners, utils from .backends import heartbeat # Register heartbeat to check signer integration. config.registry.heartbeats["signer"] = heartbeat # Load settings from KINTO_SIGNER_* environment variables. settings = config.get_settings() for setting, default_value in DEFAULT_SETTINGS.items(): settings[f"signer.{setting}"] = utils.get_first_matching_setting( setting_name=setting, settings=settings, prefixes=["signer."], default=default_value, ) # Check source and destination resources are configured. resources = utils.parse_resources(settings["signer.resources"]) # Expand the resources with the ones that come from per-bucket resources # and have specific settings. # For example, consider the case where resource is ``/buckets/dev -> /buckets/prod`` # and there is a setting ``signer.dev.recipes.signer_backend = foo`` output_resources = resources.copy() for key, resource in resources.items(): # If collection is not None, there is nothing to expand :) if resource["source"]["collection"] is not None: continue bid = resource["source"]["bucket"] # Match setting names like signer.stage.specific.autograph.hawk_id matches = [(v, re.search(rf"signer\.{bid}\.([^\.]+)\.(.+)", k)) for k, v in settings.items()] found = [(v, m.group(1), m.group(2)) for (v, m) in matches if m] # Expand the list of resources with the ones that contain collection # specific settings. for setting_value, cid, setting_name in found: signer_key = f"/buckets/{bid}/collections/{cid}" if signer_key not in output_resources: specific = copy.deepcopy(resource) specific["source"]["collection"] = cid specific["destination"]["collection"] = cid if "preview" in specific: specific["preview"]["collection"] = cid output_resources[signer_key] = specific output_resources[signer_key][setting_name] = setting_value resources = output_resources # Determine which are the settings that apply to all buckets/collections. global_settings = { "editors_group": "{collection_id}-editors", "reviewers_group": "{collection_id}-reviewers", "to_review_enabled": asbool(settings["signer.to_review_enabled"]), } # For each resource that is configured, we determine what signer is # configured and what are the review settings. # Note: the `resource` values are mutated in place. config.registry.signers = {} for signer_key, resource in resources.items(): bid = resource["source"]["bucket"] server_wide = "signer." bucket_wide = f"signer.{bid}." prefixes = [bucket_wide, server_wide] per_bucket_config = resource["source"]["collection"] is None if not per_bucket_config: cid = resource["source"]["collection"] collection_wide = f"signer.{bid}.{cid}." deprecated = f"signer.{bid}_{cid}." prefixes = [collection_wide, deprecated] + prefixes # Instantiates the signers associated to this resource. dotted_location = utils.get_first_matching_setting( "signer_backend", settings, prefixes, default=DEFAULT_SETTINGS["signer_backend"], ) signer_module = config.maybe_dotted(dotted_location) backend = signer_module.load_from_settings(settings, prefixes=prefixes) config.registry.signers[signer_key] = backend # Check if review enabled/disabled for this particular resources. resource_to_review_enabled = asbool( utils.get_first_matching_setting( "to_review_enabled", settings, prefixes, default=global_settings["to_review_enabled"], )) # Keep the `to_review_enabled` field in the resource object # only if it was overriden. In other words, this will be exposed in # the capabilities if the resource's review setting is different from # the global server setting. if resource_to_review_enabled != global_settings["to_review_enabled"]: resource["to_review_enabled"] = resource_to_review_enabled else: resource.pop("to_review_enabled", None) # Expose the capabilities in the root endpoint. exposed_resources = [ core_utils.dict_subset( r, ["source", "destination", "preview", "to_review_enabled"]) for r in resources.values() ] message = "Digital signatures for integrity and authenticity of records." docs = "https://github.com/Kinto/kinto-signer#kinto-signer" config.add_api_capability( "signer", message, docs, version=__version__, resources=exposed_resources, # Backward compatibility with < v26 group_check_enabled=True, **global_settings, ) config.add_subscriber(on_review_approved, ReviewApproved) config.add_subscriber( functools.partial(listeners.set_work_in_progress_status, resources=resources), ResourceChanged, for_resources=("record", ), ) config.add_subscriber( functools.partial(listeners.check_collection_status, resources=resources, **global_settings), ResourceChanged, for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE), for_resources=("collection", ), ) config.add_subscriber( functools.partial(listeners.check_collection_tracking, resources=resources), ResourceChanged, for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE), for_resources=("collection", ), ) config.add_subscriber( functools.partial( listeners.create_editors_reviewers_groups, resources=resources, editors_group=global_settings["editors_group"], reviewers_group=global_settings["reviewers_group"], ), ResourceChanged, for_actions=(ACTIONS.CREATE, ), for_resources=("collection", ), ) config.add_subscriber( functools.partial(listeners.cleanup_preview_destination, resources=resources), ResourceChanged, for_actions=(ACTIONS.DELETE, ), for_resources=("collection", ), ) config.add_subscriber( functools.partial(listeners.prevent_collection_delete, resources=resources), ResourceChanged, for_actions=(ACTIONS.DELETE, ), for_resources=("collection", ), ) if not asbool(settings["signer.allow_floats"]): config.add_subscriber( functools.partial(listeners.prevent_float_value, resources=resources), ResourceChanged, for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE), for_resources=("record", ), ) sign_data_listener = functools.partial(listeners.sign_collection_data, resources=resources, **global_settings) # If StatsD is enabled, monitor execution time of listener. if config.registry.statsd: # Due to https://github.com/jsocol/pystatsd/issues/85 for attr in ("__module__", "__name__"): origin = getattr(listeners.sign_collection_data, attr) setattr(sign_data_listener, attr, origin) statsd_client = config.registry.statsd key = "plugins.signer" sign_data_listener = statsd_client.timer(key)(sign_data_listener) config.add_subscriber( sign_data_listener, ResourceChanged, for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE), for_resources=("collection", ), ) def on_new_request(event): """Send the kinto-signer events in the before commit hook. This allows database operations done in subscribers to be automatically committed or rolledback. """ # Since there is one transaction per batch, ignore subrequests. if hasattr(event.request, "parent"): return current = transaction.get() current.addBeforeCommitHook(listeners.send_signer_events, args=(event, )) config.add_subscriber(on_new_request, NewRequest) try: from kinto_emailer import send_notification config.add_subscriber(send_notification, ReviewRequested) config.add_subscriber(send_notification, ReviewApproved) config.add_subscriber(send_notification, ReviewRejected) except ImportError: pass # Automatically create resources on startup if option is enabled. def auto_create_resources(event, resources): storage = event.app.registry.storage permission = event.app.registry.permission write_principals = aslist( event.app.registry. settings["signer.auto_create_resources_principals"]) for resource in resources.values(): perms = {"write": write_principals} bucket = resource["source"]["bucket"] collection = resource["source"]["collection"] bucket_uri = f"/buckets/{bucket}" storage_create_raw( storage_backend=storage, permission_backend=permission, resource_name="bucket", parent_id="", object_uri=bucket_uri, object_id=bucket, permissions=perms, ) # If resource is configured for specific collection, create it too. if collection: collection_uri = f"{bucket_uri}/collections/{collection}" storage_create_raw( storage_backend=storage, permission_backend=permission, resource_name="collection", parent_id=bucket_uri, object_uri=collection_uri, object_id=collection, permissions=perms, ) # Create resources on startup (except when executing `migrate`). if (asbool(settings.get("signer.auto_create_resources", False)) and "migrate" not in sys.argv): config.add_subscriber( functools.partial( auto_create_resources, resources=resources, ), ApplicationCreated, )
def test_can_filter_subobjects_recursively(self): input = dict(b=dict(c=2, d=dict(e=4, f=5, g=6))) obtained = dict_subset(input, ["b.d.e", "b.d.f"]) expected = dict(b=dict(d=dict(e=4, f=5))) self.assertEqual(obtained, expected)
def test_ignores_if_subobject_is_not_dict(self): input = dict(a=1, b=dict(c=2, d=3)) obtained = dict_subset(input, ["a", "b.c.d", "b.d"]) expected = dict(a=1, b=dict(c=2, d=3)) self.assertEqual(obtained, expected)
def test_ignores_unknown_keys(self): obtained = dict_subset(dict(a=1, b=2), ["a", "a.b", "d.b", "c"]) expected = dict(a=1) self.assertEqual(obtained, expected)