def show_type_schema(cls, context, type_name): policy.authorize("artifact:type_get", {}, context) schemas = cls._get_schemas(cls.registry) if type_name not in schemas: msg = _("Artifact type %s does not exist") % type_name raise exception.NotFound(message=msg) return schemas[type_name]
def download_blob_dict(cls, context, type_name, artifact_id, field_name, blob_key): """Download blob from artifact""" af = cls._get_artifact(context, type_name, artifact_id, read_only=True) policy.authorize("artifact:download", af.to_dict(), context) return af.download_blob_dict(context, af, field_name, blob_key)
def delete(self, context, type_name, artifact_id): """Delete artifact from Glare. :param context: User context :param type_name: Artifact type name :param artifact_id: id of artifact to delete """ af = self._show_artifact(context, type_name, artifact_id) action_name = 'artifact:delete' policy.authorize(action_name, af.to_dict(), context) af.pre_delete_hook(context, af) blobs = af.delete(context, af) delayed_delete = getattr(CONF, 'artifact_type:' + type_name).delayed_delete # use global parameter if delayed delete isn't set per artifact type if delayed_delete is None: delayed_delete = CONF.delayed_delete if not delayed_delete: if blobs: # delete blobs one by one self._delete_blobs(context, af, blobs) LOG.info("Blobs successfully deleted for artifact %s", af.id) # delete artifact itself af.db_api.delete(context, af.id) af.post_delete_hook(context, af) Notifier.notify(context, action_name, af)
def show_type_schema(cls, context, type_name): policy.authorize("artifact:type_list", {}, context) schemas = cls._get_schemas(cls.registry) if type_name not in schemas: msg = _("Artifact type %s does not exist") % type_name raise exception.NotFound(message=msg) return schemas[type_name]
def list(cls, context, type_name, filters, marker=None, limit=None, sort=None, latest=False): """Return list of artifacts requested by user. :param context: user context :param type_name: Artifact type name :param filters: filters that need to be applied to artifact :param marker: the artifact that considered as begin of the list so all artifacts before marker (including marker itself) will not be added to artifact list :param limit: maximum number of items in list :param sort: sorting options :param latest: flag that indicates, that only artifacts with highest versions should be returned in output :return: list of artifacts """ policy.authorize("artifact:list", {}, context) artifact_type = cls.registry.get_artifact_type(type_name) # return list to the user af_list = [ af.to_dict() for af in artifact_type.list(context, filters, marker, limit, sort, latest) ] return af_list
def delete_external_blob(self, context, type_name, artifact_id, field_name, blob_key=None): """Delete artifact blob with external location. :param context: user context :param type_name: name of artifact type :param artifact_id: id of artifact with the blob to delete :param field_name: name of blob or blob dict field :param blob_key: if field_name is blob dict it specifies key in this dictionary """ af = self._show_artifact(context, type_name, artifact_id) action_name = 'artifact:delete_blob' policy.authorize(action_name, af.to_dict(), context) blob_name = self._generate_blob_name(field_name, blob_key) blob = self._get_blob_info(af, field_name, blob_key) if blob is None: msg = _("Blob %s wasn't found for artifact") % blob_name raise exception.NotFound(message=msg) if not blob['external']: msg = _("Blob %s is not external") % blob_name raise exception.Forbidden(message=msg) af = self._save_blob_info(context, af, field_name, blob_key, None) Notifier.notify(context, action_name, af) return af.to_dict()
def add_blob_location(cls, context, type_name, artifact_id, field_name, location): af = cls._get_artifact(context, type_name, artifact_id) action_name = 'artifact:set_location' policy.authorize(action_name, af.to_dict(), context) modified_af = af.add_blob_location(context, af, field_name, location) Notifier.notify(context, action_name, modified_af) return modified_af.to_dict()
def show_type_schemas(self, context, type_name=None): policy.authorize("artifact:type_list", {}, context) if type_name is None: return self.schemas if type_name not in self.schemas: msg = _("Artifact type %s does not exist") % type_name raise exception.NotFound(message=msg) return self.schemas[type_name]
def upload_blob(cls, context, type_name, artifact_id, field_name, fd, content_type): """Upload Artifact blob""" af = cls._get_artifact(context, type_name, artifact_id) action_name = "artifact:upload" policy.authorize(action_name, af.to_dict(), context) modified_af = af.upload_blob(context, af, field_name, fd, content_type) Notifier.notify(context, action_name, modified_af) return modified_af.to_dict()
def upload_blob(cls, context, type_name, artifact_id, field_name, fd, content_type): """Upload Artifact blob""" af = cls._get_artifact(context, type_name, artifact_id) action_name = "artifact:upload" policy.authorize(action_name, af.to_dict(), context) modified_af = af.upload_blob(context, af, field_name, fd, content_type) Notifier.notify(context, action_name, modified_af) return modified_af.to_dict()
def set_quotas(context, values): """Set quota records in Glare. :param context: user request context :param values: dict with quota values to set """ action_name = "artifact:set_quotas" policy.authorize(action_name, {}, context) qs = quota.set_quotas(values) Notifier.notify(context, action_name, qs)
def create(cls, context, type_name, field_values): """Create new artifact in Glare""" action_name = "artifact:create" policy.authorize(action_name, field_values, context) artifact_type = cls.registry.get_artifact_type(type_name) # acquire version lock and execute artifact create af = artifact_type.create(context, field_values) # notify about new artifact Notifier.notify(context, action_name, af) # return artifact to the user return af.to_dict()
def create(cls, context, type_name, field_values): """Create new artifact in Glare""" action_name = "artifact:create" policy.authorize(action_name, field_values, context) artifact_type = cls.registry.get_artifact_type(type_name) # acquire version lock and execute artifact create af = artifact_type.create(context, field_values) # notify about new artifact Notifier.notify(context, action_name, af) # return artifact to the user return af.to_dict()
def list_all_quotas(self, context): """Get detailed info about all available quotas. :param context: user request context :return: dict with definitions of redefined quotas for all projects and global defaults """ action_name = "artifact:list_all_quotas" policy.authorize(action_name, {}, context) return { 'quotas': quota.list_quotas(), 'global_quotas': self.config_quotas }
def list_project_quotas(self, context, project_id=None): """Get detailed info about project quotas. :param context: user request context :param project_id: id of the project for which to show quotas :return: definition of requested quotas for the project """ project_id = project_id or context.project_id action_name = "artifact:list_project_quotas" policy.authorize(action_name, {'project_id': project_id}, context) qs = self.config_quotas.copy() qs.update(quota.list_quotas(project_id)[project_id]) return {project_id: qs}
def list(cls, context, type_name, filters, marker=None, limit=None, sort=None): """Return list of artifacts requested by user :param filters: list of requested filters :return: list of artifacts """ policy.authorize("artifact:list", {}, context) artifact_type = cls.registry.get_artifact_type(type_name) # return list to the user af_list = [af.to_dict() for af in artifact_type.list(context, filters, marker, limit, sort)] return af_list
def show(self, context, type_name, artifact_id): """Show detailed artifact info. :param context: user context :param type_name: Artifact type name :param artifact_id: id of artifact to show :return: definition of requested artifact """ get_any_artifact = False if policy.authorize("artifact:get_any_artifact", {}, context, do_raise=False): get_any_artifact = True policy.authorize("artifact:get", {}, context) af = self._show_artifact(context, type_name, artifact_id, read_only=True, get_any_artifact=get_any_artifact) return af.to_dict()
def create(self, context, type_name, values): """Create artifact record in Glare. :param context: user context :param type_name: artifact type name :param values: dict with artifact fields :return: dict representation of created artifact """ action_name = "artifact:create" policy.authorize(action_name, values, context) artifact_type = registry.ArtifactRegistry.get_artifact_type(type_name) version = values.get('version', artifact_type.DEFAULT_ARTIFACT_VERSION) init_values = { 'id': uuidutils.generate_uuid(), 'name': values.pop('name'), 'version': version, 'owner': context.project_id, 'created_at': timeutils.utcnow(), 'updated_at': timeutils.utcnow() } af = artifact_type.init_artifact(context, init_values) # acquire scoped lock and execute artifact create with self._create_scoped_lock(context, type_name, af.name, af.version, context.project_id): quota.verify_artifact_count(context, type_name) for field_name, value in values.items(): if af.is_blob(field_name) or af.is_blob_dict(field_name): msg = _("Cannot add blob with this request. " "Use special Blob API for that.") raise exception.BadRequest(msg) utils.validate_change_allowed(af, field_name) setattr(af, field_name, value) artifact_type.pre_create_hook(context, af) af = af.create(context) artifact_type.post_create_hook(context, af) # notify about new artifact Notifier.notify(context, action_name, af) # return artifact to the user return af.to_dict()
def delete(cls, context, type_name, artifact_id): """Delete artifact from glare""" af = cls._get_artifact(context, type_name, artifact_id) policy.authorize("artifact:delete", af.to_dict(), context) af.delete(context, af) Notifier.notify(context, "artifact.delete", af)
def update(cls, context, type_name, artifact_id, patch): """Update artifact with json patch. Apply patch to artifact and validate artifact before updating it in database. If there is request for visibility change or custom location change then call specific method for that. :param context: user context :param type_name: name of artifact type :param artifact_id: id of the artifact to be updated :param patch: json patch :return: updated artifact """ def get_updates(af_dict, patch_with_upd): """Get updated values for artifact and json patch :param af_dict: current artifact definition as dict :param patch_with_upd: json-patch :return: dict of updated attributes and their values """ try: af_dict_patched = patch_with_upd.apply(af_dict) diff = utils.DictDiffer(af_dict_patched, af_dict) # we mustn't add or remove attributes from artifact if diff.added() or diff.removed(): msg = _( "Forbidden to add or remove attributes from artifact. " "Added attributes %(added)s. " "Removed attributes %(removed)s") % { 'added': diff.added(), 'removed': diff.removed() } raise exception.BadRequest(message=msg) return {key: af_dict_patched[key] for key in diff.changed()} except (jsonpatch.JsonPatchException, jsonpatch.JsonPointerException, KeyError) as e: raise exception.BadRequest(message=str(e)) except TypeError as e: msg = _("Incorrect type of the element. Reason: %s") % str(e) raise exception.BadRequest(msg) artifact = cls._get_artifact(context, type_name, artifact_id) af_dict = artifact.to_dict() updates = get_updates(af_dict, patch) LOG.debug( "Update diff successfully calculated for artifact %(af)s " "%(diff)s", { 'af': artifact_id, 'diff': updates }) if not updates: return af_dict else: action = artifact.get_action_for_updates(context, artifact, updates, cls.registry) action_name = "artifact:%s" % action.__name__ policy.authorize(action_name, af_dict, context) modified_af = action(context, artifact, updates) Notifier.notify(context, action_name, modified_af) return modified_af.to_dict()
def get(cls, context, type_name, artifact_id): """Return artifact representation from artifact repo.""" policy.authorize("artifact:get", {}, context) af = cls._get_artifact(context, type_name, artifact_id, read_only=True) return af.to_dict()
def download_blob(self, context, type_name, artifact_id, field_name, blob_key=None): """Download binary data from Glare Artifact. :param context: user context :param type_name: name of artifact type :param artifact_id: id of the artifact to be updated :param field_name: name of blob or blob dict field :param blob_key: if field_name is blob dict it specifies key in this dict :return: file iterator for requested file """ download_from_any_artifact = False if policy.authorize("artifact:download_from_any_artifact", {}, context, do_raise=False): download_from_any_artifact = True af = self._show_artifact(context, type_name, artifact_id, read_only=True, get_any_artifact=download_from_any_artifact) if not download_from_any_artifact: policy.authorize("artifact:download", af.to_dict(), context) blob_name = self._generate_blob_name(field_name, blob_key) if af.status == 'deleted': msg = _("Cannot download data when artifact is deleted") raise exception.Forbidden(message=msg) blob = self._get_blob_info(af, field_name, blob_key) if blob is None: msg = _("No data found for blob %s") % blob_name raise exception.NotFound(message=msg) if blob['status'] != 'active': msg = _("%s is not ready for download") % blob_name raise exception.Conflict(message=msg) af.pre_download_hook(context, af, field_name, blob_key) meta = { 'md5': blob.get('md5'), 'sha1': blob.get('sha1'), 'sha256': blob.get('sha256'), 'external': blob.get('external') } if blob['external']: data = {'url': blob['url']} else: data = store_api.load_from_store(uri=blob['url'], context=context) meta['size'] = blob.get('size') meta['content_type'] = blob.get('content_type') try: # call download hook in the end data = af.post_download_hook(context, af, field_name, blob_key, data) except exception.GlareException: raise except Exception as e: raise exception.BadRequest(message=str(e)) return data, meta
def list_type_schemas(cls, context): policy.authorize("artifact:type_list", {}, context) return cls._get_schemas(cls.registry)
def list_type_schemas(cls, context): policy.authorize("artifact:type_list", {}, context) return cls._get_schemas(cls.registry)
def add_blob_location(self, context, type_name, artifact_id, field_name, location, blob_meta, blob_key=None): """Add external/internal location to blob. :param context: user context :param type_name: name of artifact type :param artifact_id: id of the artifact to be updated :param field_name: name of blob or blob dict field :param location: blob url :param blob_meta: dictionary containing blob metadata like md5 checksum :param blob_key: if field_name is blob dict it specifies key in this dict :return: dict representation of updated artifact """ blob_name = self._generate_blob_name(field_name, blob_key) location_type = blob_meta.pop('location_type', 'external') if location_type == 'external': action_name = 'artifact:set_location' elif location_type == 'internal': scheme = urlparse.urlparse(location).scheme if scheme in store_api.RESTRICTED_URI_SCHEMES: msg = _("Forbidden to set internal locations with " "scheme '%s'") % scheme raise exception.Forbidden(msg) if scheme not in store_api.get_known_schemes(): msg = _("Unknown scheme '%s'") % scheme raise exception.BadRequest(msg) action_name = 'artifact:set_internal_location' else: msg = _("Invalid location type: %s") % location_type raise exception.BadRequest(msg) blob = { 'url': location, 'size': None, 'md5': blob_meta.get("md5"), 'sha1': blob_meta.get("sha1"), 'id': uuidutils.generate_uuid(), 'sha256': blob_meta.get("sha256"), 'status': 'active', 'external': location_type == 'external', 'content_type': None } lock_key = "%s:%s" % (type_name, artifact_id) with self.lock_engine.acquire(context, lock_key): af = self._show_artifact(context, type_name, artifact_id) policy.authorize(action_name, af.to_dict(), context) if self._get_blob_info(af, field_name, blob_key): msg = _("Blob %(blob)s already exists for artifact " "%(af)s") % { 'blob': field_name, 'af': af.id } raise exception.Conflict(message=msg) utils.validate_change_allowed(af, field_name) af.pre_add_location_hook(context, af, field_name, location, blob_key) af = self._save_blob_info(context, af, field_name, blob_key, blob) LOG.info( "External location %(location)s has been created " "successfully for artifact %(artifact)s blob %(blob)s", { 'location': location, 'artifact': af.id, 'blob': blob_name }) af.post_add_location_hook(context, af, field_name, blob_key) Notifier.notify(context, action_name, af) return af.to_dict()
def _apply_patch(self, context, af, patch): # This function is a collection of hacks and workarounds to make # json patch apply changes to artifact object. action_names = ['update'] af_dict = af.to_dict() policy.authorize('artifact:update', af_dict, context) af.pre_update_hook(context, af) try: for operation in patch._ops: # apply the change to make sure that it's correct af_dict = operation.apply(af_dict) # format of location is "/key/value" or just "/key" # first case symbolizes that we have dict or list insertion, # second, that we work with a field itself. items = operation.location.split('/', 2) field_name = items[1] if af.is_blob(field_name) or af.is_blob_dict(field_name): msg = _("Cannot add blob with this request. " "Use special Blob API for that.") raise exception.BadRequest(msg) if len(items) == 2 and operation.operation['op'] == 'remove': msg = _("Cannot remove field '%s' from " "artifact.") % field_name raise exception.BadRequest(msg) # work with hooks and define action names if field_name == 'visibility': utils.validate_visibility_transition( af, from_visibility=af.visibility, to_visibility=af_dict['visibility']) if af_dict['visibility'] == 'public': policy.authorize('artifact:publish', af_dict, context) af.pre_publish_hook(context, af) action_names.append('publish') elif field_name == 'status': utils.validate_status_transition( af, from_status=af.status, to_status=af_dict['status']) if af_dict['status'] == 'deactivated': policy.authorize('artifact:deactivate', af_dict, context) af.pre_deactivate_hook(context, af) action_names.append('deactivate') elif af_dict['status'] == 'active': if af.status == 'deactivated': policy.authorize('artifact:reactivate', af_dict, context) af.pre_reactivate_hook(context, af) action_names.append('reactivate') else: policy.authorize('artifact:activate', af_dict, context) af.pre_activate_hook(context, af) action_names.append('activate') else: utils.validate_change_allowed(af, field_name) old_val = getattr(af, field_name) setattr(af, field_name, af_dict[field_name]) if operation.operation.get("op") == "move": source_field = operation.from_path setattr(af, source_field, af_dict[source_field]) new_val = getattr(af, field_name) if new_val == old_val: # No need to save value to db if it's not changed af.obj_reset_changes([field_name]) except (jsonpatch.JsonPatchException, jsonpatch.JsonPointerException, TypeError) as e: raise exception.BadRequest(message=str(e)) return action_names
def get(cls, context, type_name, artifact_id): """Return artifact representation from artifact repo.""" policy.authorize("artifact:get", {}, context) af = cls._get_artifact(context, type_name, artifact_id, read_only=True) return af.to_dict()
def delete(cls, context, type_name, artifact_id): """Delete artifact from glare""" af = cls._get_artifact(context, type_name, artifact_id) policy.authorize("artifact:delete", af.to_dict(), context) af.delete(context, af) Notifier.notify(context, "artifact.delete", af)
def update(cls, context, type_name, artifact_id, patch): """Update artifact with json patch. Apply patch to artifact and validate artifact before updating it in database. If there is request for visibility change or custom location change then call specific method for that. :param context: user context :param type_name: name of artifact type :param artifact_id: id of the artifact to be updated :param patch: json patch :return: updated artifact """ def get_updates(af_dict, patch_with_upd): """Get updated values for artifact and json patch :param af_dict: current artifact definition as dict :param patch_with_upd: json-patch :return: dict of updated attributes and their values """ class DictDiffer(object): """ Calculate the difference between two dictionaries as: (1) items added (2) items removed (3) keys same in both but changed values (4) keys same in both and unchanged values """ def __init__(self, current_dict, past_dict): self.current_dict, self.past_dict = current_dict, past_dict self.current_keys, self.past_keys = [ set(d.keys()) for d in (current_dict, past_dict) ] self.intersect = self.current_keys.intersection( self.past_keys) def added(self): return self.current_keys - self.intersect def removed(self): return self.past_keys - self.intersect def changed(self): return set(o for o in self.intersect if self.past_dict[o] != self.current_dict[o]) def unchanged(self): return set(o for o in self.intersect if self.past_dict[o] == self.current_dict[o]) try: af_dict_patched = patch_with_upd.apply(af_dict) diff = DictDiffer(af_dict_patched, af_dict) # we mustn't add or remove attributes from artifact if diff.added() or diff.removed(): msg = _( "Forbidden to add or remove attributes from artifact. " "Added attributes %(added)s. " "Removed attributes %(removed)s") % { 'added': diff.added(), 'removed': diff.removed() } raise exception.BadRequest(message=msg) return {key: af_dict_patched[key] for key in diff.changed()} except (jsonpatch.JsonPatchException, jsonpatch.JsonPointerException, KeyError) as e: raise exception.BadRequest(message=e.message) except TypeError as e: msg = _("Incorrect type of the element. Reason: %s") % str(e) raise exception.BadRequest(msg) artifact = cls._get_artifact(context, type_name, artifact_id) af_dict = artifact.to_dict() updates = get_updates(af_dict, patch) LOG.debug("Update diff successfully calculated for artifact %(af)s " "%(diff)s", {'af': artifact_id, 'diff': updates}) if not updates: return af_dict else: action = artifact.get_action_for_updates(context, artifact, updates, cls.registry) action_name = "artifact:%s" % action.__name__ policy.authorize(action_name, af_dict, context) modified_af = action(context, artifact, updates) Notifier.notify(context, action_name, modified_af) return modified_af.to_dict()
def upload_blob(self, context, type_name, artifact_id, field_name, fd, content_type, content_length=None, blob_key=None): """Upload Artifact blob. :param context: user context :param type_name: name of artifact type :param artifact_id: id of the artifact to be updated :param field_name: name of blob or blob dict field :param fd: file descriptor that Glare uses to upload the file :param content_type: data content-type :param content_length: amount of data user wants to upload :param blob_key: if field_name is blob dict it specifies key in this dictionary :return: dict representation of updated artifact """ blob_name = self._generate_blob_name(field_name, blob_key) blob_id = uuidutils.generate_uuid() blob_info = { 'url': None, 'size': None, 'md5': None, 'sha1': None, 'sha256': None, 'id': blob_id, 'status': 'saving', 'external': False, 'content_type': content_type } # Step 1. Initialize blob lock_key = "%s:%s" % (type_name, artifact_id) with self.lock_engine.acquire(context, lock_key): af = self._show_artifact(context, type_name, artifact_id) action_name = "artifact:upload" policy.authorize(action_name, af.to_dict(), context) # create an an empty blob instance in db with 'saving' status existing_blob = self._get_blob_info(af, field_name, blob_key) existing_blob_status = existing_blob.get("status")\ if existing_blob else None if existing_blob_status == "saving": msg = _("Blob %(blob)s already exists for artifact and it" "is in %(status) %(af)s") % { 'blob': field_name, 'af': af.id, 'status': existing_blob_status } raise exception.Conflict(message=msg) utils.validate_change_allowed(af, field_name) blob_info['size'] = self._calculate_allowed_space( context, af, field_name, content_length, blob_key) af = self._save_blob_info(context, af, field_name, blob_key, blob_info) LOG.debug( "Parameters validation for artifact %(artifact)s blob " "upload passed for blob %(blob_name)s. " "Start blob uploading to backend.", { 'artifact': af.id, 'blob_name': blob_name }) # Step 2. Call pre_upload_hook and upload data to the store try: try: # call upload hook first if hasattr(af, 'validate_upload'): LOG.warning("Method 'validate_upload' was deprecated. " "Please use 'pre_upload_hook' instead.") fd, path = tpool.execute(af.validate_upload, context, af, field_name, fd) else: fd = tpool.execute(af.pre_upload_hook, context, af, field_name, blob_key, fd) except exception.GlareException: raise except Exception as e: raise exception.BadRequest(message=str(e)) default_store = getattr(CONF, 'artifact_type:' + type_name).default_store # use global parameter if default store isn't set per artifact type if default_store is None: default_store = CONF.glance_store.default_store location_uri, size, checksums = store_api.save_blob_to_store( blob_id, fd, context, blob_info['size'], store_type=default_store) blob_info.update({ 'url': location_uri, 'status': 'active', 'size': size }) blob_info.update(checksums) except Exception: # if upload failed remove blob from db and storage with excutils.save_and_reraise_exception(logger=LOG): LOG.error("Exception occured: %s", Exception) self._save_blob_info(context, af, field_name, blob_key, None) LOG.info( "Successfully finished blob uploading for artifact " "%(artifact)s blob field %(blob)s.", { 'artifact': af.id, 'blob': blob_name }) # Step 3. Change blob status to 'active' with self.lock_engine.acquire(context, lock_key): af = af.show(context, artifact_id) af = self._save_blob_info(context, af, field_name, blob_key, blob_info) af.post_upload_hook(context, af, field_name, blob_key) Notifier.notify(context, action_name, af) return af.to_dict()