def delete_external_blob(self,
context,
type_name,
artifact_id,
field_name,
blob_key=None):
"""Delete artifact blob with external location.
:param context: user context
:param type_name: name of artifact type
:param artifact_id: id of artifact with the blob to delete
:param field_name: name of blob or blob dict field
:param blob_key: if field_name is blob dict it specifies key
in this dictionary
"""
af = self._show_artifact(context, type_name, artifact_id)
action_name = 'artifact:delete_blob'
policy.authorize(action_name, af.to_dict(), context)
blob_name = self._generate_blob_name(field_name, blob_key)
blob = self._get_blob_info(af, field_name, blob_key)
if blob is None:
msg = _("Blob %s wasn't found for artifact") % blob_name
raise exception.NotFound(message=msg)
if not blob['external']:
msg = _("Blob %s is not external") % blob_name
raise exception.Forbidden(message=msg)
af = self._save_blob_info(context, af, field_name, blob_key, None)
Notifier.notify(context, action_name, af)
return af.to_dict()
def delete(self, context, type_name, artifact_id):
"""Delete artifact from Glare.
:param context: User context
:param type_name: Artifact type name
:param artifact_id: id of artifact to delete
"""
af = self._show_artifact(context, type_name, artifact_id)
action_name = 'artifact:delete'
policy.authorize(action_name, af.to_dict(), context)
af.pre_delete_hook(context, af)
blobs = af.delete(context, af)
delayed_delete = getattr(CONF,
'artifact_type:' + type_name).delayed_delete
# use global parameter if delayed delete isn't set per artifact type
if delayed_delete is None:
delayed_delete = CONF.delayed_delete
if not delayed_delete:
if blobs:
# delete blobs one by one
self._delete_blobs(context, af, blobs)
LOG.info("Blobs successfully deleted for artifact %s", af.id)
# delete artifact itself
af.db_api.delete(context, af.id)
af.post_delete_hook(context, af)
Notifier.notify(context, action_name, af)
def add_blob_location(cls, context, type_name,
artifact_id, field_name, location):
af = cls._get_artifact(context, type_name, artifact_id)
action_name = 'artifact:set_location'
policy.authorize(action_name, af.to_dict(), context)
modified_af = af.add_blob_location(context, af, field_name, location)
Notifier.notify(context, action_name, modified_af)
return modified_af.to_dict()
def upload_blob(cls, context, type_name, artifact_id, field_name, fd,
content_type):
"""Upload Artifact blob"""
af = cls._get_artifact(context, type_name, artifact_id)
action_name = "artifact:upload"
policy.authorize(action_name, af.to_dict(), context)
modified_af = af.upload_blob(context, af, field_name, fd, content_type)
Notifier.notify(context, action_name, modified_af)
return modified_af.to_dict()
def upload_blob(cls, context, type_name, artifact_id, field_name, fd,
content_type):
"""Upload Artifact blob"""
af = cls._get_artifact(context, type_name, artifact_id)
action_name = "artifact:upload"
policy.authorize(action_name, af.to_dict(), context)
modified_af = af.upload_blob(context, af, field_name, fd, content_type)
Notifier.notify(context, action_name, modified_af)
return modified_af.to_dict()
def set_quotas(context, values):
"""Set quota records in Glare.
:param context: user request context
:param values: dict with quota values to set
"""
action_name = "artifact:set_quotas"
policy.authorize(action_name, {}, context)
qs = quota.set_quotas(values)
Notifier.notify(context, action_name, qs)
def create(cls, context, type_name, field_values):
"""Create new artifact in Glare"""
action_name = "artifact:create"
policy.authorize(action_name, field_values, context)
artifact_type = cls.registry.get_artifact_type(type_name)
# acquire version lock and execute artifact create
af = artifact_type.create(context, field_values)
# notify about new artifact
Notifier.notify(context, action_name, af)
# return artifact to the user
return af.to_dict()
def create(cls, context, type_name, field_values):
"""Create new artifact in Glare"""
action_name = "artifact:create"
policy.authorize(action_name, field_values, context)
artifact_type = cls.registry.get_artifact_type(type_name)
# acquire version lock and execute artifact create
af = artifact_type.create(context, field_values)
# notify about new artifact
Notifier.notify(context, action_name, af)
# return artifact to the user
return af.to_dict()
def save(self, context, type_name, artifact_id, patch):
"""Update artifact with json patch.
Apply patch to artifact and validate artifact before updating it
in database. If there is request for visibility or status change
then call specific method for that.
:param context: user context
:param type_name: name of artifact type
:param artifact_id: id of the artifact to be updated
:param patch: json patch object
:return: dict representation of updated artifact
"""
lock_key = "%s:%s" % (type_name, artifact_id)
with self.lock_engine.acquire(context, lock_key):
af = self._show_artifact(context, type_name, artifact_id)
af.obj_reset_changes()
action_names = self._apply_patch(context, af, patch)
updates = af.obj_changes_to_primitive()
LOG.debug(
"Update diff successfully calculated for artifact "
"%(af)s %(diff)s", {
'af': artifact_id,
'diff': updates
})
if not updates:
return af.to_dict()
if any(i in updates for i in ('name', 'version', 'visibility')):
# to change an artifact scope it's required to set a lock first
with self._create_scoped_lock(
context, type_name, updates.get('name', af.name),
updates.get('version', af.version), af.owner,
updates.get('visibility', af.visibility)):
af = af.save(context)
else:
af = af.save(context)
# call post hooks for all operations when data is written in db and
# send broadcast notifications
for action_name in action_names:
getattr(af, 'post_%s_hook' % action_name)(context, af)
Notifier.notify(context, 'artifact:' + action_name, af)
return af.to_dict()
def create(self, context, type_name, values):
"""Create artifact record in Glare.
:param context: user context
:param type_name: artifact type name
:param values: dict with artifact fields
:return: dict representation of created artifact
"""
action_name = "artifact:create"
policy.authorize(action_name, values, context)
artifact_type = registry.ArtifactRegistry.get_artifact_type(type_name)
version = values.get('version', artifact_type.DEFAULT_ARTIFACT_VERSION)
init_values = {
'id': uuidutils.generate_uuid(),
'name': values.pop('name'),
'version': version,
'owner': context.project_id,
'created_at': timeutils.utcnow(),
'updated_at': timeutils.utcnow()
}
af = artifact_type.init_artifact(context, init_values)
# acquire scoped lock and execute artifact create
with self._create_scoped_lock(context, type_name, af.name, af.version,
context.project_id):
quota.verify_artifact_count(context, type_name)
for field_name, value in values.items():
if af.is_blob(field_name) or af.is_blob_dict(field_name):
msg = _("Cannot add blob with this request. "
"Use special Blob API for that.")
raise exception.BadRequest(msg)
utils.validate_change_allowed(af, field_name)
setattr(af, field_name, value)
artifact_type.pre_create_hook(context, af)
af = af.create(context)
artifact_type.post_create_hook(context, af)
# notify about new artifact
Notifier.notify(context, action_name, af)
# return artifact to the user
return af.to_dict()
def delete(cls, context, type_name, artifact_id):
"""Delete artifact from glare"""
af = cls._get_artifact(context, type_name, artifact_id)
policy.authorize("artifact:delete", af.to_dict(), context)
af.delete(context, af)
Notifier.notify(context, "artifact.delete", af)
def update(cls, context, type_name, artifact_id, patch):
"""Update artifact with json patch.
Apply patch to artifact and validate artifact before updating it
in database. If there is request for visibility change or custom
location change then call specific method for that.
:param context: user context
:param type_name: name of artifact type
:param artifact_id: id of the artifact to be updated
:param patch: json patch
:return: updated artifact
"""
def get_updates(af_dict, patch_with_upd):
"""Get updated values for artifact and json patch
:param af_dict: current artifact definition as dict
:param patch_with_upd: json-patch
:return: dict of updated attributes and their values
"""
try:
af_dict_patched = patch_with_upd.apply(af_dict)
diff = utils.DictDiffer(af_dict_patched, af_dict)
# we mustn't add or remove attributes from artifact
if diff.added() or diff.removed():
msg = _(
"Forbidden to add or remove attributes from artifact. "
"Added attributes %(added)s. "
"Removed attributes %(removed)s") % {
'added': diff.added(),
'removed': diff.removed()
}
raise exception.BadRequest(message=msg)
return {key: af_dict_patched[key] for key in diff.changed()}
except (jsonpatch.JsonPatchException,
jsonpatch.JsonPointerException, KeyError) as e:
raise exception.BadRequest(message=str(e))
except TypeError as e:
msg = _("Incorrect type of the element. Reason: %s") % str(e)
raise exception.BadRequest(msg)
artifact = cls._get_artifact(context, type_name, artifact_id)
af_dict = artifact.to_dict()
updates = get_updates(af_dict, patch)
LOG.debug(
"Update diff successfully calculated for artifact %(af)s "
"%(diff)s", {
'af': artifact_id,
'diff': updates
})
if not updates:
return af_dict
else:
action = artifact.get_action_for_updates(context, artifact,
updates, cls.registry)
action_name = "artifact:%s" % action.__name__
policy.authorize(action_name, af_dict, context)
modified_af = action(context, artifact, updates)
Notifier.notify(context, action_name, modified_af)
return modified_af.to_dict()
def delete(cls, context, type_name, artifact_id):
"""Delete artifact from glare"""
af = cls._get_artifact(context, type_name, artifact_id)
policy.authorize("artifact:delete", af.to_dict(), context)
af.delete(context, af)
Notifier.notify(context, "artifact.delete", af)
def update(cls, context, type_name, artifact_id, patch):
"""Update artifact with json patch.
Apply patch to artifact and validate artifact before updating it
in database. If there is request for visibility change or custom
location change then call specific method for that.
:param context: user context
:param type_name: name of artifact type
:param artifact_id: id of the artifact to be updated
:param patch: json patch
:return: updated artifact
"""
def get_updates(af_dict, patch_with_upd):
"""Get updated values for artifact and json patch
:param af_dict: current artifact definition as dict
:param patch_with_upd: json-patch
:return: dict of updated attributes and their values
"""
class DictDiffer(object):
"""
Calculate the difference between two dictionaries as:
(1) items added
(2) items removed
(3) keys same in both but changed values
(4) keys same in both and unchanged values
"""
def __init__(self, current_dict, past_dict):
self.current_dict, self.past_dict = current_dict, past_dict
self.current_keys, self.past_keys = [
set(d.keys()) for d in (current_dict, past_dict)
]
self.intersect = self.current_keys.intersection(
self.past_keys)
def added(self):
return self.current_keys - self.intersect
def removed(self):
return self.past_keys - self.intersect
def changed(self):
return set(o for o in self.intersect
if self.past_dict[o] != self.current_dict[o])
def unchanged(self):
return set(o for o in self.intersect
if self.past_dict[o] == self.current_dict[o])
try:
af_dict_patched = patch_with_upd.apply(af_dict)
diff = DictDiffer(af_dict_patched, af_dict)
# we mustn't add or remove attributes from artifact
if diff.added() or diff.removed():
msg = _(
"Forbidden to add or remove attributes from artifact. "
"Added attributes %(added)s. "
"Removed attributes %(removed)s") % {
'added': diff.added(), 'removed': diff.removed()
}
raise exception.BadRequest(message=msg)
return {key: af_dict_patched[key] for key in diff.changed()}
except (jsonpatch.JsonPatchException,
jsonpatch.JsonPointerException,
KeyError) as e:
raise exception.BadRequest(message=e.message)
except TypeError as e:
msg = _("Incorrect type of the element. Reason: %s") % str(e)
raise exception.BadRequest(msg)
artifact = cls._get_artifact(context, type_name, artifact_id)
af_dict = artifact.to_dict()
updates = get_updates(af_dict, patch)
LOG.debug("Update diff successfully calculated for artifact %(af)s "
"%(diff)s", {'af': artifact_id, 'diff': updates})
if not updates:
return af_dict
else:
action = artifact.get_action_for_updates(context, artifact,
updates, cls.registry)
action_name = "artifact:%s" % action.__name__
policy.authorize(action_name, af_dict, context)
modified_af = action(context, artifact, updates)
Notifier.notify(context, action_name, modified_af)
return modified_af.to_dict()
def upload_blob(self,
context,
type_name,
artifact_id,
field_name,
fd,
content_type,
content_length=None,
blob_key=None):
"""Upload Artifact blob.
:param context: user context
:param type_name: name of artifact type
:param artifact_id: id of the artifact to be updated
:param field_name: name of blob or blob dict field
:param fd: file descriptor that Glare uses to upload the file
:param content_type: data content-type
:param content_length: amount of data user wants to upload
:param blob_key: if field_name is blob dict it specifies key
in this dictionary
:return: dict representation of updated artifact
"""
blob_name = self._generate_blob_name(field_name, blob_key)
blob_id = uuidutils.generate_uuid()
blob_info = {
'url': None,
'size': None,
'md5': None,
'sha1': None,
'sha256': None,
'id': blob_id,
'status': 'saving',
'external': False,
'content_type': content_type
}
# Step 1. Initialize blob
lock_key = "%s:%s" % (type_name, artifact_id)
with self.lock_engine.acquire(context, lock_key):
af = self._show_artifact(context, type_name, artifact_id)
action_name = "artifact:upload"
policy.authorize(action_name, af.to_dict(), context)
# create an an empty blob instance in db with 'saving' status
existing_blob = self._get_blob_info(af, field_name, blob_key)
existing_blob_status = existing_blob.get("status")\
if existing_blob else None
if existing_blob_status == "saving":
msg = _("Blob %(blob)s already exists for artifact and it"
"is in %(status) %(af)s") % {
'blob': field_name,
'af': af.id,
'status': existing_blob_status
}
raise exception.Conflict(message=msg)
utils.validate_change_allowed(af, field_name)
blob_info['size'] = self._calculate_allowed_space(
context, af, field_name, content_length, blob_key)
af = self._save_blob_info(context, af, field_name, blob_key,
blob_info)
LOG.debug(
"Parameters validation for artifact %(artifact)s blob "
"upload passed for blob %(blob_name)s. "
"Start blob uploading to backend.", {
'artifact': af.id,
'blob_name': blob_name
})
# Step 2. Call pre_upload_hook and upload data to the store
try:
try:
# call upload hook first
if hasattr(af, 'validate_upload'):
LOG.warning("Method 'validate_upload' was deprecated. "
"Please use 'pre_upload_hook' instead.")
fd, path = tpool.execute(af.validate_upload, context, af,
field_name, fd)
else:
fd = tpool.execute(af.pre_upload_hook, context, af,
field_name, blob_key, fd)
except exception.GlareException:
raise
except Exception as e:
raise exception.BadRequest(message=str(e))
default_store = getattr(CONF,
'artifact_type:' + type_name).default_store
# use global parameter if default store isn't set per artifact type
if default_store is None:
default_store = CONF.glance_store.default_store
location_uri, size, checksums = store_api.save_blob_to_store(
blob_id,
fd,
context,
blob_info['size'],
store_type=default_store)
blob_info.update({
'url': location_uri,
'status': 'active',
'size': size
})
blob_info.update(checksums)
except Exception:
# if upload failed remove blob from db and storage
with excutils.save_and_reraise_exception(logger=LOG):
LOG.error("Exception occured: %s", Exception)
self._save_blob_info(context, af, field_name, blob_key, None)
LOG.info(
"Successfully finished blob uploading for artifact "
"%(artifact)s blob field %(blob)s.", {
'artifact': af.id,
'blob': blob_name
})
# Step 3. Change blob status to 'active'
with self.lock_engine.acquire(context, lock_key):
af = af.show(context, artifact_id)
af = self._save_blob_info(context, af, field_name, blob_key,
blob_info)
af.post_upload_hook(context, af, field_name, blob_key)
Notifier.notify(context, action_name, af)
return af.to_dict()
def add_blob_location(self,
context,
type_name,
artifact_id,
field_name,
location,
blob_meta,
blob_key=None):
"""Add external/internal location to blob.
:param context: user context
:param type_name: name of artifact type
:param artifact_id: id of the artifact to be updated
:param field_name: name of blob or blob dict field
:param location: blob url
:param blob_meta: dictionary containing blob metadata like md5 checksum
:param blob_key: if field_name is blob dict it specifies key
in this dict
:return: dict representation of updated artifact
"""
blob_name = self._generate_blob_name(field_name, blob_key)
location_type = blob_meta.pop('location_type', 'external')
if location_type == 'external':
action_name = 'artifact:set_location'
elif location_type == 'internal':
scheme = urlparse.urlparse(location).scheme
if scheme in store_api.RESTRICTED_URI_SCHEMES:
msg = _("Forbidden to set internal locations with "
"scheme '%s'") % scheme
raise exception.Forbidden(msg)
if scheme not in store_api.get_known_schemes():
msg = _("Unknown scheme '%s'") % scheme
raise exception.BadRequest(msg)
action_name = 'artifact:set_internal_location'
else:
msg = _("Invalid location type: %s") % location_type
raise exception.BadRequest(msg)
blob = {
'url': location,
'size': None,
'md5': blob_meta.get("md5"),
'sha1': blob_meta.get("sha1"),
'id': uuidutils.generate_uuid(),
'sha256': blob_meta.get("sha256"),
'status': 'active',
'external': location_type == 'external',
'content_type': None
}
lock_key = "%s:%s" % (type_name, artifact_id)
with self.lock_engine.acquire(context, lock_key):
af = self._show_artifact(context, type_name, artifact_id)
policy.authorize(action_name, af.to_dict(), context)
if self._get_blob_info(af, field_name, blob_key):
msg = _("Blob %(blob)s already exists for artifact "
"%(af)s") % {
'blob': field_name,
'af': af.id
}
raise exception.Conflict(message=msg)
utils.validate_change_allowed(af, field_name)
af.pre_add_location_hook(context, af, field_name, location,
blob_key)
af = self._save_blob_info(context, af, field_name, blob_key, blob)
LOG.info(
"External location %(location)s has been created "
"successfully for artifact %(artifact)s blob %(blob)s", {
'location': location,
'artifact': af.id,
'blob': blob_name
})
af.post_add_location_hook(context, af, field_name, blob_key)
Notifier.notify(context, action_name, af)
return af.to_dict()