def list(self, req):
params = req.params.copy()
marker = params.pop('marker', None)
query_params = {}
# step 1 - apply marker to query if exists
if marker is not None:
query_params['marker'] = marker
# step 2 - apply limit (if exists OR setup default limit)
limit = params.pop('limit', CONF.default_api_limit)
try:
limit = int(limit)
except ValueError:
msg = _("Limit param must be an integer.")
raise exc.BadRequest(message=msg)
if limit < 0:
msg = _("Limit param must be positive.")
raise exc.BadRequest(message=msg)
query_params['limit'] = min(CONF.max_api_limit, limit)
# step 3 - parse sort parameters
if 'sort' in params:
sort = []
for sort_param in params.pop('sort').strip().split(','):
key, _sep, direction = sort_param.partition(':')
if direction and direction not in ('asc', 'desc'):
raise exc.BadRequest('Sort direction must be one of '
'["asc", "desc"]. Got %s direction' %
direction)
sort.append((key, direction or 'desc'))
query_params['sort'] = sort
query_params['filters'] = params
return query_params
def list(self, req):
params = req.params.copy()
marker = params.pop('marker', None)
query_params = {}
# step 1 - apply marker to query if exists
if marker is not None:
query_params['marker'] = marker
# step 2 - apply limit (if exists OR setup default limit)
limit = params.pop('limit', CONF.default_api_limit)
try:
limit = int(limit)
except ValueError:
msg = _("Limit param must be an integer.")
raise exc.BadRequest(message=msg)
if limit < 0:
msg = _("Limit param must be positive.")
raise exc.BadRequest(message=msg)
query_params['limit'] = min(CONF.max_api_limit, limit)
# step 3 - parse sort parameters
if 'sort' in params:
sort = []
for sort_param in params.pop('sort').strip().split(','):
key, _sep, direction = sort_param.partition(':')
if direction and direction not in ('asc', 'desc'):
raise exc.BadRequest('Sort direction must be one of '
'["asc", "desc"]. Got %s direction'
% direction)
sort.append((key, direction or 'desc'))
query_params['sort'] = sort
query_params['filters'] = params
return query_params
def process_request(req):
auth_token = req.headers.get('X-Auth-Token')
if not auth_token:
msg = _("Auth token must be provided")
raise exception.Unauthorized(msg)
try:
user, tenant, roles = auth_token.strip().split(':', 3)
except ValueError:
msg = _("Wrong auth token format. It must be 'user:tenant:roles'")
raise exception.Unauthorized(msg)
if not tenant:
msg = _("Tenant must be specified in auth token. "
"Format of the token is 'user:tenant:roles'")
raise exception.Unauthorized(msg)
elif tenant.lower() == 'none':
tenant = None
req.headers['X-Identity-Status'] = 'Nope'
else:
req.headers['X-Identity-Status'] = 'Confirmed'
req.headers['X-User-Id'] = user
req.headers['X-Tenant-Id'] = tenant
req.headers['X-Roles'] = roles
if req.headers.get('X-Identity-Status') == 'Confirmed':
kwargs = {'request_id': req.environ.get(request_id.ENV_REQUEST_ID)}
req.context = RequestContext.from_environ(req.environ, **kwargs)
elif CONF.allow_anonymous_access:
req.context = RequestContext(read_only=True, is_admin=False)
else:
raise exception.Unauthorized()
def coerce(obj, attr, value):
# to remove the existing link user sets its value to None,
# we have to consider this case.
if value is None:
return value
# check that value is string
if not isinstance(value, six.string_types):
raise ValueError(
_('A string is required in field %(attr)s, '
'not a %(type)s') % {
'attr': attr,
'type': type(value).__name__
})
# determine if link is external or internal
external = LinkFieldType.is_external(value)
# validate link itself
if external:
link = urlparse.urlparse(value)
if link.scheme not in ('http', 'https'):
raise ValueError(
_('Only http and https requests '
'are allowed in url %s') % value)
else:
result = value.split('/')
if len(result) != 4 or result[1] != 'artifacts':
raise ValueError(
_('Link %(link)s is not valid in field '
'%(attr)s. The link must be either valid url or '
'reference to artifact. Example: '
'/artifacts/<artifact_type>/<artifact_id>') % {
'link': value,
'attr': attr
})
return value
def create(cls, context, values):
"""Create new Artifact in Glare repo
:param context: user context
:param values: Dict with specified artifact properties
:return: definition of create Artifact
"""
if context.tenant is None or context.read_only:
msg = _("It's forbidden to anonymous users to create artifacts.")
raise exception.Forbidden(msg)
else:
with cls.lock_engine.acquire(
context, cls._get_versioning_scope(context, values)):
ver = values.setdefault(
'version', cls.DEFAULT_ARTIFACT_VERSION)
cls._validate_versioning(context, values.get('name'), ver)
# validate other values
cls._validate_input_values(context, values)
# validate visibility
if 'visibility' in values:
msg = _("visibility is not allowed in a request "
"for artifact create.")
raise exception.BadRequest(msg)
values['id'] = str(uuid.uuid4())
values['owner'] = context.tenant
values['created_at'] = timeutils.utcnow()
values['updated_at'] = values['created_at']
af = cls._init_artifact(context, values)
LOG.info(_LI("Parameters validation for artifact creation "
"passed for request %s."), context.request_id)
af_vals = cls.db_api.create(context,
af.obj_changes_to_primitive())
return cls._init_artifact(context, af_vals)
def get_updates(af_dict, patch_with_upd):
"""Get updated values for artifact and json patch
:param af_dict: current artifact definition as dict
:param patch_with_upd: json-patch
:return: dict of updated attributes and their values
"""
try:
af_dict_patched = patch_with_upd.apply(af_dict)
diff = utils.DictDiffer(af_dict_patched, af_dict)
# we mustn't add or remove attributes from artifact
if diff.added() or diff.removed():
msg = _(
"Forbidden to add or remove attributes from artifact. "
"Added attributes %(added)s. "
"Removed attributes %(removed)s") % {
'added': diff.added(),
'removed': diff.removed()
}
raise exception.BadRequest(message=msg)
return {key: af_dict_patched[key] for key in diff.changed()}
except (jsonpatch.JsonPatchException,
jsonpatch.JsonPointerException, KeyError) as e:
raise exception.BadRequest(message=str(e))
except TypeError as e:
msg = _("Incorrect type of the element. Reason: %s") % str(e)
raise exception.BadRequest(msg)
def __call__(self, request):
if 'X-Auth-Token' not in request.headers:
msg = _("Auth token must be provided in 'X-Auth-Token' header.")
LOG.error(msg)
raise exception.Unauthorized()
access_token = request.headers.get('X-Auth-Token')
try:
decoded = jwt.decode(access_token, algorithms=['RS256'],
verify=False)
except Exception as e:
msg = _("Token can't be decoded because of wrong format %s")\
% str(e)
LOG.error(msg)
raise exception.Unauthorized()
# Get user realm from parsed token
# Format is "iss": "http://<host>:<port>/auth/realms/<realm_name>",
__, __, realm_name = decoded['iss'].strip().rpartition('/realms/')
# Get roles from from parsed token
roles = ','.join(decoded['realm_access']['roles']) \
if 'realm_access' in decoded else ''
self.authenticate(access_token, realm_name)
request.headers["X-Identity-Status"] = "Confirmed"
request.headers["X-Project-Id"] = realm_name
request.headers["X-Roles"] = roles
return request.get_response(self.application)
def download_blob_dict(cls, context, af, field_name, blob_key):
"""Download binary data from Glare Artifact.
:param context: user context
:param af: Artifact definition in Glare repo
:param blob_key: name of blob key in dict
:param field_name: name of blob dict field
:return: file iterator for requested file
"""
if not cls.is_blob_dict(field_name):
msg = _("%s is not a blob dict") % field_name
raise exception.BadRequest(msg)
if af.status == cls.STATUS.DEACTIVATED and not context.is_admin:
msg = _("Only admin is allowed to download image data "
"when it's deactivated")
raise exception.Forbidden(message=msg)
try:
blob = getattr(af, field_name)[blob_key]
except KeyError:
msg = _("Blob with name %(blob_name)s is not found in blob "
"dictionary %(blob_dict)s") % (blob_key, field_name)
raise exception.NotFound(message=msg)
if blob is None or blob['status'] != BlobStatus.ACTIVE:
msg = _("Blob %(blob_name)s from blob dictionary %(blob_dict)s "
"is not ready for download") % (blob_key, field_name)
LOG.error(msg)
raise exception.BadRequest(message=msg)
data = store_api.load_from_store(uri=blob['url'], context=context)
meta = {'size': blob['size'], 'checksum': blob['checksum'],
'content_type': blob['content_type']}
return data, meta
def coerce(obj, attr, value):
# to remove the existing dependency user sets its value to None,
# we have to consider this case.
if value is None:
return value
# check that value is string
if not isinstance(value, six.string_types):
raise ValueError(_('A string is required in field %(attr)s, '
'not a %(type)s') %
{'attr': attr, 'type': type(value).__name__})
# determine if link is external or internal
external = DependencyFieldType.is_external(value)
# validate link itself
if external:
link = urlparse.urlparse(value)
if link.scheme not in ('http', 'https'):
raise ValueError(_('Only http and https requests '
'are allowed in url %s') % value)
else:
result = value.split('/')
if len(result) != 4 or result[1] != 'artifacts':
raise ValueError(
_('Dependency link %(link)s is not valid in field '
'%(attr)s. The link must be either valid url or '
'reference to artifact. Example: '
'/artifacts/<artifact_type>/<artifact_id>'
) % {'link': value, 'attr': attr})
return value
def validate_key_cert(key_file, cert_file):
try:
error_key_name = "private key"
error_filename = key_file
with open(key_file, 'r') as keyfile:
key_str = keyfile.read()
key = crypto.load_privatekey(crypto.FILETYPE_PEM, key_str)
error_key_name = "certificate"
error_filename = cert_file
with open(cert_file, 'r') as certfile:
cert_str = certfile.read()
cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_str)
except IOError as ioe:
raise RuntimeError(
_("There is a problem with your %(error_key_name)s "
"%(error_filename)s. Please verify it."
" Error: %(ioe)s") % {
'error_key_name': error_key_name,
'error_filename': error_filename,
'ioe': ioe
})
except crypto.Error as ce:
raise RuntimeError(
_("There is a problem with your %(error_key_name)s "
"%(error_filename)s. Please verify it. OpenSSL"
" error: %(ce)s") % {
'error_key_name': error_key_name,
'error_filename': error_filename,
'ce': ce
})
try:
data = str(uuid.uuid4())
# On Python 3, explicitly encode to UTF-8 to call crypto.sign() which
# requires bytes. Otherwise, it raises a deprecation warning (and
# will raise an error later).
data = encodeutils.to_utf8(data)
digest = CONF.digest_algorithm
if digest == 'sha1':
LOG.warn(
_LW('The FIPS (FEDERAL INFORMATION PROCESSING STANDARDS)'
' state that the SHA-1 is not suitable for'
' general-purpose digital signature applications (as'
' specified in FIPS 186-3) that require 112 bits of'
' security. The default value is sha1 in Kilo for a'
' smooth upgrade process, and it will be updated'
' with sha256 in next release(L).'))
out = crypto.sign(key, data, digest)
crypto.verify(cert, out, data, digest)
except crypto.Error as ce:
raise RuntimeError(
_("There is a problem with your key pair. "
"Please verify that cert %(cert_file)s and "
"key %(key_file)s belong together. OpenSSL "
"error %(ce)s") % {
'cert_file': cert_file,
'key_file': key_file,
'ce': ce
})
def delete_external_blob(self,
context,
type_name,
artifact_id,
field_name,
blob_key=None):
"""Delete artifact blob with external location.
:param context: user context
:param type_name: name of artifact type
:param artifact_id: id of artifact with the blob to delete
:param field_name: name of blob or blob dict field
:param blob_key: if field_name is blob dict it specifies key
in this dictionary
"""
af = self._show_artifact(context, type_name, artifact_id)
action_name = 'artifact:delete_blob'
policy.authorize(action_name, af.to_dict(), context)
blob_name = self._generate_blob_name(field_name, blob_key)
blob = self._get_blob_info(af, field_name, blob_key)
if blob is None:
msg = _("Blob %s wasn't found for artifact") % blob_name
raise exception.NotFound(message=msg)
if not blob['external']:
msg = _("Blob %s is not external") % blob_name
raise exception.Forbidden(message=msg)
af = self._save_blob_info(context, af, field_name, blob_key, None)
Notifier.notify(context, action_name, af)
return af.to_dict()
def activate(cls, context, af, values):
"""Activate Artifact and make it available for users
:param context: User Context
:param af: current Artifact definition in Glare
:return: definition of activated Artifact
"""
# validate that came to artifact as updates
if values != {'status': cls.STATUS.ACTIVE}:
msg = _("Only {'status': %s} is allowed in a request "
"for activation.") % cls.STATUS.ACTIVE
raise exception.BadRequest(msg)
for name, type_obj in six.iteritems(af.fields):
if type_obj.required_on_activate and getattr(af, name) is None:
msg = _("'%s' attribute must be set before activation") % name
raise exception.BadRequest(msg)
cls.validate_activate(context, af)
if af.status != cls.STATUS.QUEUED:
raise exception.InvalidStatusTransition(
orig=af.status, new=cls.STATUS.ACTIVE
)
LOG.info(_LI("Parameters validation for artifact %(artifact)s "
"activate passed for request %(request)s."),
{'artifact': af.id, 'request': context.request_id})
active_af = cls.db_api.update(context, af.id, values)
return cls._init_artifact(context, active_af)
def publish(cls, context, af, values):
"""Make Artifact available for everyone
:param context: user context
:param af: definition of published Artifact
:return: definition of active Artifact
"""
if values != {'visibility': 'public'}:
msg = _("Only {'visibility': 'public'} is allowed in a request "
"for artifact publish.")
raise exception.BadRequest(msg)
with cls.lock_engine.acquire(context, cls._get_versioning_scope(
context, values, af)):
if af.status != cls.STATUS.ACTIVE:
msg = _("Cannot publish non-active artifact")
raise exception.BadRequest(msg)
cls._validate_versioning(context, af.name, af.version,
is_public=True)
cls.validate_publish(context, af)
LOG.info(_LI("Parameters validation for artifact %(artifact)s "
"publish passed for request %(request)s."),
{'artifact': af.id, 'request': context.request_id})
af = cls.db_api.update(context, af.id, values)
return cls._init_artifact(context, af)
def coerce(obj, field, value):
# to remove the existing link user sets its value to None,
# we have to consider this case.
if value is None:
return value
# check that value is string
if not isinstance(value, six.string_types):
raise ValueError(
_('A string is required in field %(field)s, '
'not a %(type)s') % {
'field': field,
'type': type(value).__name__
})
# determine if link is external or internal
external = LinkFieldType.is_external(value)
# validate link itself
if external:
link = urlparse.urlparse(value)
if link.scheme not in ('http', 'https'):
raise ValueError(
_('Only http and https requests '
'are allowed in url %s') % value)
try:
with urlrequest.urlopen(value) as data:
data.read(1)
except Exception:
raise ValueError(
_('Link %(link)s is not valid in field '
'%(field)s. The link must be either valid url or '
'reference to artifact. Example: '
'http://glarehost:9494/artifacts/<artifact_type>/'
'<artifact_id>') % {
'link': value,
'field': field
})
else:
result = value.split('/')
if len(result) != 4 or result[1] != 'artifacts':
raise ValueError(
_('Link %(link)s is not valid in field '
'%(field)s. The link must be either valid url or '
'reference to artifact. Example: '
'/artifacts/<artifact_type>/<artifact_id>') % {
'link': value,
'field': field
})
# try to find the referenced artifact
try:
obj.db_api.get(obj.obj_context, None, result[3])
except exception.NotFound:
raise ValueError(
_("Link %(link)s is not valid in field %(field)s, because "
"artifact with id %(art_id)s doesn't exist") % {
'link': value,
'field': field,
'art_id': result[3]
})
return value
def verify_uploaded_data_amount(context, type_name, data_amount=None):
"""Verify if user can upload data based on his quota limits.
:param context: user context
:param type_name: name of artifact type
:param data_amount: number of bytes user wants to upload. Value None means
that user hasn't specified data amount. In this case don't raise an
exception, but just return the amount of data he is able to upload.
:return: number of bytes user can upload if data_amount isn't specified
"""
global_limit = CONF.max_uploaded_data
type_limit = getattr(CONF, 'artifact_type:' + type_name).max_uploaded_data
# update limits if they were reassigned for project
project_id = context.project_id
quotas = list_quotas(project_id).get(project_id, {})
if 'max_uploaded_data' in quotas:
global_limit = quotas['max_uploaded_data']
if 'max_uploaded_data:' + type_name in quotas:
type_limit = quotas['max_uploaded_data:' + type_name]
session = api.get_session()
res = -1
if global_limit != -1:
# the whole amount of created artifacts
whole_number = api.calculate_uploaded_data(context, session)
if data_amount is None:
res = global_limit - whole_number
elif whole_number + data_amount > global_limit:
msg = _("Can't upload %(data_amount)d byte(s) because of global "
"quota limit: %(global_limit)d. "
"You have %(whole_number)d bytes uploaded.") % {
'data_amount': data_amount,
'global_limit': global_limit,
'whole_number': whole_number
}
raise exception.RequestEntityTooLarge(msg)
if type_limit != -1:
# the amount of artifacts for specific type
type_number = api.calculate_uploaded_data(context, session, type_name)
if data_amount is None:
available = type_limit - type_number
res = available if res == -1 else min(res, available)
elif type_number + data_amount > type_limit:
msg = _("Can't upload %(data_amount)d byte(s) because of "
"quota limit for artifact type '%(type_name)s': "
"%(type_limit)d. You have %(type_number)d bytes "
"uploaded for this type.") % {
'data_amount': data_amount,
'type_name': type_name,
'type_limit': type_limit,
'type_number': type_number
}
raise exception.RequestEntityTooLarge(msg)
return res
def delete(cls, context, af):
"""Delete Artifact and all blobs from Glare.
:param context: user context
:param af: definition of artifact targeted to delete
"""
if af.visibility == 'public' and not context.is_admin:
msg = _("Only admins are allowed to delete public images")
raise exception.Forbidden(msg)
# marking all blobs as pending delete
blobs = {}
for name, field in six.iteritems(af.fields):
if cls.is_blob(name):
b = getattr(af, name)
if b:
if b['status'] == BlobStatus.PENDING_DELETE:
msg = _('Blob %(name)s is already deleting '
'for artifact %(id)s') % {'name': name,
'id': af.id}
raise exception.Conflict(msg)
else:
b['status'] = BlobStatus.PENDING_DELETE
blobs[name] = b
elif cls.is_blob_dict(name):
bd = getattr(af, name)
if bd:
for key, b in six.iteritems(bd):
if b['status'] == BlobStatus.PENDING_DELETE:
msg = _('Blob %(name)s is already deleting '
'for artifact %(id)s') % {'name': name,
'id': af.id}
raise exception.Conflict(msg)
else:
b['status'] = BlobStatus.PENDING_DELETE
blobs[name] = bd
if blobs:
LOG.debug("Marked all blobs %(blobs) for artifact %(artifact)s "
"as pending delete. Start blobs delete.",
{'blobs': blobs, 'artifact': af.id})
cls.db_api.update(context, af.id, blobs)
# delete blobs one by one
if not CONF.delayed_blob_delete:
for name, blob in six.iteritems(blobs):
if cls.is_blob(name):
store_api.delete_blob(blob['url'], context=context)
cls.db_api.update(context, af.id, {name: None})
elif cls.is_blob_dict(name):
upd_blob = deepcopy(blob)
for key, val in six.iteritems(blob):
store_api.delete_blob(val['url'], context=context)
del upd_blob[key]
cls.db_api.update(context, af.id, {name: upd_blob})
LOG.info(_LI("Blobs successfully deleted for artifact %s"), af.id)
# delete artifact itself
cls.db_api.delete(context, af.id)
def from_json(self, datastring):
try:
jsondata = jsonutils.loads(datastring, object_hook=self._sanitizer)
if not isinstance(jsondata, (dict, list)):
msg = _('Unexpected body type. Expected list/dict.')
raise webob.exc.HTTPBadRequest(explanation=msg)
return jsondata
except ValueError:
msg = _('Malformed JSON in request body.')
raise webob.exc.HTTPBadRequest(explanation=msg)
def from_json(self, datastring):
try:
jsondata = jsonutils.loads(datastring, object_hook=self._sanitizer)
if not isinstance(jsondata, (dict, list)):
msg = _('Unexpected body type. Expected list/dict.')
raise webob.exc.HTTPBadRequest(explanation=msg)
return jsondata
except ValueError:
msg = _('Malformed JSON in request body.')
raise webob.exc.HTTPBadRequest(explanation=msg)
def _get_blob_info(af, field_name, blob_key=None):
"""Return requested blob info."""
if blob_key:
if not af.is_blob_dict(field_name):
msg = _("%s is not a blob dict") % field_name
raise exception.BadRequest(msg)
return getattr(af, field_name).get(blob_key)
else:
if not af.is_blob(field_name):
msg = _("%s is not a blob") % field_name
raise exception.BadRequest(msg)
return getattr(af, field_name, None)
def validate_key_cert(key_file, cert_file):
try:
error_key_name = "private key"
error_filename = key_file
with open(key_file, 'r') as keyfile:
key_str = keyfile.read()
key = crypto.load_privatekey(crypto.FILETYPE_PEM, key_str)
error_key_name = "certificate"
error_filename = cert_file
with open(cert_file, 'r') as certfile:
cert_str = certfile.read()
cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_str)
except IOError as ioe:
raise RuntimeError(_("There is a problem with your %(error_key_name)s "
"%(error_filename)s. Please verify it."
" Error: %(ioe)s") %
{'error_key_name': error_key_name,
'error_filename': error_filename,
'ioe': ioe})
except crypto.Error as ce:
raise RuntimeError(_("There is a problem with your %(error_key_name)s "
"%(error_filename)s. Please verify it. OpenSSL"
" error: %(ce)s") %
{'error_key_name': error_key_name,
'error_filename': error_filename,
'ce': ce})
try:
data = str(uuid.uuid4())
# On Python 3, explicitly encode to UTF-8 to call crypto.sign() which
# requires bytes. Otherwise, it raises a deprecation warning (and
# will raise an error later).
data = encodeutils.to_utf8(data)
digest = CONF.digest_algorithm
if digest == 'sha1':
LOG.warn(
_LW('The FIPS (FEDERAL INFORMATION PROCESSING STANDARDS)'
' state that the SHA-1 is not suitable for'
' general-purpose digital signature applications (as'
' specified in FIPS 186-3) that require 112 bits of'
' security. The default value is sha1 in Kilo for a'
' smooth upgrade process, and it will be updated'
' with sha256 in next release(L).'))
out = crypto.sign(key, data, digest)
crypto.verify(cert, out, data, digest)
except crypto.Error as ce:
raise RuntimeError(_("There is a problem with your key pair. "
"Please verify that cert %(cert_file)s and "
"key %(key_file)s belong together. OpenSSL "
"error %(ce)s") % {'cert_file': cert_file,
'key_file': key_file,
'ce': ce})
def get_location_info(url, context, max_size, calc_checksum=True):
"""Validate location and get information about external blob
:param url: blob url
:param context: user context
:param calc_checksum: define if checksum must be calculated
:return: blob size and checksum
"""
# validate uri
scheme = urlparse.urlparse(url).scheme
if scheme not in ('http', 'https'):
msg = _("Location %s is invalid.") % url
raise exception.BadRequest(message=msg)
res = urllib.urlopen(url)
http_message = res.info()
content_type = getattr(http_message, 'type') or 'application/octet-stream'
# calculate blob checksum to ensure that location blob won't be changed
# in future
# TODO(kairat) need to support external location signatures
checksum = None
size = 0
if calc_checksum:
checksum = hashlib.md5()
blob_data = load_from_store(url, context)
for buf in blob_data:
checksum.update(buf)
size += len(buf)
if size > max_size:
msg = _("External blob size %(size)d exceeds maximum allowed "
"size %(max)d."), {
'size': size,
'max': max_size
}
raise exception.BadRequest(message=msg)
checksum = checksum.hexdigest()
else:
# request blob size
size = get_blob_size(url, context=context)
if size < 0 or size > max_size:
msg = _("Invalid blob size %d.") % size
raise exception.BadRequest(message=msg)
LOG.debug(
"Checksum %(checksum)s and size %(size)s calculated "
"successfully for location %(location)s", {
'checksum': str(checksum),
'size': str(size),
'location': url
})
return size, checksum, content_type
def _check_dict(data_dict):
# a dict of dicts has to be checked recursively
for key, value in six.iteritems(data_dict):
if isinstance(value, dict):
_check_dict(value)
else:
if _is_match(key):
msg = _("Property names can't contain 4 byte unicode.")
raise exception.Invalid(msg)
if _is_match(value):
msg = (_("%s can't contain 4 byte unicode characters.")
% key.title())
raise exception.Invalid(msg)
def _check_dict(data_dict):
# a dict of dicts has to be checked recursively
for key, value in data_dict.items():
if isinstance(value, dict):
_check_dict(value)
else:
if _is_match(key):
msg = _("Property names can't contain 4 byte unicode.")
raise exception.BadRequest(msg)
if _is_match(value):
msg = (_("%s can't contain 4 byte unicode characters.")
% key.title())
raise exception.BadRequest(msg)
def validate_visibility_transition(af, from_visibility, to_visibility):
if to_visibility == 'private':
if from_visibility != 'private':
msg = _("Cannot make artifact private again.")
raise exception.Forbidden()
elif to_visibility == 'public':
if af.status != 'active':
msg = _("Cannot change visibility to 'public' if artifact"
" is not active.")
raise exception.Forbidden(msg)
else:
msg = _("Unknown artifact visibility: %s.") % to_visibility
raise exception.BadRequest(msg)
def _parse_sort_values(cls, sort):
new_sort = []
for key, direction in sort:
if key not in cls.fields:
msg = _("The field %s doesn't exist.") % key
raise exception.BadRequest(msg)
# check if field can be sorted
if not cls.fields[key].sortable:
msg = _("The field %s is not sortable.") % key
raise exception.BadRequest(msg)
new_sort.append((key, direction, cls._get_field_type(
cls.fields.get(key))))
return new_sort
def __call__(self, request):
"""WSGI method that controls (de)serialization and method dispatch."""
action_args = self.get_action_args(request.environ)
action = action_args.pop('action', None)
body_reject = strutils.bool_from_string(
action_args.pop('body_reject', None))
try:
if body_reject and self.deserializer.has_body(request):
msg = _('A body is not expected with this request.')
raise webob.exc.HTTPBadRequest(explanation=msg)
deserialized_request = self.dispatch(self.deserializer,
action, request)
action_args.update(deserialized_request)
action_result = self.dispatch(self.controller, action,
request, **action_args)
except webob.exc.WSGIHTTPException as e:
exc_info = sys.exc_info()
e = translate_exception(request, e)
six.reraise(type(e), e, exc_info[2])
except glare_exc.GlareException:
raise
except UnicodeDecodeError:
msg = _("Error decoding your request. Either the URL or the "
"request body contained characters that could not be "
"decoded by Glare")
raise webob.exc.HTTPBadRequest(explanation=msg)
except Exception as e:
LOG.exception(_LE("Caught error: %s"),
encodeutils.exception_to_unicode(e))
response = webob.exc.HTTPInternalServerError(explanation=str(e))
return response
try:
response = webob.Response(request=request)
self.dispatch(self.serializer, action, response, action_result)
# encode all headers in response to utf-8 to prevent unicode errors
for name, value in list(response.headers.items()):
if six.PY2 and isinstance(value, six.text_type):
response.headers[name] = encodeutils.safe_encode(value)
return response
except webob.exc.WSGIHTTPException as e:
return translate_exception(request, e)
except webob.exc.HTTPException as e:
return e
except glare_exc.GlareException:
raise
# return unserializable result (typically a webob exc)
except Exception:
return action_result
def _get_content_type(req, expected=None):
"""Determine content type of the request body."""
if "Content-Type" not in req.headers:
msg = _("Content-Type must be specified.")
LOG.error(msg)
raise exc.BadRequest(msg)
content_type = req.content_type
if expected is not None and content_type not in expected:
msg = (_('Invalid content type: %(ct)s. Expected: %(exp)s') %
{'ct': content_type, 'exp': ', '.join(expected)})
raise exc.UnsupportedMediaType(message=msg)
return content_type
def get_socket(default_port):
"""
Bind socket to bind ip:port in conf
note: Mostly comes from Swift with a few small changes...
:param default_port: port to bind to if none is specified in conf
:returns: a socket object as returned from socket.listen or
ssl.wrap_socket if conf specifies cert_file
"""
bind_addr = get_bind_addr(default_port)
# TODO(jaypipes): eventlet's greened socket module does not actually
# support IPv6 in getaddrinfo(). We need to get around this in the
# future or monitor upstream for a fix
address_family = [
addr[0] for addr in socket.getaddrinfo(
bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)
if addr[0] in (socket.AF_INET, socket.AF_INET6)
][0]
use_ssl = CONF.key_file or CONF.cert_file
if use_ssl and (not CONF.key_file or not CONF.cert_file):
raise RuntimeError(
_("When running server in SSL mode, you must "
"specify both a cert_file and key_file "
"option value in your configuration file"))
sock = utils.get_test_suite_socket()
retry_until = time.time() + 30
while not sock and time.time() < retry_until:
try:
sock = eventlet.listen(bind_addr,
backlog=CONF.backlog,
family=address_family)
except socket.error as err:
if err.args[0] != errno.EADDRINUSE:
raise
eventlet.sleep(0.1)
if not sock:
raise RuntimeError(
_("Could not bind to %(host)s:%(port)s after"
" trying for 30 seconds") % {
'host': bind_addr[0],
'port': bind_addr[1]
})
return sock
def __call__(self, request):
"""WSGI method that controls (de)serialization and method dispatch."""
action_args = self.get_action_args(request.environ)
action = action_args.pop('action', None)
body_reject = strutils.bool_from_string(
action_args.pop('body_reject', None))
try:
if body_reject and self.deserializer.has_body(request):
msg = _('A body is not expected with this request.')
raise webob.exc.HTTPBadRequest(explanation=msg)
deserialized_request = self.dispatch(self.deserializer,
action, request)
action_args.update(deserialized_request)
action_result = self.dispatch(self.controller, action,
request, **action_args)
except webob.exc.WSGIHTTPException as e:
exc_info = sys.exc_info()
e = translate_exception(request, e)
six.reraise(type(e), e, exc_info[2])
except glare_exc.GlareException:
raise
except UnicodeDecodeError:
msg = _("Error decoding your request. Either the URL or the "
"request body contained characters that could not be "
"decoded by Glance")
raise webob.exc.HTTPBadRequest(explanation=msg)
except Exception as e:
LOG.exception(_LE("Caught error: %s"),
encodeutils.exception_to_unicode(e))
response = webob.exc.HTTPInternalServerError()
return response
try:
response = webob.Response(request=request)
self.dispatch(self.serializer, action, response, action_result)
# encode all headers in response to utf-8 to prevent unicode errors
for name, value in list(response.headers.items()):
if six.PY2 and isinstance(value, six.text_type):
response.headers[name] = encodeutils.safe_encode(value)
return response
except webob.exc.WSGIHTTPException as e:
return translate_exception(request, e)
except webob.exc.HTTPException as e:
return e
except glare_exc.GlareException:
raise
# return unserializable result (typically a webob exc)
except Exception:
return action_result
def _parse_sort_values(cls, sort):
"""Prepare sorting parameters for database."""
new_sort = []
for key, direction in sort:
if key not in cls.fields:
msg = _("The field %s doesn't exist.") % key
raise exception.BadRequest(msg)
# check if field can be sorted
if not cls.fields[key].sortable:
msg = _("The field %s is not sortable.") % key
raise exception.BadRequest(msg)
new_sort.append(
(key, direction, cls._get_field_type(cls.fields.get(key))))
return new_sort
def get_socket(default_port):
"""
Bind socket to bind ip:port in conf
note: Mostly comes from Swift with a few small changes...
:param default_port: port to bind to if none is specified in conf
:returns: a socket object as returned from socket.listen or
ssl.wrap_socket if conf specifies cert_file
"""
bind_addr = get_bind_addr(default_port)
# TODO(jaypipes): eventlet's greened socket module does not actually
# support IPv6 in getaddrinfo(). We need to get around this in the
# future or monitor upstream for a fix
address_family = [
addr[0] for addr in socket.getaddrinfo(bind_addr[0],
bind_addr[1],
socket.AF_UNSPEC,
socket.SOCK_STREAM)
if addr[0] in (socket.AF_INET, socket.AF_INET6)
][0]
use_ssl = CONF.key_file or CONF.cert_file
if use_ssl and (not CONF.key_file or not CONF.cert_file):
raise RuntimeError(_("When running server in SSL mode, you must "
"specify both a cert_file and key_file "
"option value in your configuration file"))
sock = utils.get_test_suite_socket()
retry_until = time.time() + 30
while not sock and time.time() < retry_until:
try:
sock = eventlet.listen(bind_addr,
backlog=CONF.backlog,
family=address_family)
except socket.error as err:
if err.args[0] != errno.EADDRINUSE:
raise
eventlet.sleep(0.1)
if not sock:
raise RuntimeError(_("Could not bind to %(host)s:%(port)s after"
" trying for 30 seconds") %
{'host': bind_addr[0],
'port': bind_addr[1]})
return sock
def validate(self, value):
for fc in self.forbidden_chars:
if fc in value:
raise ValueError(
_("Forbidden character %(char) found in string "
"%(string)s")
% {"char": fc, "string": value})
def validate(self, value):
l = len(value)
if l > self.size:
raise ValueError(
_("Number of items must be less than "
"%(size)s. Current size: %(cur)s") %
{'size': self.size, 'cur': l})
def version_select(*args, **kwargs):
"""Look for the method which matches the name supplied and version
constraints and calls it with the supplied arguments.
:returns: Returns the result of the method called
:raises: VersionNotFoundForAPIMethod if there is no method which
matches the name and version constraints
"""
# versioning is used in 3 classes: request deserializer and
# controller have request as first argument
# response serializer has response as first argument
# we must respect all three cases
if hasattr(args[0], 'api_version_request'):
ver = args[0].api_version_request
elif hasattr(args[0], 'request'):
ver = args[0].request.api_version_request
else:
raise exc.VersionNotFoundForAPIMethod(
message=_("Api version not found in the request."))
func_list = self.versioned_methods[key]
for func in func_list:
if ver.matches(func.start_version, func.end_version):
# Update the version_select wrapper function so
# other decorator attributes like wsgi.response
# are still respected.
functools.update_wrapper(version_select, func.func)
return func.func(self, *args, **kwargs)
# No version match
raise exc.VersionNotFoundForAPIMethod(version=ver)
def validate(self, value):
for item in value:
if item not in self.allowed_items:
raise ValueError(_("Key %(item)s is not allowed in dict. "
"Allowed key values: %(allowed)s") %
{"item": item,
"allowed": ', '.join(self.allowed_items)})
def validate(self, value):
for item in self.required_items:
if item not in value:
raise ValueError(_("Key %(item)s is required in dict. "
"Required key values: %(required)s") %
{"item": item,
"required": ', '.join(self.required_items)})
def validate(self, value):
l = len(value)
if l > self.size:
raise ValueError(
_("String length must be less than %(size)s. "
"Current size: %(cur)s") % {'size': self.size,
'cur': l})
def show_type_schema(cls, context, type_name):
policy.authorize("artifact:type_list", {}, context)
schemas = cls._get_schemas(cls.registry)
if type_name not in schemas:
msg = _("Artifact type %s does not exist") % type_name
raise exception.NotFound(message=msg)
return schemas[type_name]
def evaluate_filter_op(value, operator, threshold):
"""Evaluate a comparison operator.
Designed for use on a comparative-filtering query field.
:param value: evaluated against the operator, as left side of expression
:param operator: any supported filter operation
:param threshold: to compare value against, as right side of expression
:raises: InvalidFilterOperatorValue if an unknown operator is provided
:returns: boolean result of applied comparison
"""
if operator == 'gt':
return value > threshold
elif operator == 'gte':
return value >= threshold
elif operator == 'lt':
return value < threshold
elif operator == 'lte':
return value <= threshold
elif operator == 'neq':
return value != threshold
elif operator == 'eq':
return value == threshold
msg = _("Unable to filter on a unknown operator.")
raise exception.InvalidFilterOperatorValue(msg)
def version_select(*args, **kwargs):
"""Look for the method which matches the name supplied and version
constraints and calls it with the supplied arguments.
:returns: Returns the result of the method called
:raises: VersionNotFoundForAPIMethod if there is no method which
matches the name and version constraints
"""
# versioning is used in 3 classes: request deserializer and
# controller have request as first argument
# response serializer has response as first argument
# we must respect all three cases
if hasattr(args[0], 'api_version_request'):
ver = args[0].api_version_request
elif hasattr(args[0], 'request'):
ver = args[0].request.api_version_request
else:
raise exc.VersionNotFoundForAPIMethod(
message=_("Api version not found in the request."))
func_list = self.versioned_methods[key]
for func in func_list:
if ver.matches(func.start_version, func.end_version):
# Update the version_select wrapper function so
# other decorator attributes like wsgi.response
# are still respected.
functools.update_wrapper(version_select, func.func)
return func.func(self, *args, **kwargs)
# No version match
raise exc.VersionNotFoundForAPIMethod(version=ver)
def wrapper(*args, **kwargs):
def _is_match(some_str):
return (isinstance(some_str, six.text_type) and
REGEX_4BYTE_UNICODE.findall(some_str) != [])
def _check_dict(data_dict):
# a dict of dicts has to be checked recursively
for key, value in six.iteritems(data_dict):
if isinstance(value, dict):
_check_dict(value)
else:
if _is_match(key):
msg = _("Property names can't contain 4 byte unicode.")
raise exception.Invalid(msg)
if _is_match(value):
msg = (_("%s can't contain 4 byte unicode characters.")
% key.title())
raise exception.Invalid(msg)
for data_dict in [arg for arg in args if isinstance(arg, dict)]:
_check_dict(data_dict)
# now check args for str values
for arg in args:
if _is_match(arg):
msg = _("Param values can't contain 4 byte unicode.")
raise exception.Invalid(msg)
# check kwargs as well, as params are passed as kwargs via
# registry calls
_check_dict(kwargs)
return f(*args, **kwargs)
def _validate_versioning(cls, context, name, version, is_public=False):
if version is not None and name not in (None, ""):
filters = {'name': name, 'version': version,
'status': 'neq:deleted'}
if is_public is False:
filters.update({'owner': context.tenant,
'visibility': 'private'})
else:
filters.update({'visibility': 'public'})
if len(cls.list(context, MultiDict(filters))) > 0:
msg = _("Artifact with this name and version is already "
"exists for this owner.")
raise exception.Conflict(msg)
else:
msg = _("Cannot set artifact version without name and version.")
raise exception.BadRequest(msg)
def validate(self, value):
for item in value:
if item not in self.allowed_items:
raise ValueError(_("Key %(item)s is not allowed in dict. "
"Allowed key values: %(allowed)s") %
{"item": item,
"allowed": ', '.join(self.allowed_items)})
def _create_scoped_lock(self,
context,
type_name,
name,
version,
owner,
visibility='private'):
"""Create scoped lock for artifact."""
# validate that artifact doesn't exist for the scope
filters = [('name', 'eq:' + name), ('version', 'eq:' + version)]
if visibility == 'public':
filters.extend([('visibility', 'public')])
elif visibility == 'private':
filters.extend([('owner', 'eq:' + owner),
('visibility', 'private')])
scope_id = "%s:%s:%s" % (type_name, name, version)
if visibility != 'public':
scope_id += ':%s' % owner
lock = self.lock_engine.acquire(context, scope_id)
try:
if self.list(context, type_name, filters).get("total_count") > 0:
msg = _("Artifact with this name and version is already "
"exists for this scope.")
raise exception.Conflict(msg)
except Exception:
with excutils.save_and_reraise_exception(logger=LOG):
self.lock_engine.release(lock)
return lock
def __call__(self, value):
l = len(value)
if l < self.size:
raise ValueError(
_("Number of items must be greater than "
"%(size)d. Current size: %(cur)d") %
{'size': self.size, 'cur': l})
def validate(self, value):
l = len(value)
if l > self.size:
raise ValueError(
_("Number of items must be less than "
"%(size)s. Current size: %(cur)s") %
{'size': self.size, 'cur': l})
def validate(self, value):
l = len(value)
if l < self.size:
raise ValueError(
_("String length must be more than %(size)s. "
"Current size: %(cur)s") % {'size': self.size,
'cur': l})
def validate(self, value):
for fc in self.forbidden_chars:
if fc in value:
raise ValueError(
_("Forbidden character %(char) found in string "
"%(string)s")
% {"char": fc, "string": value})
def _get_content_type(req, expected=None):
"""Determine content type of the request body."""
if "Content-Type" not in req.headers:
msg = _("Content-Type must be specified.")
LOG.error(msg)
raise exc.BadRequest(msg)
content_type = req.content_type
if expected is not None and content_type not in expected:
msg = (_('Invalid content type: %(ct)s. Expected: %(exp)s') % {
'ct': content_type,
'exp': ', '.join(expected)
})
raise exc.UnsupportedMediaType(message=msg)
return content_type
def create(self, req):
self._get_content_type(req, expected=['application/json'])
body = self._get_request_body(req)
if not isinstance(body, dict):
msg = _("Dictionary expected as body value. Got %s.") % type(body)
raise exc.BadRequest(msg)
return {'values': body}
def wrapper(*args, **kwargs):
def _is_match(some_str):
return (isinstance(some_str, six.text_type)
and REGEX_4BYTE_UNICODE.findall(some_str) != [])
def _check_dict(data_dict):
# a dict of dicts has to be checked recursively
for key, value in six.iteritems(data_dict):
if isinstance(value, dict):
_check_dict(value)
else:
if _is_match(key):
msg = _("Property names can't contain 4 byte unicode.")
raise exception.Invalid(msg)
if _is_match(value):
msg = (
_("%s can't contain 4 byte unicode characters.") %
key.title())
raise exception.Invalid(msg)
for data_dict in [arg for arg in args if isinstance(arg, dict)]:
_check_dict(data_dict)
# now check args for str values
for arg in args:
if _is_match(arg):
msg = _("Param values can't contain 4 byte unicode.")
raise exception.Invalid(msg)
# check kwargs as well, as params are passed as kwargs via
# registry calls
_check_dict(kwargs)
return f(*args, **kwargs)
def evaluate_filter_op(value, operator, threshold):
"""Evaluate a comparison operator.
Designed for use on a comparative-filtering query field.
:param value: evaluated against the operator, as left side of expression
:param operator: any supported filter operation
:param threshold: to compare value against, as right side of expression
:raises: InvalidFilterOperatorValue if an unknown operator is provided
:returns: boolean result of applied comparison
"""
if operator == 'gt':
return value > threshold
elif operator == 'gte':
return value >= threshold
elif operator == 'lt':
return value < threshold
elif operator == 'lte':
return value <= threshold
elif operator == 'neq':
return value != threshold
elif operator == 'eq':
return value == threshold
msg = _("Unable to filter on a unknown operator.")
raise exception.InvalidFilterOperatorValue(msg)
def get_type_name(link):
url = link.split('/')
if len(url) == 4:
return url[2]
else:
raise ValueError(_("It is not possible to "
"extract type_name from link %s"), link)
def create(self, req):
self._get_content_type(req, expected=['application/json'])
body = self._get_request_body(req)
if not isinstance(body, dict):
msg = _("Dictionary expected as body value. Got %s.") % type(body)
raise exc.BadRequest(msg)
return {'values': body}
def deactivate(cls, context, af, values):
"""Deny Artifact downloading due to security concerns
If user uploaded suspicious Artifact then Cloud Admins(or other users -
it depends on policy configurations) can deny Artifact download by
users by making Artifact de-activated. After additional investigation
Artifact can be re-activated or deleted from Glare.
:param context: user context
:param af: Artifact definition in Glare
:return: definition of de-activated Artifact
"""
if values != {'status': cls.STATUS.DEACTIVATED}:
msg = _("Only {'status': %s} is allowed in a request "
"for deactivation.") % cls.STATUS.DEACTIVATED
raise exception.BadRequest(msg)
if af.status != cls.STATUS.ACTIVE:
raise exception.InvalidStatusTransition(
orig=af.status, new=cls.STATUS.ACTIVE
)
LOG.info(_LI("Parameters validation for artifact %(artifact)s "
"deactivate passed for request %(request)s."),
{'artifact': af.id, 'request': context.request_id})
af = cls.db_api.update(context, af.id, values)
return cls._init_artifact(context, af)
def show_type_schema(cls, context, type_name):
policy.authorize("artifact:type_get", {}, context)
schemas = cls._get_schemas(cls.registry)
if type_name not in schemas:
msg = _("Artifact type %s does not exist") % type_name
raise exception.NotFound(message=msg)
return schemas[type_name]
def __call__(self, value):
l = len(value)
if l < self.size:
raise ValueError(
_("String length must be more than %(size)d. "
"Current length: %(cur)d") % {'size': self.size,
'cur': l})
def get_location_info(url, context, max_size, calc_checksum=True):
"""Validate location and get information about external blob
:param url: blob url
:param context: user context
:param calc_checksum: define if checksum must be calculated
:return: blob size and checksum
"""
# validate uri
scheme = urlparse.urlparse(url).scheme
if scheme not in ('http', 'https'):
msg = _("Location %s is invalid.") % url
raise exception.BadRequest(message=msg)
res = urllib.urlopen(url)
http_message = res.info()
content_type = getattr(http_message, 'type') or 'application/octet-stream'
# calculate blob checksum to ensure that location blob won't be changed
# in future
# TODO(kairat) need to support external location signatures
checksum = None
size = 0
if calc_checksum:
checksum = hashlib.md5()
blob_data = load_from_store(url, context)
for buf in blob_data:
checksum.update(buf)
size += len(buf)
if size > max_size:
msg = _("External blob size %(size)d exceeds maximum allowed "
"size %(max)d."), {'size': size, 'max': max_size}
raise exception.BadRequest(message=msg)
checksum = checksum.hexdigest()
else:
# request blob size
size = get_blob_size(url, context=context)
if size < 0 or size > max_size:
msg = _("Invalid blob size %d.") % size
raise exception.BadRequest(message=msg)
LOG.debug("Checksum %(checksum)s and size %(size)s calculated "
"successfully for location %(location)s",
{'checksum': str(checksum), 'size': str(size),
'location': url})
return size, checksum, content_type
def _validate_input_values(cls, context, values):
# validate that we are not specifying any system attribute
# and that we do not upload blobs or add locations here
for field_name in values:
if field_name in cls.fields:
if cls.fields[field_name].system is True:
msg = _("Cannot specify system property %s. It is not "
"available for modifying by users.") % field_name
raise exception.Forbidden(msg)
elif cls.is_blob(field_name) or cls.is_blob_dict(field_name):
msg = _("Cannot add blob %s with this request. "
"Use special Blob API for that.") % field_name
raise exception.BadRequest(msg)
else:
msg = (_("Cannot add non-existing property %s to artifact. ")
% field_name)
raise exception.BadRequest(msg)
def _validate_filter_ops(cls, filter_name, op):
field = cls.fields.get(filter_name)
if op not in field.filter_ops:
msg = (_("Unsupported filter type '%s(key)'."
"The following filters are supported "
"%(filters)s") % {
'key': op, 'filters': str(field.filter_ops)})
raise exception.BadRequest(message=msg)