def get_updates(af_dict, patch_with_upd): """Get updated values for artifact and json patch :param af_dict: current artifact definition as dict :param patch_with_upd: json-patch :return: dict of updated attributes and their values """ try: af_dict_patched = patch_with_upd.apply(af_dict) diff = utils.DictDiffer(af_dict_patched, af_dict) # we mustn't add or remove attributes from artifact if diff.added() or diff.removed(): msg = _( "Forbidden to add or remove attributes from artifact. " "Added attributes %(added)s. " "Removed attributes %(removed)s") % { 'added': diff.added(), 'removed': diff.removed() } raise exception.BadRequest(message=msg) return {key: af_dict_patched[key] for key in diff.changed()} except (jsonpatch.JsonPatchException, jsonpatch.JsonPointerException, KeyError) as e: raise exception.BadRequest(message=str(e)) except TypeError as e: msg = _("Incorrect type of the element. Reason: %s") % str(e) raise exception.BadRequest(msg)
def list(self, req): params = req.params.copy() marker = params.pop('marker', None) query_params = {} # step 1 - apply marker to query if exists if marker is not None: query_params['marker'] = marker # step 2 - apply limit (if exists OR setup default limit) limit = params.pop('limit', CONF.default_api_limit) try: limit = int(limit) except ValueError: msg = _("Limit param must be an integer.") raise exc.BadRequest(message=msg) if limit < 0: msg = _("Limit param must be positive.") raise exc.BadRequest(message=msg) query_params['limit'] = min(CONF.max_api_limit, limit) # step 3 - parse sort parameters if 'sort' in params: sort = [] for sort_param in params.pop('sort').strip().split(','): key, _sep, direction = sort_param.partition(':') if direction and direction not in ('asc', 'desc'): raise exc.BadRequest('Sort direction must be one of ' '["asc", "desc"]. Got %s direction' % direction) sort.append((key, direction or 'desc')) query_params['sort'] = sort query_params['filters'] = params return query_params
def get_location_info(url, context, max_size, calc_checksum=True): """Validate location and get information about external blob :param url: blob url :param context: user context :param calc_checksum: define if checksum must be calculated :return: blob size and checksum """ # validate uri scheme = urlparse.urlparse(url).scheme if scheme not in ('http', 'https'): msg = _("Location %s is invalid.") % url raise exception.BadRequest(message=msg) res = urllib.urlopen(url) http_message = res.info() content_type = getattr(http_message, 'type') or 'application/octet-stream' # calculate blob checksum to ensure that location blob won't be changed # in future # TODO(kairat) need to support external location signatures checksum = None size = 0 if calc_checksum: checksum = hashlib.md5() blob_data = load_from_store(url, context) for buf in blob_data: checksum.update(buf) size += len(buf) if size > max_size: msg = _("External blob size %(size)d exceeds maximum allowed " "size %(max)d."), { 'size': size, 'max': max_size } raise exception.BadRequest(message=msg) checksum = checksum.hexdigest() else: # request blob size size = get_blob_size(url, context=context) if size < 0 or size > max_size: msg = _("Invalid blob size %d.") % size raise exception.BadRequest(message=msg) LOG.debug( "Checksum %(checksum)s and size %(size)s calculated " "successfully for location %(location)s", { 'checksum': str(checksum), 'size': str(size), 'location': url }) return size, checksum, content_type
def _get_blob_info(af, field_name, blob_key=None): """Return requested blob info.""" if blob_key: if not af.is_blob_dict(field_name): msg = _("%s is not a blob dict") % field_name raise exception.BadRequest(msg) return getattr(af, field_name).get(blob_key) else: if not af.is_blob(field_name): msg = _("%s is not a blob") % field_name raise exception.BadRequest(msg) return getattr(af, field_name, None)
def _check_dict(data_dict): # a dict of dicts has to be checked recursively for key, value in data_dict.items(): if isinstance(value, dict): _check_dict(value) else: if _is_match(key): msg = _("Property names can't contain 4 byte unicode.") raise exception.BadRequest(msg) if _is_match(value): msg = (_("%s can't contain 4 byte unicode characters.") % key.title()) raise exception.BadRequest(msg)
def _parse_sort_values(cls, sort): """Prepare sorting parameters for database.""" new_sort = [] for key, direction in sort: if key not in cls.fields: msg = _("The field %s doesn't exist.") % key raise exception.BadRequest(msg) # check if field can be sorted if not cls.fields[key].sortable: msg = _("The field %s is not sortable.") % key raise exception.BadRequest(msg) new_sort.append( (key, direction, cls._get_field_type(cls.fields.get(key)))) return new_sort
def create(self, req): self._get_content_type(req, expected=['application/json']) body = self._get_request_body(req) if not isinstance(body, dict): msg = _("Dictionary expected as body value. Got %s.") % type(body) raise exc.BadRequest(msg) return {'values': body}
def validate_status_transition(af, from_status, to_status): if from_status == 'deleted': msg = _("Cannot change status if artifact is deleted.") raise exception.Forbidden(msg) if to_status == 'active': if from_status == 'drafted': for name, type_obj in af.fields.items(): if type_obj.required_on_activate and getattr(af, name) is None: msg = _("'%s' field value must be set before " "activation.") % name raise exception.Forbidden(msg) elif to_status == 'drafted': if from_status != 'drafted': msg = _("Cannot change status to 'drafted'") % from_status raise exception.Forbidden(msg) elif to_status == 'deactivated': if from_status not in ('active', 'deactivated'): msg = _("Cannot deactivate artifact if it's not active.") raise exception.Forbidden(msg) elif to_status == 'deleted': msg = _("Cannot delete artifact with PATCH requests. Use special " "API to do this.") raise exception.Forbidden(msg) else: msg = _("Unknown artifact status: %s.") % to_status raise exception.BadRequest(msg)
def wrapper(*args, **kwargs): def _is_match(some_str): return (isinstance(some_str, six.text_type) and REGEX_4BYTE_UNICODE.findall(some_str) != []) def _check_dict(data_dict): # a dict of dicts has to be checked recursively for key, value in data_dict.items(): if isinstance(value, dict): _check_dict(value) else: if _is_match(key): msg = _("Property names can't contain 4 byte unicode.") raise exception.BadRequest(msg) if _is_match(value): msg = (_("%s can't contain 4 byte unicode characters.") % key.title()) raise exception.BadRequest(msg) for data_dict in [arg for arg in args if isinstance(arg, dict)]: _check_dict(data_dict) # now check args for str values for arg in args: if _is_match(arg): msg = _("Param values can't contain 4 byte unicode.") raise exception.BadRequest(msg) # check kwargs as well, as params are passed as kwargs via # registry calls _check_dict(kwargs) return f(*args, **kwargs)
def update(self, req): self._get_content_type(req, expected=['application/json-patch+json']) body = self._get_request_body(req) patch = jsonpatch.JsonPatch(body) try: # Initially patch object doesn't validate input. It's only checked # we call get operation on each method tuple(map(patch._get_operation, patch.patch)) except (jsonpatch.InvalidJsonPatch, TypeError): msg = _("Json Patch body is malformed") raise exc.BadRequest(msg) for patch_item in body: if patch_item['path'] == '/tags': msg = _("Cannot modify artifact tags with PATCH " "request. Use special Tag API for that.") raise exc.BadRequest(msg) return {'patch': patch}
def set_tags(self, req): self._get_content_type(req, expected=['application/json']) body = self._get_request_body(req) if 'tags' not in body: msg = _("Tag list must be in the body of request.") raise exc.BadRequest(msg) return {'tag_list': body['tags']}
def create(self, req, type_name, values): """Create artifact record in Glare. :param req: user request :param type_name: artifact type name :param values: dict with artifact fields :return: definition of created artifact """ if req.context.project_id is None or req.context.read_only: msg = _("It's forbidden to anonymous users to create artifacts.") raise exc.Forbidden(msg) if not values.get('name'): msg = _("Name must be specified at creation.") raise exc.BadRequest(msg) for field in ('visibility', 'status', 'display_type_name'): if field in values: msg = _("%s is not allowed in a request at creation.") % field raise exc.BadRequest(msg) return self.engine.create(req.context, type_name, values)
def _validate_filter_ops(cls, filter_name, op): field = cls.fields.get(filter_name) if op not in field.filter_ops: msg = (_("Unsupported filter type '%(key)s'." "The following filters are supported " "%(filters)s") % { 'key': op, 'filters': str(field.filter_ops) }) raise exception.BadRequest(message=msg)
def coerce_wrapper(obj, attr, value): try: val = coerce_func(obj, attr, value) if val is not None: for check_func in vals: check_func(val) return val except (KeyError, ValueError) as e: msg = "Type: %s. Field: %s. Exception: %s" % ( obj.get_type_name(), attr, str(e)) raise exc.BadRequest(message=msg)
def create(self, context, values): global DATA values['created_at'] = values['updated_at'] = timeutils.utcnow() artifact_id = values['id'] if artifact_id in DATA['artifacts']: msg = _("Artifact with id '%s' already exists") % artifact_id raise glare_exc.BadRequest(msg) values['_type'] = self.type DATA['artifacts'][artifact_id] = values return values
def _deserialize_blob(self, req): content_type = self._get_content_type(req) if content_type == ('application/vnd+openstack.glare-custom-location' '+json'): data = self._get_request_body(req) if 'url' not in data: msg = _("url is required when specifying external location. " "Cannot find url in body: %s") % str(data) raise exc.BadRequest(msg) else: data = req.body_file return {'data': data, 'content_type': content_type}
def validate_visibility_transition(af, from_visibility, to_visibility): if to_visibility == 'private': if from_visibility != 'private': msg = _("Cannot make artifact private again.") raise exception.Forbidden() elif to_visibility == 'public': if af.status != 'active': msg = _("Cannot change visibility to 'public' if artifact" " is not active.") raise exception.Forbidden(msg) else: msg = _("Unknown artifact visibility: %s.") % to_visibility raise exception.BadRequest(msg)
def update(self, req): self._get_content_type(req, expected=['application/json-patch+json']) body = self._get_request_body(req) patch = jsonpatch.JsonPatch(body) try: # Initially patch object doesn't validate input. It's only checked # when we call get operation on each method tuple(map(patch._get_operation, patch.patch)) except (jsonpatch.InvalidJsonPatch, TypeError, AttributeError, jsonpatch.JsonPointerException): msg = _("Json Patch body is malformed") raise exc.BadRequest(msg) return {'patch': patch}
def authenticate(self, access_token, realm_name): info = None if self.mcclient: info = self.mcclient.get(access_token) if info is None and CONF.keycloak_oidc.user_info_endpoint_url: url = self.url_template % realm_name verify = None if urllib.parse.urlparse(url).scheme == "https": verify = False if self.insecure else self.cafile cert = (self.certfile, self.keyfile) \ if self.certfile and self.keyfile else None try: resp = requests.get( url, headers={"Authorization": "Bearer %s" % access_token}, verify=verify, cert=cert ) except requests.ConnectionError: msg = _("Can't connect to keycloak server with address '%s'." ) % CONF.keycloak_oidc.auth_url LOG.error(msg) raise exception.GlareException(message=msg) if resp.status_code == 400: raise exception.BadRequest(message=resp.text) if resp.status_code == 401: LOG.warning("HTTP response from OIDC provider:" " [%s] with WWW-Authenticate: [%s]", pprint.pformat(resp.text), resp.headers.get("WWW-Authenticate")) raise exception.Unauthorized(message=resp.text) if resp.status_code == 403: raise exception.Forbidden(message=resp.text) elif resp.status_code > 400: raise exception.GlareException(message=resp.text) if self.mcclient: self.mcclient.set(access_token, resp.json(), time=CONF.keycloak_oidc.token_cache_time) info = resp.json() LOG.debug("HTTP response from OIDC provider: %s", pprint.pformat(info)) return info
def upload_blob(self, req): content_type = self._get_content_type(req) content_length = self._get_content_length(req) if content_type == ('application/vnd+openstack.glare-custom-location' '+json'): data = self._get_request_body(req) if 'url' not in data: msg = _("url is required when specifying external location. " "Cannot find 'url' in request body: %s") % str(data) raise exc.BadRequest(msg) location_type = data.get('location_type', 'external') if location_type not in self.ALLOWED_LOCATION_TYPES: msg = (_("Incorrect location type '%(location_type)s'. It " "must be one of the following %(allowed)s") % { 'location_type': location_type, 'allowed': ', '.join(self.ALLOWED_LOCATION_TYPES) }) raise exc.BadRequest(msg) if location_type == 'external': url = data.get('url') if not url.startswith('http'): msg = _("Url '%s' doesn't have http(s) scheme") % url raise exc.BadRequest(msg) if 'md5' not in data: msg = _("Incorrect blob metadata. MD5 must be specified " "for external location in artifact blob.") raise exc.BadRequest(msg) else: data = req.body_file if self.is_valid_encoding(req) and self.is_valid_method(req): req.is_body_readable = True return { 'data': data, 'content_type': content_type, 'content_length': content_length }
def __init__(self, version_string): """Create an API version request object. :param version_string: String representation of APIVersionRequest. Correct format is 'X.Y', where 'X' and 'Y' are int values. """ match = re.match(r"^([1-9]\d*)\.([1-9]\d*|0)$", version_string) if match: self.ver_major = int(match.group(1)) self.ver_minor = int(match.group(2)) else: msg = _("API version string %s is not valid. " "Cannot determine API version.") % version_string raise exception.BadRequest(msg)
def set_quotas(self, req): self._get_content_type(req, expected=['application/json']) body = self._get_request_body(req) try: jsonschema.validate(body, QUOTA_INPUT_SCHEMA) except jsonschema.exceptions.ValidationError as e: raise exc.BadRequest(e) values = {} for item in body: project_id = item['project_id'] values[project_id] = {} for quota in item['project_quotas']: values[project_id][quota['quota_name']] = quota['quota_value'] return {'values': values}
def _get_content_length(req): """Determine content length of the request body.""" if req.content_length is None: return try: content_length = int(req.content_length) if content_length < 0: raise ValueError except ValueError: msg = _("Content-Length must be a non negative integer.") LOG.error(msg) raise exc.BadRequest(msg) return content_length
def _get_content_type(req, expected=None): """Determine content type of the request body.""" if "Content-Type" not in req.headers: msg = _("Content-Type must be specified.") LOG.error(msg) raise exc.BadRequest(msg) content_type = req.content_type if expected is not None and content_type not in expected: msg = (_('Invalid content type: %(ct)s. Expected: %(exp)s') % { 'ct': content_type, 'exp': ', '.join(expected) }) raise exc.UnsupportedMediaType(message=msg) return content_type
def validate_change_allowed(af, field_name): """Validate if fields can be set for the artifact.""" if field_name not in af.fields: msg = _("Cannot add new field '%s' to artifact.") % field_name raise exception.BadRequest(msg) if af.status not in ('active', 'drafted'): msg = _("Forbidden to change fields " "if artifact is not active or drafted.") raise exception.Forbidden(message=msg) if af.fields[field_name].system is True: msg = _("Forbidden to specify system field %s. It is not " "available for modifying by users.") % field_name raise exception.Forbidden(msg) if af.status == 'active' and not af.fields[field_name].mutable: msg = (_("Forbidden to change field '%s' after activation.") % field_name) raise exception.Forbidden(message=msg)
def _create_or_update(context, artifact_id, values, session): with session.begin(): _drop_protected_attrs(models.Artifact, values) if artifact_id is None: if 'type_name' not in values: msg = _('Type name must be set.') raise exception.BadRequest(msg) # create new artifact artifact = models.Artifact() if 'id' not in values: artifact.id = str(uuid.uuid4()) else: artifact.id = values.pop('id') artifact.created_at = timeutils.utcnow() else: # update the existing artifact artifact = _get(context, artifact_id, session) if 'version' in values: values['version'] = semver_db.parse(values['version']) if 'tags' in values: tags = values.pop('tags') artifact.tags = _do_tags(artifact, tags) if 'properties' in values: properties = values.pop('properties', {}) artifact.properties = _do_properties(artifact, properties) if 'blobs' in values: blobs = values.pop('blobs') artifact.blobs = _do_blobs(artifact, blobs) artifact.updated_at = timeutils.utcnow() if 'status' in values and values['status'] == 'active': artifact.activated_at = timeutils.utcnow() artifact.update(values) artifact.save(session=session) return artifact.to_dict()
def create(self, context, type_name, values): """Create artifact record in Glare. :param context: user context :param type_name: artifact type name :param values: dict with artifact fields :return: dict representation of created artifact """ action_name = "artifact:create" policy.authorize(action_name, values, context) artifact_type = registry.ArtifactRegistry.get_artifact_type(type_name) version = values.get('version', artifact_type.DEFAULT_ARTIFACT_VERSION) init_values = { 'id': uuidutils.generate_uuid(), 'name': values.pop('name'), 'version': version, 'owner': context.project_id, 'created_at': timeutils.utcnow(), 'updated_at': timeutils.utcnow() } af = artifact_type.init_artifact(context, init_values) # acquire scoped lock and execute artifact create with self._create_scoped_lock(context, type_name, af.name, af.version, context.project_id): quota.verify_artifact_count(context, type_name) for field_name, value in values.items(): if af.is_blob(field_name) or af.is_blob_dict(field_name): msg = _("Cannot add blob with this request. " "Use special Blob API for that.") raise exception.BadRequest(msg) utils.validate_change_allowed(af, field_name) setattr(af, field_name, value) artifact_type.pre_create_hook(context, af) af = af.create(context) artifact_type.post_create_hook(context, af) # notify about new artifact Notifier.notify(context, action_name, af) # return artifact to the user return af.to_dict()
def download_blob(self, context, type_name, artifact_id, field_name, blob_key=None): """Download binary data from Glare Artifact. :param context: user context :param type_name: name of artifact type :param artifact_id: id of the artifact to be updated :param field_name: name of blob or blob dict field :param blob_key: if field_name is blob dict it specifies key in this dict :return: file iterator for requested file """ download_from_any_artifact = False if policy.authorize("artifact:download_from_any_artifact", {}, context, do_raise=False): download_from_any_artifact = True af = self._show_artifact(context, type_name, artifact_id, read_only=True, get_any_artifact=download_from_any_artifact) if not download_from_any_artifact: policy.authorize("artifact:download", af.to_dict(), context) blob_name = self._generate_blob_name(field_name, blob_key) if af.status == 'deleted': msg = _("Cannot download data when artifact is deleted") raise exception.Forbidden(message=msg) blob = self._get_blob_info(af, field_name, blob_key) if blob is None: msg = _("No data found for blob %s") % blob_name raise exception.NotFound(message=msg) if blob['status'] != 'active': msg = _("%s is not ready for download") % blob_name raise exception.Conflict(message=msg) af.pre_download_hook(context, af, field_name, blob_key) meta = { 'md5': blob.get('md5'), 'sha1': blob.get('sha1'), 'sha256': blob.get('sha256'), 'external': blob.get('external') } if blob['external']: data = {'url': blob['url']} else: data = store_api.load_from_store(uri=blob['url'], context=context) meta['size'] = blob.get('size') meta['content_type'] = blob.get('content_type') try: # call download hook in the end data = af.post_download_hook(context, af, field_name, blob_key, data) except exception.GlareException: raise except Exception as e: raise exception.BadRequest(message=str(e)) return data, meta
def get_updates(af_dict, patch_with_upd): """Get updated values for artifact and json patch :param af_dict: current artifact definition as dict :param patch_with_upd: json-patch :return: dict of updated attributes and their values """ class DictDiffer(object): """ Calculate the difference between two dictionaries as: (1) items added (2) items removed (3) keys same in both but changed values (4) keys same in both and unchanged values """ def __init__(self, current_dict, past_dict): self.current_dict, self.past_dict = current_dict, past_dict self.current_keys, self.past_keys = [ set(d.keys()) for d in (current_dict, past_dict) ] self.intersect = self.current_keys.intersection( self.past_keys) def added(self): return self.current_keys - self.intersect def removed(self): return self.past_keys - self.intersect def changed(self): return set(o for o in self.intersect if self.past_dict[o] != self.current_dict[o]) def unchanged(self): return set(o for o in self.intersect if self.past_dict[o] == self.current_dict[o]) try: af_dict_patched = patch_with_upd.apply(af_dict) diff = DictDiffer(af_dict_patched, af_dict) # we mustn't add or remove attributes from artifact if diff.added() or diff.removed(): msg = _( "Forbidden to add or remove attributes from artifact. " "Added attributes %(added)s. " "Removed attributes %(removed)s") % { 'added': diff.added(), 'removed': diff.removed() } raise exception.BadRequest(message=msg) return {key: af_dict_patched[key] for key in diff.changed()} except (jsonpatch.JsonPatchException, jsonpatch.JsonPointerException, KeyError) as e: raise exception.BadRequest(message=e.message) except TypeError as e: msg = _("Incorrect type of the element. Reason: %s") % str(e) raise exception.BadRequest(msg)
def _do_paginate_query(query, marker=None, limit=None, sort=None): # Add sorting number_of_custom_props = 0 for sort_key, sort_dir, sort_type in sort: try: sort_dir_func = { 'asc': sqlalchemy.asc, 'desc': sqlalchemy.desc, }[sort_dir] except KeyError: msg = _("Unknown sort direction, must be 'desc' or 'asc'.") raise exception.BadRequest(msg) # Note(mfedosin): Workaround to deal with situation that sqlalchemy # cannot work with composite keys correctly if sort_key == 'version': query = query.order_by(sort_dir_func(models.Artifact.version_prefix))\ .order_by(sort_dir_func(models.Artifact.version_suffix))\ .order_by(sort_dir_func(models.Artifact.version_meta)) elif sort_key in BASE_ARTIFACT_PROPERTIES: # sort by generic property query = query.order_by( sort_dir_func(getattr(models.Artifact, sort_key))) else: # sort by custom property number_of_custom_props += 1 if number_of_custom_props > 1: msg = _("For performance sake it's not allowed to sort by " "more than one custom property with this db backend.") raise exception.BadRequest(msg) prop_table = aliased(models.ArtifactProperty) query = (query.join(prop_table).filter( prop_table.name == sort_key).order_by( sort_dir_func(getattr(prop_table, sort_type + '_value')))) # Add pagination if marker is not None: marker_values = [] for sort_key, __, __ in sort: v = marker.get(sort_key, None) marker_values.append(v) # Build up an array of sort criteria as in the docstring criteria_list = [] for i in range(len(sort)): crit_attrs = [] for j in range(i): value = marker_values[j] if sort[j][0] in BASE_ARTIFACT_PROPERTIES: if sort[j][0] == 'version': value = semver_db.parse(value) crit_attrs.append( [getattr(models.Artifact, sort[j][0]) == value]) else: conds = [models.ArtifactProperty.name == sort[j][0]] conds.extend([ getattr(models.ArtifactProperty, sort[j][2] + '_value') == value ]) crit_attrs.append(conds) value = marker_values[i] sort_dir_func = operator.gt if sort[i][1] == 'asc' else operator.lt if sort[i][0] in BASE_ARTIFACT_PROPERTIES: if sort[i][0] == 'version': value = semver_db.parse(value) crit_attrs.append([ sort_dir_func(getattr(models.Artifact, sort[i][0]), value) ]) else: query = query.join(models.ArtifactProperty, aliased=True) conds = [models.ArtifactProperty.name == sort[i][0]] conds.extend([ sort_dir_func( getattr(models.ArtifactProperty, sort[i][2] + '_value'), value) ]) crit_attrs.append(conds) criteria = [and_(*crit_attr) for crit_attr in crit_attrs] criteria_list.append(criteria) criteria_list = [and_(*cr) for cr in criteria_list] query = query.filter(or_(*criteria_list)) if limit is not None: query = query.group_by(models.Artifact.id) query = query.limit(limit) return query