def add_state(self, state, state_reason):
"""
Add a RequestState associated with the current request.
:param str state: the state name
:param str state_reason: the reason explaining the state transition
:raises ValidationError: if the state is invalid
"""
try:
state_int = RequestStateMapping.__members__[state].value
except KeyError:
raise ValidationError(
'The state "{}" is invalid. It must be one of: {}.'.format(
state, ', '.join(RequestStateMapping.get_names())))
for s in ('complete', 'failed'):
# A complete or failed state cannot change states, but the state reason
# can be updated
if self.state and self.state.state_name == s and state != s:
raise ValidationError(
f'A {self.state.state_name} request cannot change states')
request_state = RequestState(state=state_int,
state_reason=state_reason)
self.states.append(request_state)
# Send the changes queued up in SQLAlchemy to the database's transaction buffer.
# This will generate an ID that can be used below.
db.session.add(request_state)
db.session.flush()
self.request_state_id = request_state.id
def from_add_json(cls, kwargs):
"""Handle JSON requests for the Add API endpoint."""
request_kwargs = deepcopy(kwargs)
bundles = request_kwargs.get('bundles', [])
if (not isinstance(bundles, list) or len(bundles) == 0
or any(not item or not isinstance(item, str)
for item in bundles)):
raise ValidationError(
f'"bundles" should be a non-empty array of strings')
for param in ('cnr_token', 'organization'):
if param not in request_kwargs:
continue
if not isinstance(request_kwargs[param], str):
raise ValidationError(f'"{param}" must be a string')
# Always remove cnr_token from request_kwargs since it's not stored in the database
request_kwargs.pop('cnr_token', None)
cls._from_json(
request_kwargs,
additional_required_params=['bundles'],
additional_optional_params=['from_index', 'organization'],
)
request_kwargs['bundles'] = [
Image.get_or_create(pull_specification=item) for item in bundles
]
request_kwargs['type'] = RequestTypeMapping.__members__['add'].value
request = cls(**request_kwargs)
request.add_state('in_progress', 'The request was initiated')
return request
def validate_batch(batch_id):
"""
Validate the input batch ID.
If the input batch ID is a string, it will be converted to an integer and returned.
:param int batch_id: the ID of the batch
:raise ValidationError: if the batch ID is invalid
:return: the validated batch ID
:rtype: int
"""
rv = batch_id
error_msg = 'The batch must be a positive integer'
if isinstance(batch_id, str):
try:
rv = int(batch_id)
except ValueError:
raise ValidationError(error_msg)
elif not isinstance(batch_id, int):
raise ValidationError(error_msg)
if rv < 1:
raise ValidationError(error_msg)
return rv
def from_json(cls, kwargs, batch=None):
"""
Handle JSON requests for the Add API endpoint.
:param dict kwargs: the JSON payload of the request.
:param Batch batch: the batch to specify with the request.
"""
request_kwargs = deepcopy(kwargs)
bundles = request_kwargs.get('bundles', [])
if not isinstance(bundles, list) or any(
not item or not isinstance(item, str) for item in bundles):
raise ValidationError(
'"bundles" should be either an empty array or an array of non-empty strings'
)
# Check if no bundles `from_index and `binary_image` are specified
# if no bundles and and no from index then a empty index will be created
# if no binary image and just a from_index then we are not updating anything and it would
# be a no-op
if not request_kwargs.get('bundles') and (
not request_kwargs.get('from_index')
or not request_kwargs.get('binary_image')):
raise ValidationError(
'"from_index" and "binary_image" must be specified if no bundles are specified'
)
for param in ('cnr_token', 'organization'):
if param not in request_kwargs:
continue
if not isinstance(request_kwargs[param], str):
raise ValidationError(f'"{param}" must be a string')
if not isinstance(request_kwargs.get('force_backport', False), bool):
raise ValidationError('"force_backport" must be a boolean')
# Remove attributes that are not stored in the database
request_kwargs.pop('cnr_token', None)
request_kwargs.pop('force_backport', None)
cls._from_json(
request_kwargs,
additional_optional_params=[
'from_index', 'organization', 'bundles'
],
batch=batch,
)
request_kwargs['bundles'] = [
Image.get_or_create(pull_specification=item) for item in bundles
]
request = cls(**request_kwargs)
request.add_state('in_progress', 'The request was initiated')
return request
def from_json(cls, kwargs, batch=None):
"""
Handle JSON requests for the Remove API endpoint.
:param dict kwargs: the JSON payload of the request.
:param Batch batch: the batch to specify with the request.
"""
request_kwargs = deepcopy(kwargs)
operators = request_kwargs.get('operators', [])
if (not isinstance(operators, list) or len(operators) == 0
or any(not item or not isinstance(item, str)
for item in operators)):
raise ValidationError(
f'"operators" should be a non-empty array of strings')
cls._from_json(request_kwargs,
additional_required_params=['operators', 'from_index'],
batch=batch)
request_kwargs['operators'] = [
Operator.get_or_create(name=item) for item in operators
]
request = cls(**request_kwargs)
request.add_state('in_progress', 'The request was initiated')
return request
def rm_operators():
"""
Submit a request to remove operators from an index image.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = RequestRm.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
overwrite_from_index = payload.get('overwrite_from_index')
args = _get_rm_args(payload, request, overwrite_from_index)
safe_args = _get_safe_args(args, payload)
error_callback = failed_request_callback.s(request.id)
try:
handle_rm_request.apply_async(
args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=_get_user_queue(serial=overwrite_from_index),
)
except kombu.exceptions.OperationalError:
handle_broker_error(request)
flask.current_app.logger.debug('Successfully scheduled request %d', request.id)
return flask.jsonify(request.to_json()), 201
def regenerate_bundle():
"""
Submit a request to regenerate an operator bundle image.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = RequestRegenerateBundle.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
error_callback = failed_request_callback.s(request.id)
handle_regenerate_bundle_request.apply_async(
args=[
payload['from_bundle_image'],
payload.get('organization'), request.id
],
link_error=error_callback,
queue=_get_user_queue(),
)
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def add_bundles():
"""
Submit a request to add operator bundles to an index image.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = Request.from_add_json(payload)
db.session.add(request)
db.session.commit()
args = [
payload['bundles'],
payload['binary_image'],
request.id,
payload.get('from_index'),
payload.get('add_arches'),
payload.get('cnr_token'),
payload.get('organization'),
_should_force_overwrite() or payload.get('overwrite_from_index'),
]
safe_args = copy.copy(args)
if payload.get('cnr_token'):
safe_args[safe_args.index(payload['cnr_token'])] = '*****'
error_callback = failed_request_callback.s(request.id)
handle_add_request.apply_async(args=args, link_error=error_callback, argsrepr=repr(safe_args))
flask.current_app.logger.debug('Successfully scheduled request %d', request.id)
return flask.jsonify(request.to_json()), 201
def rm_operators():
"""
Submit a request to remove operators from an index image.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = Request.from_remove_json(payload)
db.session.add(request)
db.session.commit()
error_callback = failed_request_callback.s(request.id)
handle_rm_request.apply_async(
args=[
payload['operators'],
payload['binary_image'],
request.id,
payload['from_index'],
payload.get('add_arches'),
_should_force_overwrite() or payload.get('overwrite_from_index'),
],
link_error=error_callback,
)
flask.current_app.logger.debug('Successfully scheduled request %d', request.id)
return flask.jsonify(request.to_json()), 201
def from_json(cls, kwargs, batch=None):
"""
Handle JSON requests for the Add API endpoint.
:param dict kwargs: the JSON payload of the request.
:param Batch batch: the batch to specify with the request.
"""
request_kwargs = deepcopy(kwargs)
bundles = request_kwargs.get('bundles', [])
if (not isinstance(bundles, list) or len(bundles) == 0
or any(not item or not isinstance(item, str)
for item in bundles)):
raise ValidationError(
f'"bundles" should be a non-empty array of strings')
for param in ('cnr_token', 'organization'):
if param not in request_kwargs:
continue
if not isinstance(request_kwargs[param], str):
raise ValidationError(f'"{param}" must be a string')
if not isinstance(request_kwargs.get('force_backport', False), bool):
raise ValidationError('"force_backport" must be a boolean')
# Remove attributes that are not stored in the database
request_kwargs.pop('cnr_token', None)
request_kwargs.pop('force_backport', None)
cls._from_json(
request_kwargs,
additional_required_params=['bundles'],
additional_optional_params=['from_index', 'organization'],
batch=batch,
)
request_kwargs['bundles'] = [
Image.get_or_create(pull_specification=item) for item in bundles
]
request = cls(**request_kwargs)
request.add_state('in_progress', 'The request was initiated')
return request
def regenerate_bundle_batch():
"""
Submit a batch of requests to regenerate operator bundle images.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
Batch.validate_batch_request_params(payload)
batch = Batch(annotations=payload.get('annotations'))
db.session.add(batch)
requests = []
# Iterate through all the build requests and verify that the requests are valid before
# committing them and scheduling the tasks
for build_request in payload['build_requests']:
try:
request = RequestRegenerateBundle.from_json(build_request, batch)
except ValidationError as e:
# Rollback the transaction if any of the build requests are invalid
db.session.rollback()
raise ValidationError(
f'{str(e).rstrip(".")}. This occurred on the build request in '
f'index {payload["build_requests"].index(build_request)}.')
db.session.add(request)
requests.append(request)
db.session.commit()
messaging.send_messages_for_new_batch_of_requests(requests)
request_jsons = []
# This list will be used for the log message below and avoids the need of having to iterate
# through the list of requests another time
request_id_strs = []
for build_request, request in zip(payload['build_requests'], requests):
request_jsons.append(request.to_json())
request_id_strs.append(str(request.id))
error_callback = failed_request_callback.s(request.id)
handle_regenerate_bundle_request.apply_async(
args=[
build_request['from_bundle_image'],
build_request.get('organization'),
request.id,
],
link_error=error_callback,
queue=_get_user_queue(),
)
flask.current_app.logger.debug(
'Successfully scheduled the batch %d with requests: %s',
batch.id,
', '.join(request_id_strs),
)
return flask.jsonify(request_jsons), 201
def validate_batch_request_params(payload):
"""
Validate batch specific parameters from the input JSON payload.
The requests in the "build_requests" key's value are not validated. Those should be
validated separately.
:raises ValidationError: if the payload is invalid
"""
if (not isinstance(payload, dict)
or not isinstance(payload.get('build_requests'), list)
or not payload['build_requests']):
raise ValidationError(
'The input data must be a JSON object and the "build_requests" value must be a '
'non-empty array')
if not isinstance(payload.get('annotations', {}), dict):
raise ValidationError(
'The value of "annotations" must be a JSON object')
def get_related_bundles(request_id):
"""
Retrieve the related bundle images from the bundle CSV for a regenerate-bundle request.
:param int request_id: the request ID that was passed in through the URL.
:rtype: flask.Response
:raise NotFound: if the request is not found or there are no related bundles for the request
:raise Gone: if the related bundles for the build request have been removed due to expiration
"""
request_related_bundles_dir = flask.current_app.config['IIB_REQUEST_RELATED_BUNDLES_DIR']
if not request_related_bundles_dir:
raise NotFound()
request = Request.query.get_or_404(request_id)
if request.type != RequestTypeMapping.regenerate_bundle.value:
raise ValidationError(
f'The request {request_id} is of type {request.type_name}. '
'This endpoint is only valid for requests of type regenerate-bundle.'
)
finalized = request.state.state_name in RequestStateMapping.get_final_states()
if not finalized:
raise ValidationError(
f'The request {request_id} is not complete yet.'
' related_bundles will be available once the request is complete.'
)
related_bundles_file_path = os.path.join(
request_related_bundles_dir, f'{request_id}_related_bundles.json'
)
if not os.path.exists(related_bundles_file_path):
expired = request.temporary_data_expiration < datetime.utcnow()
if expired:
raise Gone(f'The related_bundles for the build request {request_id} no longer exist')
raise IIBError(
'IIB is done processing the request and cannot find related_bundles. Please make '
f'sure the iib_organizaiton_customizations for organization {request.organization}'
' has related_bundles customization type set'
)
with open(related_bundles_file_path) as f:
return flask.Response(f.read(), mimetype='application/json')
def from_json(cls, kwargs, batch=None):
"""
Handle JSON requests for the Regenerate Bundle API endpoint.
:param dict kwargs: the JSON payload of the request.
:param Batch batch: the batch to specify with the request. If one is not specified, one will
be created automatically.
"""
batch = batch or Batch()
request_kwargs = deepcopy(kwargs)
validate_request_params(
request_kwargs,
required_params={'from_bundle_image'},
optional_params={'organization'},
)
# Validate organization is correctly provided
organization = request_kwargs.get('organization')
if organization and not isinstance(organization, str):
raise ValidationError('"organization" must be a string')
# Validate from_bundle_image is correctly provided
from_bundle_image = request_kwargs.pop('from_bundle_image')
if not isinstance(from_bundle_image, str):
raise ValidationError('"from_bundle_image" must be a string')
request_kwargs['from_bundle_image'] = Image.get_or_create(
pull_specification=from_bundle_image)
# current_user.is_authenticated is only ever False when auth is disabled
if current_user.is_authenticated:
request_kwargs['user'] = current_user
# Add the request to a new batch
db.session.add(batch)
request_kwargs['batch'] = batch
request = cls(**request_kwargs)
request.add_state('in_progress', 'The request was initiated')
return request
def validate_architecture_json(arches):
"""
Validate the JSON representation of architectures.
:param list arches: the JSON representation of architectures for a build request
:raise ValidationError: if the JSON does not match the required schema
"""
if not isinstance(arches, list) or any(
not arch or not isinstance(arch, str) for arch in arches):
raise ValidationError(
'Architectures should be specified as a non-empty array of strings'
)
def add_bundles():
"""
Submit a request to add operator bundles to an index image.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = RequestAdd.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
overwrite_from_index = _should_force_overwrite() or payload.get(
'overwrite_from_index')
celery_queue = _get_user_queue(serial=overwrite_from_index)
args = [
payload['bundles'],
payload['binary_image'],
request.id,
payload.get('from_index'),
payload.get('add_arches'),
payload.get('cnr_token'),
payload.get('organization'),
payload.get('force_backport'),
overwrite_from_index,
payload.get('overwrite_from_index_token'),
flask.current_app.config['IIB_GREENWAVE_CONFIG'].get(celery_queue),
]
safe_args = copy.copy(args)
if payload.get('cnr_token'):
safe_args[safe_args.index(payload['cnr_token'])] = '*****'
if payload.get('overwrite_from_index_token'):
safe_args[safe_args.index(
payload['overwrite_from_index_token'])] = '*****'
error_callback = failed_request_callback.s(request.id)
try:
handle_add_request.apply_async(args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue)
except kombu.exceptions.OperationalError:
handle_broker_error(request)
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def get_build_logs(request_id):
"""
Retrieve the logs for the build request.
:param int request_id: the request ID that was passed in through the URL.
:rtype: flask.Response
:raise NotFound: if the request is not found or there are no logs for the request
:raise Gone: if the logs for the build request have been removed due to expiration
:raise ValidationError: if the request has not completed yet
"""
request_log_dir = flask.current_app.config['IIB_REQUEST_LOGS_DIR']
s3_bucket_name = flask.current_app.config['IIB_AWS_S3_BUCKET_NAME']
if not s3_bucket_name and not request_log_dir:
raise NotFound()
request = Request.query.get_or_404(request_id)
finalized = request.state.state_name in RequestStateMapping.get_final_states(
)
if not finalized:
raise ValidationError(
f'The request {request_id} is not complete yet.'
' logs will be available once the request is complete.')
# If S3 bucket is configured, fetch the log file from the S3 bucket.
# Else, check if logs are stored on the system itself and return them.
# Otherwise, raise an IIBError.
if s3_bucket_name:
log_file = _get_artifact_file_from_s3_bucket(
'request_logs',
f'{request_id}.log',
request_id,
request.temporary_data_expiration,
s3_bucket_name,
)
return flask.Response(log_file.read(), mimetype='text/plain')
local_log_file_path = os.path.join(request_log_dir, f'{request_id}.log')
if not os.path.exists(local_log_file_path):
expired = request.temporary_data_expiration < datetime.utcnow()
if expired:
raise Gone(
f'The logs for the build request {request_id} no longer exist')
flask.current_app.logger.warning(
' Please make sure either an S3 bucket is configured or the logs are'
' stored locally in a directory by specifying IIB_REQUEST_LOGS_DIR'
)
raise IIBError(
'IIB is done processing the request and could not find logs.')
with open(local_log_file_path) as f:
return flask.Response(f.read(), mimetype='text/plain')
def validate_state(cls, state):
"""
Verify that the input state is valid.
:param str state: the state to validate
:raises iib.exceptions.ValidationError: if the state is invalid
"""
state_names = cls.get_names()
if state not in state_names:
states = ', '.join(state_names)
raise ValidationError(
f'{state} is not a valid build request state. Valid states are: {states}'
)
def validate_type(self, key, type_num):
"""
Verify the type number used is valid.
:param str key: the name of the database column
:param int type_num: the request type number to be verified
:return: the request type number
:rtype: int
:raises ValidationError: if the request type is invalid
"""
if not any(type_num == num.value for num in RequestTypeMapping):
raise ValidationError(
f'{type_num} is not a valid request type number')
return type_num
def validate_request_params(request_params, required_params, optional_params):
"""
Validate parameters for a build request.
All required parameters must be set in the request_params and
unknown parameters are not allowed.
:param dict request_params: the request parameters provided by the user
:param set required_params: the set of required parameters
:param set optional_params: the set of optional parameters
:raises iib.exceptions.ValidationError: if validation of parameters fails
"""
missing_params = required_params - request_params.keys()
if missing_params:
raise ValidationError('Missing required parameter(s): {}'.format(
', '.join(missing_params)))
# Don't allow the user to set arbitrary columns or relationships
invalid_params = request_params.keys() - required_params - optional_params
if invalid_params:
raise ValidationError(
'The following parameters are invalid: {}'.format(
', '.join(invalid_params)))
# Verify that all the required parameters are set and not empty
for param in required_params:
if not request_params.get(param):
raise ValidationError(f'"{param}" must be set')
# If any optional parameters are set but are empty, just remove them since they are
# treated as null values
for param in optional_params:
if (param in request_params
and not isinstance(request_params.get(param), bool)
and not request_params[param]):
del request_params[param]
def merge_index_image():
"""
Submit a request to merge two index images.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = RequestMergeIndexImage.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
overwrite_target_index = payload.get('overwrite_target_index', False)
celery_queue = _get_user_queue(serial=overwrite_target_index)
args = [
payload['source_from_index'],
payload.get('deprecation_list', []),
request.id,
payload.get('binary_image'),
payload.get('target_index'),
overwrite_target_index,
payload.get('overwrite_target_index_token'),
request.distribution_scope,
flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'],
payload.get('build_tags', []),
]
safe_args = _get_safe_args(args, payload)
error_callback = failed_request_callback.s(request.id)
try:
handle_merge_request.apply_async(args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue)
except kombu.exceptions.OperationalError:
handle_broker_error(request)
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def rm_operators():
"""
Submit a request to remove operators from an index image.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = RequestRm.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
args = [
payload['operators'],
payload['binary_image'],
request.id,
payload['from_index'],
payload.get('add_arches'),
_should_force_overwrite() or payload.get('overwrite_from_index'),
payload.get('overwrite_from_index_token'),
]
safe_args = copy.copy(args)
if payload.get('overwrite_from_index_token'):
safe_args[safe_args.index(
payload['overwrite_from_index_token'])] = '*****'
error_callback = failed_request_callback.s(request.id)
handle_rm_request.apply_async(
args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=_get_user_queue(),
)
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def create_empty_index():
"""
Submit a request to create an index image without bundles.
Note: Any duplicate bundle will be removed from payload.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
request = RequestCreateEmptyIndex.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
args = [
payload['from_index'],
request.id,
payload.get('output_fbc'),
payload.get('binary_image'),
payload.get('labels'),
flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'],
]
safe_args = _get_safe_args(args, payload)
error_callback = failed_request_callback.s(request.id)
try:
handle_create_empty_index_request.apply_async(
args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=_get_user_queue())
except kombu.exceptions.OperationalError:
handle_broker_error(request)
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def add_bundles():
"""
Submit a request to add operator bundles to an index image.
Note: Any duplicate bundle will be removed from payload.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
# Only run `_get_unique_bundles` if it is a list. If it's not, `from_json`
# will raise an error to the user.
if payload.get('bundles') and isinstance(payload['bundles'], list):
payload['bundles'] = _get_unique_bundles(payload['bundles'])
request = RequestAdd.from_json(payload)
db.session.add(request)
db.session.commit()
messaging.send_message_for_state_change(request, new_batch_msg=True)
overwrite_from_index = payload.get('overwrite_from_index')
celery_queue = _get_user_queue(serial=overwrite_from_index)
args = _get_add_args(payload, request, overwrite_from_index, celery_queue)
safe_args = _get_safe_args(args, payload)
error_callback = failed_request_callback.s(request.id)
try:
handle_add_request.apply_async(args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue)
except kombu.exceptions.OperationalError:
handle_broker_error(request)
flask.current_app.logger.debug('Successfully scheduled request %d',
request.id)
return flask.jsonify(request.to_json()), 201
def get_or_create(cls, pull_specification):
"""
Get the image from the database and create it if it doesn't exist.
:param str pull_specification: pull_specification of the image
:return: an Image object based on the input pull_specification; the Image object will be
added to the database session, but not committed, if it was created
:rtype: Image
:raise ValidationError: if pull_specification for the image is invalid
"""
if '@' not in pull_specification and ':' not in pull_specification:
raise ValidationError(
f'Image {pull_specification} should have a tag or a digest specified.'
)
image = cls.query.filter_by(
pull_specification=pull_specification).first()
if not image:
image = Image(pull_specification=pull_specification)
db.session.add(image)
return image
def from_remove_json(cls, kwargs):
"""Handle JSON requests for the Remove API endpoint."""
request_kwargs = deepcopy(kwargs)
operators = request_kwargs.get('operators', [])
if (not isinstance(operators, list) or len(operators) == 0
or any(not item or not isinstance(item, str)
for item in operators)):
raise ValidationError(
f'"operators" should be a non-empty array of strings')
cls._from_json(request_kwargs,
additional_required_params=['operators', 'from_index'])
request_kwargs['operators'] = [
Operator.get_or_create(name=item) for item in operators
]
request_kwargs['type'] = RequestTypeMapping.__members__['rm'].value
request = cls(**request_kwargs)
request.add_state('in_progress', 'The request was initiated')
return request
def patch_request(request_id):
"""
Modify the given request.
:param int request_id: the request ID from the URL
:return: a Flask JSON response
:rtype: flask.Response
:raise Forbidden: If the user trying to patch a request is not an IIB worker
:raise NotFound: if the request is not found
:raise ValidationError: if the JSON is invalid
"""
allowed_users = flask.current_app.config['IIB_WORKER_USERNAMES']
# current_user.is_authenticated is only ever False when auth is disabled
if current_user.is_authenticated and current_user.username not in allowed_users:
raise Forbidden('This API endpoint is restricted to IIB workers')
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
if not payload:
raise ValidationError(
'At least one key must be specified to update the request')
request = Request.query.get_or_404(request_id)
invalid_keys = payload.keys() - request.get_mutable_keys()
if invalid_keys:
raise ValidationError('The following keys are not allowed: {}'.format(
', '.join(invalid_keys)))
for key, value in payload.items():
if key == 'arches':
Architecture.validate_architecture_json(value)
elif key == 'bundle_mapping':
exc_msg = f'The "{key}" key must be an object with the values as lists of strings'
if not isinstance(value, dict):
raise ValidationError(exc_msg)
for v in value.values():
if not isinstance(v, list) or any(not isinstance(s, str)
for s in v):
raise ValidationError(exc_msg)
elif not value or not isinstance(value, str):
raise ValidationError(
f'The value for "{key}" must be a non-empty string')
if 'state' in payload and 'state_reason' not in payload:
raise ValidationError(
'The "state_reason" key is required when "state" is supplied')
elif 'state_reason' in payload and 'state' not in payload:
raise ValidationError(
'The "state" key is required when "state_reason" is supplied')
state_updated = False
if 'state' in payload and 'state_reason' in payload:
RequestStateMapping.validate_state(payload['state'])
new_state = payload['state']
new_state_reason = payload['state_reason']
# This is to protect against a Celery task getting executed twice and setting the
# state each time
if request.state.state == new_state and request.state.state_reason == new_state_reason:
flask.current_app.logger.info(
'Not adding a new state since it matches the last state')
else:
request.add_state(new_state, new_state_reason)
state_updated = True
image_keys = (
'binary_image_resolved',
'bundle_image',
'from_bundle_image_resolved',
'from_index_resolved',
'index_image',
)
for key in image_keys:
if key not in payload:
continue
key_value = payload.get(key, None)
key_object = Image.get_or_create(key_value)
# SQLAlchemy will not add the object to the database if it's already present
setattr(request, key, key_object)
for arch in payload.get('arches', []):
request.add_architecture(arch)
for operator, bundles in payload.get('bundle_mapping', {}).items():
operator_img = Operator.get_or_create(operator)
for bundle in bundles:
bundle_img = Image.get_or_create(bundle)
bundle_img.operator = operator_img
db.session.commit()
if state_updated:
messaging.send_message_for_state_change(request)
if current_user.is_authenticated:
flask.current_app.logger.info('The user %s patched request %d',
current_user.username, request.id)
else:
flask.current_app.logger.info('An anonymous user patched request %d',
request.id)
return flask.jsonify(request.to_json()), 200
def add_rm_batch():
"""
Submit a batch of requests to add or remove operators from an index image.
Note: Any duplicate bundle will be removed from payload when adding operators.
:rtype: flask.Response
:raise ValidationError: if required parameters are not supplied
"""
payload = flask.request.get_json()
Batch.validate_batch_request_params(payload)
batch = Batch(annotations=payload.get('annotations'))
db.session.add(batch)
requests = []
# Iterate through all the build requests and verify that the requests are valid before
# committing them and scheduling the tasks
for build_request in payload['build_requests']:
try:
if build_request.get('operators'):
# Check for the validity of a RM request
request = RequestRm.from_json(build_request, batch)
elif build_request.get('bundles'):
build_request_uniq = copy.deepcopy(build_request)
build_request_uniq['bundles'] = _get_unique_bundles(build_request_uniq['bundles'])
# Check for the validity of an Add request
request = RequestAdd.from_json(build_request_uniq, batch)
else:
raise ValidationError('Build request is not a valid Add/Rm request.')
except ValidationError as e:
raise ValidationError(
f'{str(e).rstrip(".")}. This occurred on the build request in '
f'index {payload["build_requests"].index(build_request)}.'
)
db.session.add(request)
requests.append(request)
db.session.commit()
messaging.send_messages_for_new_batch_of_requests(requests)
request_jsons = []
# This list will be used for the log message below and avoids the need of having to iterate
# through the list of requests another time
processed_request_ids = []
for build_request, request in zip(payload['build_requests'], requests):
request_jsons.append(request.to_json())
overwrite_from_index = build_request.get('overwrite_from_index')
celery_queue = _get_user_queue(serial=overwrite_from_index)
if isinstance(request, RequestAdd):
args = _get_add_args(build_request, request, overwrite_from_index, celery_queue)
elif isinstance(request, RequestRm):
args = _get_rm_args(build_request, request, overwrite_from_index)
safe_args = _get_safe_args(args, build_request)
error_callback = failed_request_callback.s(request.id)
try:
if isinstance(request, RequestAdd):
handle_add_request.apply_async(
args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue,
)
else:
handle_rm_request.apply_async(
args=args,
link_error=error_callback,
argsrepr=repr(safe_args),
queue=celery_queue,
)
except kombu.exceptions.OperationalError:
unprocessed_requests = [r for r in requests if str(r.id) not in processed_request_ids]
handle_broker_batch_error(unprocessed_requests)
processed_request_ids.append(str(request.id))
flask.current_app.logger.debug(
'Successfully scheduled the batch %d with requests: %s',
batch.id,
', '.join(processed_request_ids),
)
return flask.jsonify(request_jsons), 201
def get_builds():
"""
Retrieve the paginated build requests.
:rtype: flask.Response
"""
batch_id = flask.request.args.get('batch')
state = flask.request.args.get('state')
verbose = str_to_bool(flask.request.args.get('verbose'))
max_per_page = flask.current_app.config['IIB_MAX_PER_PAGE']
request_type = flask.request.args.get('request_type')
user = flask.request.args.get('user')
index_image = flask.request.args.get('index_image')
query_params = {}
# Create an alias class to load the polymorphic classes
poly_request = with_polymorphic(Request, '*')
query = poly_request.query.options(*get_request_query_options(
verbose=verbose))
if state:
query_params['state'] = state
RequestStateMapping.validate_state(state)
state_int = RequestStateMapping.__members__[state].value
query = query.join(Request.state)
query = query.filter(RequestState.state == state_int)
if batch_id is not None:
query_params['batch'] = batch_id
batch_id = Batch.validate_batch(batch_id)
query = query.filter_by(batch_id=batch_id)
if request_type:
query_params['request_type'] = request_type
RequestTypeMapping.validate_type(request_type)
request_type = request_type.replace('-', '_')
request_type_int = RequestTypeMapping.__members__[request_type].value
query = query.filter(Request.type == request_type_int)
if user:
# join with the user table and then filter on username
# request table only has the user_id
query_params['user'] = user
query = query.join(Request.user).filter(User.username == user)
if index_image:
query_params['index_image'] = index_image
# Get the image id of the image to be searched
image_result = Image.query.filter_by(
pull_specification=index_image).first()
if image_result:
# join with the Request* tables to get the response as image_ids are stored there
query = (query.outerjoin(
RequestCreateEmptyIndex,
Request.id == RequestCreateEmptyIndex.id).outerjoin(
RequestAdd, Request.id == RequestAdd.id).outerjoin(
RequestMergeIndexImage,
Request.id == RequestMergeIndexImage.id).outerjoin(
RequestRm, Request.id == RequestRm.id))
query = query.filter(
or_(
RequestCreateEmptyIndex.index_image_id == image_result.id,
RequestAdd.index_image_id == image_result.id,
RequestMergeIndexImage.index_image_id == image_result.id,
RequestRm.index_image_id == image_result.id,
))
# if index_image is not found in image table, then raise an error
else:
raise ValidationError(f'{index_image} is not a valid index image')
pagination_query = query.order_by(
Request.id.desc()).paginate(max_per_page=max_per_page)
requests = pagination_query.items
response = {
'items': [request.to_json(verbose=verbose) for request in requests],
'meta': pagination_metadata(pagination_query, **query_params),
}
return flask.jsonify(response)
def _from_json(request_kwargs,
additional_required_params=None,
additional_optional_params=None,
batch=None):
"""
Validate and process request agnostic parameters.
As part of the processing, the input ``request_kwargs`` parameter
is updated to reference database objects where appropriate.
:param dict request_kwargs: copy of args provided in API request
:param Batch batch: the batch to specify with the request. If one is not specified, one will
be created automatically.
"""
# Validate all required parameters are present
required_params = {'binary_image'} | set(additional_required_params
or [])
optional_params = {
'add_arches',
'overwrite_from_index',
'overwrite_from_index_token',
} | set(additional_optional_params or [])
validate_request_params(
request_kwargs,
required_params=required_params,
optional_params=optional_params,
)
# Check if both `from_index` and `add_arches` are not specified
if not request_kwargs.get('from_index') and not request_kwargs.get(
'add_arches'):
raise ValidationError(
'One of "from_index" or "add_arches" must be specified')
# Verify that `overwrite_from_index` is the correct type
overwrite = request_kwargs.pop('overwrite_from_index', False)
if not isinstance(overwrite, bool):
raise ValidationError(
'The "overwrite_from_index" parameter must be a boolean')
# Verify that `overwrite_from_index_token` is the correct type
overwrite_token = request_kwargs.pop('overwrite_from_index_token',
None)
if overwrite_token:
if not isinstance(overwrite_token, str):
raise ValidationError(
'The "overwrite_from_index_token" parameter must be a string'
)
if overwrite_token and not overwrite:
raise ValidationError(
'The "overwrite_from_index" parameter is required when'
' the "overwrite_from_index_token" parameter is used')
# Verify the user is authorized to use overwrite_from_index
# current_user.is_authenticated is only ever False when auth is disabled
if current_user.is_authenticated:
privileged_users = current_app.config['IIB_PRIVILEGED_USERNAMES']
if overwrite and not overwrite_token and current_user.username not in privileged_users:
raise Forbidden(
'You must be a privileged user to set "overwrite_from_index" without'
' setting "overwrite_from_index_token"')
# Validate add_arches are correctly provided
add_arches = request_kwargs.pop('add_arches', [])
Architecture.validate_architecture_json(add_arches)
# Validate binary_image is correctly provided
binary_image = request_kwargs.pop('binary_image')
if not isinstance(binary_image, str):
raise ValidationError('"binary_image" must be a string')
request_kwargs['binary_image'] = Image.get_or_create(
pull_specification=binary_image)
if 'from_index' in request_kwargs:
if not isinstance(request_kwargs['from_index'], str):
raise ValidationError('"from_index" must be a string')
request_kwargs['from_index'] = Image.get_or_create(
pull_specification=request_kwargs['from_index'])
# current_user.is_authenticated is only ever False when auth is disabled
if current_user.is_authenticated:
request_kwargs['user'] = current_user
# Add the request to a new batch
batch = batch or Batch()
db.session.add(batch)
request_kwargs['batch'] = batch