def get_builds():
"""
Retrieve the paginated build requests.
:rtype: flask.Response
"""
state = flask.request.args.get('state')
verbose = str_to_bool(flask.request.args.get('verbose'))
max_per_page = flask.current_app.config['IIB_MAX_PER_PAGE']
query = Request.query.options(*Request.get_query_options(verbose=verbose))
if state:
RequestStateMapping.validate_state(state)
state_int = RequestStateMapping.__members__[state].value
query = query.join(Request.state)
query = query.filter(RequestState.state == state_int)
pagination_query = query.order_by(Request.id.desc()).paginate(max_per_page=max_per_page)
requests = pagination_query.items
query_params = {}
if state:
query_params['state'] = state
if verbose:
query_params['verbose'] = verbose
response = {
'items': [request.to_json(verbose=verbose) for request in requests],
'meta': pagination_metadata(pagination_query, **query_params),
}
return flask.jsonify(response)
def get_build_logs(request_id):
"""
Retrieve the logs for the build request.
:param int request_id: the request ID that was passed in through the URL.
:rtype: flask.Response
:raise NotFound: if the request is not found or there are no logs for the request
:raise Gone: if the logs for the build request have been removed due to expiration
"""
request_log_dir = flask.current_app.config['IIB_REQUEST_LOGS_DIR']
if not request_log_dir:
raise NotFound()
request = Request.query.get_or_404(request_id)
log_file_path = os.path.join(request_log_dir, f'{request_id}.log')
if not os.path.exists(log_file_path):
expired = request.logs_expiration < datetime.utcnow()
if expired:
raise Gone(
f'The logs for the build request {request_id} no longer exist')
finalized = request.state.state_name in RequestStateMapping.get_final_states(
)
if finalized:
raise NotFound()
# The request may not have been initiated yet. Return empty logs until it's processed.
return flask.Response('', mimetype='text/plain')
with open(log_file_path) as f:
return flask.Response(f.read(), mimetype='text/plain')
def get_builds():
"""
Retrieve the paginated build requests.
:rtype: flask.Response
"""
batch_id = flask.request.args.get('batch')
state = flask.request.args.get('state')
verbose = str_to_bool(flask.request.args.get('verbose'))
max_per_page = flask.current_app.config['IIB_MAX_PER_PAGE']
# Create an alias class to load the polymorphic classes
poly_request = with_polymorphic(Request, '*')
query = poly_request.query.options(*get_request_query_options(
verbose=verbose))
if state:
RequestStateMapping.validate_state(state)
state_int = RequestStateMapping.__members__[state].value
query = query.join(Request.state)
query = query.filter(RequestState.state == state_int)
if batch_id is not None:
batch_id = Batch.validate_batch(batch_id)
query = query.filter_by(batch_id=batch_id)
pagination_query = query.order_by(
Request.id.desc()).paginate(max_per_page=max_per_page)
requests = pagination_query.items
query_params = {}
if state:
query_params['state'] = state
if verbose:
query_params['verbose'] = verbose
if batch_id:
query_params['batch'] = batch_id
response = {
'items': [request.to_json(verbose=verbose) for request in requests],
'meta': pagination_metadata(pagination_query, **query_params),
}
return flask.jsonify(response)
def _get_batch_state_change_envelope(batch, new_batch=False):
"""
Generate a batch state change ``Envelope`` object.
No message will be generated if IIB is not configured to send batch state change messages or
no batch state change message is needed .
:param iib.web.models.Batch batch: the batch that changed states
:param bool new_batch: if ``True``, a new batch message will be generated; if ``False``,
IIB will generate a batch state change message if the batch is no longer ``in_progress``
:return: the ``Envelope`` for the batch state change or ``None``
:rtype: Envelope or None
"""
batch_address = current_app.config.get(
'IIB_MESSAGING_BATCH_STATE_DESTINATION')
if not batch_address:
current_app.logger.debug(
'No batch state change message will be generated since the configuration '
'"IIB_MESSAGING_BATCH_STATE_DESTINATION" is not set')
return
if new_batch:
# Avoid querying the database for the batch state since we know it's a new batch
batch_state = 'in_progress'
else:
batch_state = batch.state
if new_batch or batch_state in RequestStateMapping.get_final_states():
current_app.logger.debug(
'Preparing to send a state change message for batch %d', batch.id)
batch_username = getattr(batch.user, 'username', None)
content = {
'batch':
batch.id,
'annotations':
batch.annotations,
'requests': [{
'id': request.id,
'organization': getattr(request, 'organization', None),
'request_type': request.type_name,
} for request in batch.requests],
'state':
batch_state,
'user':
batch_username,
}
properties = {
'batch': batch.id,
'state': batch_state,
'user': batch_username,
}
return json_to_envelope(batch_address, content, properties)
def get_build_logs(request_id):
"""
Retrieve the logs for the build request.
:param int request_id: the request ID that was passed in through the URL.
:rtype: flask.Response
:raise NotFound: if the request is not found or there are no logs for the request
:raise Gone: if the logs for the build request have been removed due to expiration
:raise ValidationError: if the request has not completed yet
"""
request_log_dir = flask.current_app.config['IIB_REQUEST_LOGS_DIR']
s3_bucket_name = flask.current_app.config['IIB_AWS_S3_BUCKET_NAME']
if not s3_bucket_name and not request_log_dir:
raise NotFound()
request = Request.query.get_or_404(request_id)
finalized = request.state.state_name in RequestStateMapping.get_final_states(
)
if not finalized:
raise ValidationError(
f'The request {request_id} is not complete yet.'
' logs will be available once the request is complete.')
# If S3 bucket is configured, fetch the log file from the S3 bucket.
# Else, check if logs are stored on the system itself and return them.
# Otherwise, raise an IIBError.
if s3_bucket_name:
log_file = _get_artifact_file_from_s3_bucket(
'request_logs',
f'{request_id}.log',
request_id,
request.temporary_data_expiration,
s3_bucket_name,
)
return flask.Response(log_file.read(), mimetype='text/plain')
local_log_file_path = os.path.join(request_log_dir, f'{request_id}.log')
if not os.path.exists(local_log_file_path):
expired = request.temporary_data_expiration < datetime.utcnow()
if expired:
raise Gone(
f'The logs for the build request {request_id} no longer exist')
flask.current_app.logger.warning(
' Please make sure either an S3 bucket is configured or the logs are'
' stored locally in a directory by specifying IIB_REQUEST_LOGS_DIR'
)
raise IIBError(
'IIB is done processing the request and could not find logs.')
with open(local_log_file_path) as f:
return flask.Response(f.read(), mimetype='text/plain')
def get_related_bundles(request_id):
"""
Retrieve the related bundle images from the bundle CSV for a regenerate-bundle request.
:param int request_id: the request ID that was passed in through the URL.
:rtype: flask.Response
:raise NotFound: if the request is not found or there are no related bundles for the request
:raise Gone: if the related bundles for the build request have been removed due to expiration
"""
request_related_bundles_dir = flask.current_app.config['IIB_REQUEST_RELATED_BUNDLES_DIR']
if not request_related_bundles_dir:
raise NotFound()
request = Request.query.get_or_404(request_id)
if request.type != RequestTypeMapping.regenerate_bundle.value:
raise ValidationError(
f'The request {request_id} is of type {request.type_name}. '
'This endpoint is only valid for requests of type regenerate-bundle.'
)
finalized = request.state.state_name in RequestStateMapping.get_final_states()
if not finalized:
raise ValidationError(
f'The request {request_id} is not complete yet.'
' related_bundles will be available once the request is complete.'
)
related_bundles_file_path = os.path.join(
request_related_bundles_dir, f'{request_id}_related_bundles.json'
)
if not os.path.exists(related_bundles_file_path):
expired = request.temporary_data_expiration < datetime.utcnow()
if expired:
raise Gone(f'The related_bundles for the build request {request_id} no longer exist')
raise IIBError(
'IIB is done processing the request and cannot find related_bundles. Please make '
f'sure the iib_organizaiton_customizations for organization {request.organization}'
' has related_bundles customization type set'
)
with open(related_bundles_file_path) as f:
return flask.Response(f.read(), mimetype='application/json')
def patch_request(request_id):
"""
Modify the given request.
:param int request_id: the request ID from the URL
:return: a Flask JSON response
:rtype: flask.Response
:raise Forbidden: If the user trying to patch a request is not an IIB worker
:raise NotFound: if the request is not found
:raise ValidationError: if the JSON is invalid
"""
allowed_users = flask.current_app.config['IIB_WORKER_USERNAMES']
# current_user.is_authenticated is only ever False when auth is disabled
if current_user.is_authenticated and current_user.username not in allowed_users:
raise Forbidden('This API endpoint is restricted to IIB workers')
payload = flask.request.get_json()
if not isinstance(payload, dict):
raise ValidationError('The input data must be a JSON object')
if not payload:
raise ValidationError(
'At least one key must be specified to update the request')
request = Request.query.get_or_404(request_id)
invalid_keys = payload.keys() - request.get_mutable_keys()
if invalid_keys:
raise ValidationError('The following keys are not allowed: {}'.format(
', '.join(invalid_keys)))
for key, value in payload.items():
if key == 'arches':
Architecture.validate_architecture_json(value)
elif key == 'bundle_mapping':
exc_msg = f'The "{key}" key must be an object with the values as lists of strings'
if not isinstance(value, dict):
raise ValidationError(exc_msg)
for v in value.values():
if not isinstance(v, list) or any(not isinstance(s, str)
for s in v):
raise ValidationError(exc_msg)
elif not value or not isinstance(value, str):
raise ValidationError(
f'The value for "{key}" must be a non-empty string')
if 'state' in payload and 'state_reason' not in payload:
raise ValidationError(
'The "state_reason" key is required when "state" is supplied')
elif 'state_reason' in payload and 'state' not in payload:
raise ValidationError(
'The "state" key is required when "state_reason" is supplied')
state_updated = False
if 'state' in payload and 'state_reason' in payload:
RequestStateMapping.validate_state(payload['state'])
new_state = payload['state']
new_state_reason = payload['state_reason']
# This is to protect against a Celery task getting executed twice and setting the
# state each time
if request.state.state == new_state and request.state.state_reason == new_state_reason:
flask.current_app.logger.info(
'Not adding a new state since it matches the last state')
else:
request.add_state(new_state, new_state_reason)
state_updated = True
image_keys = (
'binary_image_resolved',
'bundle_image',
'from_bundle_image_resolved',
'from_index_resolved',
'index_image',
)
for key in image_keys:
if key not in payload:
continue
key_value = payload.get(key, None)
key_object = Image.get_or_create(key_value)
# SQLAlchemy will not add the object to the database if it's already present
setattr(request, key, key_object)
for arch in payload.get('arches', []):
request.add_architecture(arch)
for operator, bundles in payload.get('bundle_mapping', {}).items():
operator_img = Operator.get_or_create(operator)
for bundle in bundles:
bundle_img = Image.get_or_create(bundle)
bundle_img.operator = operator_img
db.session.commit()
if state_updated:
messaging.send_message_for_state_change(request)
if current_user.is_authenticated:
flask.current_app.logger.info('The user %s patched request %d',
current_user.username, request.id)
else:
flask.current_app.logger.info('An anonymous user patched request %d',
request.id)
return flask.jsonify(request.to_json()), 200
def get_builds():
"""
Retrieve the paginated build requests.
:rtype: flask.Response
"""
batch_id = flask.request.args.get('batch')
state = flask.request.args.get('state')
verbose = str_to_bool(flask.request.args.get('verbose'))
max_per_page = flask.current_app.config['IIB_MAX_PER_PAGE']
request_type = flask.request.args.get('request_type')
user = flask.request.args.get('user')
index_image = flask.request.args.get('index_image')
query_params = {}
# Create an alias class to load the polymorphic classes
poly_request = with_polymorphic(Request, '*')
query = poly_request.query.options(*get_request_query_options(
verbose=verbose))
if state:
query_params['state'] = state
RequestStateMapping.validate_state(state)
state_int = RequestStateMapping.__members__[state].value
query = query.join(Request.state)
query = query.filter(RequestState.state == state_int)
if batch_id is not None:
query_params['batch'] = batch_id
batch_id = Batch.validate_batch(batch_id)
query = query.filter_by(batch_id=batch_id)
if request_type:
query_params['request_type'] = request_type
RequestTypeMapping.validate_type(request_type)
request_type = request_type.replace('-', '_')
request_type_int = RequestTypeMapping.__members__[request_type].value
query = query.filter(Request.type == request_type_int)
if user:
# join with the user table and then filter on username
# request table only has the user_id
query_params['user'] = user
query = query.join(Request.user).filter(User.username == user)
if index_image:
query_params['index_image'] = index_image
# Get the image id of the image to be searched
image_result = Image.query.filter_by(
pull_specification=index_image).first()
if image_result:
# join with the Request* tables to get the response as image_ids are stored there
query = (query.outerjoin(
RequestCreateEmptyIndex,
Request.id == RequestCreateEmptyIndex.id).outerjoin(
RequestAdd, Request.id == RequestAdd.id).outerjoin(
RequestMergeIndexImage,
Request.id == RequestMergeIndexImage.id).outerjoin(
RequestRm, Request.id == RequestRm.id))
query = query.filter(
or_(
RequestCreateEmptyIndex.index_image_id == image_result.id,
RequestAdd.index_image_id == image_result.id,
RequestMergeIndexImage.index_image_id == image_result.id,
RequestRm.index_image_id == image_result.id,
))
# if index_image is not found in image table, then raise an error
else:
raise ValidationError(f'{index_image} is not a valid index image')
pagination_query = query.order_by(
Request.id.desc()).paginate(max_per_page=max_per_page)
requests = pagination_query.items
response = {
'items': [request.to_json(verbose=verbose) for request in requests],
'meta': pagination_metadata(pagination_query, **query_params),
}
return flask.jsonify(response)
def get_related_bundles(request_id):
"""
Retrieve the related bundle images from the bundle CSV for a regenerate-bundle request.
:param int request_id: the request ID that was passed in through the URL.
:rtype: flask.Response
:raise NotFound: if the request is not found or there are no related bundles for the request
:raise Gone: if the related bundles for the build request have been removed due to expiration
:raise ValidationError: if the request is of invalid type or is not completed yet
"""
request_related_bundles_dir = flask.current_app.config[
'IIB_REQUEST_RELATED_BUNDLES_DIR']
s3_bucket_name = flask.current_app.config['IIB_AWS_S3_BUCKET_NAME']
if not s3_bucket_name and not request_related_bundles_dir:
raise NotFound()
request = Request.query.get_or_404(request_id)
if request.type != RequestTypeMapping.regenerate_bundle.value:
raise ValidationError(
f'The request {request_id} is of type {request.type_name}. '
'This endpoint is only valid for requests of type regenerate-bundle.'
)
finalized = request.state.state_name in RequestStateMapping.get_final_states(
)
if not finalized:
raise ValidationError(
f'The request {request_id} is not complete yet.'
' related_bundles will be available once the request is complete.')
# If S3 bucket is configured, fetch the related bundles file from the S3 bucket.
# Else, check if related bundles are stored on the system itself and return them.
# Otherwise, raise an IIBError.
if s3_bucket_name:
log_file = _get_artifact_file_from_s3_bucket(
'related_bundles',
f'{request_id}_related_bundles.json',
request_id,
request.temporary_data_expiration,
s3_bucket_name,
)
return flask.Response(log_file.read(), mimetype='application/json')
related_bundles_file_path = os.path.join(
request_related_bundles_dir, f'{request_id}_related_bundles.json')
if not os.path.exists(related_bundles_file_path):
expired = request.temporary_data_expiration < datetime.utcnow()
if expired:
raise Gone(
f'The related_bundles for the build request {request_id} no longer exist'
)
if request.organization:
raise IIBError(
'IIB is done processing the request and cannot find related_bundles. Please make '
f'sure the iib_organization_customizations for organization {request.organization}'
' has related_bundles customization type set')
flask.current_app.logger.warning(
' Please make sure either an S3 bucket is configured or the logs are'
' stored locally in a directory by specifying IIB_REQUEST_LOGS_DIR'
)
raise IIBError(
'IIB is done processing the request and could not find related_bundles.'
)
with open(related_bundles_file_path) as f:
return flask.Response(f.read(), mimetype='application/json')