def add_state(self, state, state_reason): """ Add a RequestState associated with the current request. :param str state: the state name :param str state_reason: the reason explaining the state transition :raises ValidationError: if the state is invalid """ try: state_int = RequestStateMapping.__members__[state].value except KeyError: raise ValidationError( 'The state "{}" is invalid. It must be one of: {}.'.format( state, ', '.join(RequestStateMapping.get_names()))) for s in ('complete', 'failed'): # A complete or failed state cannot change states, but the state reason # can be updated if self.state and self.state.state_name == s and state != s: raise ValidationError( f'A {self.state.state_name} request cannot change states') request_state = RequestState(state=state_int, state_reason=state_reason) self.states.append(request_state) # Send the changes queued up in SQLAlchemy to the database's transaction buffer. # This will generate an ID that can be used below. db.session.add(request_state) db.session.flush() self.request_state_id = request_state.id
def from_add_json(cls, kwargs): """Handle JSON requests for the Add API endpoint.""" request_kwargs = deepcopy(kwargs) bundles = request_kwargs.get('bundles', []) if (not isinstance(bundles, list) or len(bundles) == 0 or any(not item or not isinstance(item, str) for item in bundles)): raise ValidationError( f'"bundles" should be a non-empty array of strings') for param in ('cnr_token', 'organization'): if param not in request_kwargs: continue if not isinstance(request_kwargs[param], str): raise ValidationError(f'"{param}" must be a string') # Always remove cnr_token from request_kwargs since it's not stored in the database request_kwargs.pop('cnr_token', None) cls._from_json( request_kwargs, additional_required_params=['bundles'], additional_optional_params=['from_index', 'organization'], ) request_kwargs['bundles'] = [ Image.get_or_create(pull_specification=item) for item in bundles ] request_kwargs['type'] = RequestTypeMapping.__members__['add'].value request = cls(**request_kwargs) request.add_state('in_progress', 'The request was initiated') return request
def validate_batch(batch_id): """ Validate the input batch ID. If the input batch ID is a string, it will be converted to an integer and returned. :param int batch_id: the ID of the batch :raise ValidationError: if the batch ID is invalid :return: the validated batch ID :rtype: int """ rv = batch_id error_msg = 'The batch must be a positive integer' if isinstance(batch_id, str): try: rv = int(batch_id) except ValueError: raise ValidationError(error_msg) elif not isinstance(batch_id, int): raise ValidationError(error_msg) if rv < 1: raise ValidationError(error_msg) return rv
def from_json(cls, kwargs, batch=None): """ Handle JSON requests for the Add API endpoint. :param dict kwargs: the JSON payload of the request. :param Batch batch: the batch to specify with the request. """ request_kwargs = deepcopy(kwargs) bundles = request_kwargs.get('bundles', []) if not isinstance(bundles, list) or any( not item or not isinstance(item, str) for item in bundles): raise ValidationError( '"bundles" should be either an empty array or an array of non-empty strings' ) # Check if no bundles `from_index and `binary_image` are specified # if no bundles and and no from index then a empty index will be created # if no binary image and just a from_index then we are not updating anything and it would # be a no-op if not request_kwargs.get('bundles') and ( not request_kwargs.get('from_index') or not request_kwargs.get('binary_image')): raise ValidationError( '"from_index" and "binary_image" must be specified if no bundles are specified' ) for param in ('cnr_token', 'organization'): if param not in request_kwargs: continue if not isinstance(request_kwargs[param], str): raise ValidationError(f'"{param}" must be a string') if not isinstance(request_kwargs.get('force_backport', False), bool): raise ValidationError('"force_backport" must be a boolean') # Remove attributes that are not stored in the database request_kwargs.pop('cnr_token', None) request_kwargs.pop('force_backport', None) cls._from_json( request_kwargs, additional_optional_params=[ 'from_index', 'organization', 'bundles' ], batch=batch, ) request_kwargs['bundles'] = [ Image.get_or_create(pull_specification=item) for item in bundles ] request = cls(**request_kwargs) request.add_state('in_progress', 'The request was initiated') return request
def from_json(cls, kwargs, batch=None): """ Handle JSON requests for the Remove API endpoint. :param dict kwargs: the JSON payload of the request. :param Batch batch: the batch to specify with the request. """ request_kwargs = deepcopy(kwargs) operators = request_kwargs.get('operators', []) if (not isinstance(operators, list) or len(operators) == 0 or any(not item or not isinstance(item, str) for item in operators)): raise ValidationError( f'"operators" should be a non-empty array of strings') cls._from_json(request_kwargs, additional_required_params=['operators', 'from_index'], batch=batch) request_kwargs['operators'] = [ Operator.get_or_create(name=item) for item in operators ] request = cls(**request_kwargs) request.add_state('in_progress', 'The request was initiated') return request
def rm_operators(): """ Submit a request to remove operators from an index image. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') request = RequestRm.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) overwrite_from_index = payload.get('overwrite_from_index') args = _get_rm_args(payload, request, overwrite_from_index) safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: handle_rm_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=_get_user_queue(serial=overwrite_from_index), ) except kombu.exceptions.OperationalError: handle_broker_error(request) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def regenerate_bundle(): """ Submit a request to regenerate an operator bundle image. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') request = RequestRegenerateBundle.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) error_callback = failed_request_callback.s(request.id) handle_regenerate_bundle_request.apply_async( args=[ payload['from_bundle_image'], payload.get('organization'), request.id ], link_error=error_callback, queue=_get_user_queue(), ) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def add_bundles(): """ Submit a request to add operator bundles to an index image. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') request = Request.from_add_json(payload) db.session.add(request) db.session.commit() args = [ payload['bundles'], payload['binary_image'], request.id, payload.get('from_index'), payload.get('add_arches'), payload.get('cnr_token'), payload.get('organization'), _should_force_overwrite() or payload.get('overwrite_from_index'), ] safe_args = copy.copy(args) if payload.get('cnr_token'): safe_args[safe_args.index(payload['cnr_token'])] = '*****' error_callback = failed_request_callback.s(request.id) handle_add_request.apply_async(args=args, link_error=error_callback, argsrepr=repr(safe_args)) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def rm_operators(): """ Submit a request to remove operators from an index image. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') request = Request.from_remove_json(payload) db.session.add(request) db.session.commit() error_callback = failed_request_callback.s(request.id) handle_rm_request.apply_async( args=[ payload['operators'], payload['binary_image'], request.id, payload['from_index'], payload.get('add_arches'), _should_force_overwrite() or payload.get('overwrite_from_index'), ], link_error=error_callback, ) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def from_json(cls, kwargs, batch=None): """ Handle JSON requests for the Add API endpoint. :param dict kwargs: the JSON payload of the request. :param Batch batch: the batch to specify with the request. """ request_kwargs = deepcopy(kwargs) bundles = request_kwargs.get('bundles', []) if (not isinstance(bundles, list) or len(bundles) == 0 or any(not item or not isinstance(item, str) for item in bundles)): raise ValidationError( f'"bundles" should be a non-empty array of strings') for param in ('cnr_token', 'organization'): if param not in request_kwargs: continue if not isinstance(request_kwargs[param], str): raise ValidationError(f'"{param}" must be a string') if not isinstance(request_kwargs.get('force_backport', False), bool): raise ValidationError('"force_backport" must be a boolean') # Remove attributes that are not stored in the database request_kwargs.pop('cnr_token', None) request_kwargs.pop('force_backport', None) cls._from_json( request_kwargs, additional_required_params=['bundles'], additional_optional_params=['from_index', 'organization'], batch=batch, ) request_kwargs['bundles'] = [ Image.get_or_create(pull_specification=item) for item in bundles ] request = cls(**request_kwargs) request.add_state('in_progress', 'The request was initiated') return request
def regenerate_bundle_batch(): """ Submit a batch of requests to regenerate operator bundle images. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() Batch.validate_batch_request_params(payload) batch = Batch(annotations=payload.get('annotations')) db.session.add(batch) requests = [] # Iterate through all the build requests and verify that the requests are valid before # committing them and scheduling the tasks for build_request in payload['build_requests']: try: request = RequestRegenerateBundle.from_json(build_request, batch) except ValidationError as e: # Rollback the transaction if any of the build requests are invalid db.session.rollback() raise ValidationError( f'{str(e).rstrip(".")}. This occurred on the build request in ' f'index {payload["build_requests"].index(build_request)}.') db.session.add(request) requests.append(request) db.session.commit() messaging.send_messages_for_new_batch_of_requests(requests) request_jsons = [] # This list will be used for the log message below and avoids the need of having to iterate # through the list of requests another time request_id_strs = [] for build_request, request in zip(payload['build_requests'], requests): request_jsons.append(request.to_json()) request_id_strs.append(str(request.id)) error_callback = failed_request_callback.s(request.id) handle_regenerate_bundle_request.apply_async( args=[ build_request['from_bundle_image'], build_request.get('organization'), request.id, ], link_error=error_callback, queue=_get_user_queue(), ) flask.current_app.logger.debug( 'Successfully scheduled the batch %d with requests: %s', batch.id, ', '.join(request_id_strs), ) return flask.jsonify(request_jsons), 201
def validate_batch_request_params(payload): """ Validate batch specific parameters from the input JSON payload. The requests in the "build_requests" key's value are not validated. Those should be validated separately. :raises ValidationError: if the payload is invalid """ if (not isinstance(payload, dict) or not isinstance(payload.get('build_requests'), list) or not payload['build_requests']): raise ValidationError( 'The input data must be a JSON object and the "build_requests" value must be a ' 'non-empty array') if not isinstance(payload.get('annotations', {}), dict): raise ValidationError( 'The value of "annotations" must be a JSON object')
def get_related_bundles(request_id): """ Retrieve the related bundle images from the bundle CSV for a regenerate-bundle request. :param int request_id: the request ID that was passed in through the URL. :rtype: flask.Response :raise NotFound: if the request is not found or there are no related bundles for the request :raise Gone: if the related bundles for the build request have been removed due to expiration """ request_related_bundles_dir = flask.current_app.config['IIB_REQUEST_RELATED_BUNDLES_DIR'] if not request_related_bundles_dir: raise NotFound() request = Request.query.get_or_404(request_id) if request.type != RequestTypeMapping.regenerate_bundle.value: raise ValidationError( f'The request {request_id} is of type {request.type_name}. ' 'This endpoint is only valid for requests of type regenerate-bundle.' ) finalized = request.state.state_name in RequestStateMapping.get_final_states() if not finalized: raise ValidationError( f'The request {request_id} is not complete yet.' ' related_bundles will be available once the request is complete.' ) related_bundles_file_path = os.path.join( request_related_bundles_dir, f'{request_id}_related_bundles.json' ) if not os.path.exists(related_bundles_file_path): expired = request.temporary_data_expiration < datetime.utcnow() if expired: raise Gone(f'The related_bundles for the build request {request_id} no longer exist') raise IIBError( 'IIB is done processing the request and cannot find related_bundles. Please make ' f'sure the iib_organizaiton_customizations for organization {request.organization}' ' has related_bundles customization type set' ) with open(related_bundles_file_path) as f: return flask.Response(f.read(), mimetype='application/json')
def from_json(cls, kwargs, batch=None): """ Handle JSON requests for the Regenerate Bundle API endpoint. :param dict kwargs: the JSON payload of the request. :param Batch batch: the batch to specify with the request. If one is not specified, one will be created automatically. """ batch = batch or Batch() request_kwargs = deepcopy(kwargs) validate_request_params( request_kwargs, required_params={'from_bundle_image'}, optional_params={'organization'}, ) # Validate organization is correctly provided organization = request_kwargs.get('organization') if organization and not isinstance(organization, str): raise ValidationError('"organization" must be a string') # Validate from_bundle_image is correctly provided from_bundle_image = request_kwargs.pop('from_bundle_image') if not isinstance(from_bundle_image, str): raise ValidationError('"from_bundle_image" must be a string') request_kwargs['from_bundle_image'] = Image.get_or_create( pull_specification=from_bundle_image) # current_user.is_authenticated is only ever False when auth is disabled if current_user.is_authenticated: request_kwargs['user'] = current_user # Add the request to a new batch db.session.add(batch) request_kwargs['batch'] = batch request = cls(**request_kwargs) request.add_state('in_progress', 'The request was initiated') return request
def validate_architecture_json(arches): """ Validate the JSON representation of architectures. :param list arches: the JSON representation of architectures for a build request :raise ValidationError: if the JSON does not match the required schema """ if not isinstance(arches, list) or any( not arch or not isinstance(arch, str) for arch in arches): raise ValidationError( 'Architectures should be specified as a non-empty array of strings' )
def add_bundles(): """ Submit a request to add operator bundles to an index image. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') request = RequestAdd.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) overwrite_from_index = _should_force_overwrite() or payload.get( 'overwrite_from_index') celery_queue = _get_user_queue(serial=overwrite_from_index) args = [ payload['bundles'], payload['binary_image'], request.id, payload.get('from_index'), payload.get('add_arches'), payload.get('cnr_token'), payload.get('organization'), payload.get('force_backport'), overwrite_from_index, payload.get('overwrite_from_index_token'), flask.current_app.config['IIB_GREENWAVE_CONFIG'].get(celery_queue), ] safe_args = copy.copy(args) if payload.get('cnr_token'): safe_args[safe_args.index(payload['cnr_token'])] = '*****' if payload.get('overwrite_from_index_token'): safe_args[safe_args.index( payload['overwrite_from_index_token'])] = '*****' error_callback = failed_request_callback.s(request.id) try: handle_add_request.apply_async(args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue) except kombu.exceptions.OperationalError: handle_broker_error(request) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def get_build_logs(request_id): """ Retrieve the logs for the build request. :param int request_id: the request ID that was passed in through the URL. :rtype: flask.Response :raise NotFound: if the request is not found or there are no logs for the request :raise Gone: if the logs for the build request have been removed due to expiration :raise ValidationError: if the request has not completed yet """ request_log_dir = flask.current_app.config['IIB_REQUEST_LOGS_DIR'] s3_bucket_name = flask.current_app.config['IIB_AWS_S3_BUCKET_NAME'] if not s3_bucket_name and not request_log_dir: raise NotFound() request = Request.query.get_or_404(request_id) finalized = request.state.state_name in RequestStateMapping.get_final_states( ) if not finalized: raise ValidationError( f'The request {request_id} is not complete yet.' ' logs will be available once the request is complete.') # If S3 bucket is configured, fetch the log file from the S3 bucket. # Else, check if logs are stored on the system itself and return them. # Otherwise, raise an IIBError. if s3_bucket_name: log_file = _get_artifact_file_from_s3_bucket( 'request_logs', f'{request_id}.log', request_id, request.temporary_data_expiration, s3_bucket_name, ) return flask.Response(log_file.read(), mimetype='text/plain') local_log_file_path = os.path.join(request_log_dir, f'{request_id}.log') if not os.path.exists(local_log_file_path): expired = request.temporary_data_expiration < datetime.utcnow() if expired: raise Gone( f'The logs for the build request {request_id} no longer exist') flask.current_app.logger.warning( ' Please make sure either an S3 bucket is configured or the logs are' ' stored locally in a directory by specifying IIB_REQUEST_LOGS_DIR' ) raise IIBError( 'IIB is done processing the request and could not find logs.') with open(local_log_file_path) as f: return flask.Response(f.read(), mimetype='text/plain')
def validate_state(cls, state): """ Verify that the input state is valid. :param str state: the state to validate :raises iib.exceptions.ValidationError: if the state is invalid """ state_names = cls.get_names() if state not in state_names: states = ', '.join(state_names) raise ValidationError( f'{state} is not a valid build request state. Valid states are: {states}' )
def validate_type(self, key, type_num): """ Verify the type number used is valid. :param str key: the name of the database column :param int type_num: the request type number to be verified :return: the request type number :rtype: int :raises ValidationError: if the request type is invalid """ if not any(type_num == num.value for num in RequestTypeMapping): raise ValidationError( f'{type_num} is not a valid request type number') return type_num
def validate_request_params(request_params, required_params, optional_params): """ Validate parameters for a build request. All required parameters must be set in the request_params and unknown parameters are not allowed. :param dict request_params: the request parameters provided by the user :param set required_params: the set of required parameters :param set optional_params: the set of optional parameters :raises iib.exceptions.ValidationError: if validation of parameters fails """ missing_params = required_params - request_params.keys() if missing_params: raise ValidationError('Missing required parameter(s): {}'.format( ', '.join(missing_params))) # Don't allow the user to set arbitrary columns or relationships invalid_params = request_params.keys() - required_params - optional_params if invalid_params: raise ValidationError( 'The following parameters are invalid: {}'.format( ', '.join(invalid_params))) # Verify that all the required parameters are set and not empty for param in required_params: if not request_params.get(param): raise ValidationError(f'"{param}" must be set') # If any optional parameters are set but are empty, just remove them since they are # treated as null values for param in optional_params: if (param in request_params and not isinstance(request_params.get(param), bool) and not request_params[param]): del request_params[param]
def merge_index_image(): """ Submit a request to merge two index images. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') request = RequestMergeIndexImage.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) overwrite_target_index = payload.get('overwrite_target_index', False) celery_queue = _get_user_queue(serial=overwrite_target_index) args = [ payload['source_from_index'], payload.get('deprecation_list', []), request.id, payload.get('binary_image'), payload.get('target_index'), overwrite_target_index, payload.get('overwrite_target_index_token'), request.distribution_scope, flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], payload.get('build_tags', []), ] safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: handle_merge_request.apply_async(args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue) except kombu.exceptions.OperationalError: handle_broker_error(request) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def rm_operators(): """ Submit a request to remove operators from an index image. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') request = RequestRm.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) args = [ payload['operators'], payload['binary_image'], request.id, payload['from_index'], payload.get('add_arches'), _should_force_overwrite() or payload.get('overwrite_from_index'), payload.get('overwrite_from_index_token'), ] safe_args = copy.copy(args) if payload.get('overwrite_from_index_token'): safe_args[safe_args.index( payload['overwrite_from_index_token'])] = '*****' error_callback = failed_request_callback.s(request.id) handle_rm_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=_get_user_queue(), ) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def create_empty_index(): """ Submit a request to create an index image without bundles. Note: Any duplicate bundle will be removed from payload. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') request = RequestCreateEmptyIndex.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) args = [ payload['from_index'], request.id, payload.get('output_fbc'), payload.get('binary_image'), payload.get('labels'), flask.current_app.config['IIB_BINARY_IMAGE_CONFIG'], ] safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: handle_create_empty_index_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=_get_user_queue()) except kombu.exceptions.OperationalError: handle_broker_error(request) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def add_bundles(): """ Submit a request to add operator bundles to an index image. Note: Any duplicate bundle will be removed from payload. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') # Only run `_get_unique_bundles` if it is a list. If it's not, `from_json` # will raise an error to the user. if payload.get('bundles') and isinstance(payload['bundles'], list): payload['bundles'] = _get_unique_bundles(payload['bundles']) request = RequestAdd.from_json(payload) db.session.add(request) db.session.commit() messaging.send_message_for_state_change(request, new_batch_msg=True) overwrite_from_index = payload.get('overwrite_from_index') celery_queue = _get_user_queue(serial=overwrite_from_index) args = _get_add_args(payload, request, overwrite_from_index, celery_queue) safe_args = _get_safe_args(args, payload) error_callback = failed_request_callback.s(request.id) try: handle_add_request.apply_async(args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue) except kombu.exceptions.OperationalError: handle_broker_error(request) flask.current_app.logger.debug('Successfully scheduled request %d', request.id) return flask.jsonify(request.to_json()), 201
def get_or_create(cls, pull_specification): """ Get the image from the database and create it if it doesn't exist. :param str pull_specification: pull_specification of the image :return: an Image object based on the input pull_specification; the Image object will be added to the database session, but not committed, if it was created :rtype: Image :raise ValidationError: if pull_specification for the image is invalid """ if '@' not in pull_specification and ':' not in pull_specification: raise ValidationError( f'Image {pull_specification} should have a tag or a digest specified.' ) image = cls.query.filter_by( pull_specification=pull_specification).first() if not image: image = Image(pull_specification=pull_specification) db.session.add(image) return image
def from_remove_json(cls, kwargs): """Handle JSON requests for the Remove API endpoint.""" request_kwargs = deepcopy(kwargs) operators = request_kwargs.get('operators', []) if (not isinstance(operators, list) or len(operators) == 0 or any(not item or not isinstance(item, str) for item in operators)): raise ValidationError( f'"operators" should be a non-empty array of strings') cls._from_json(request_kwargs, additional_required_params=['operators', 'from_index']) request_kwargs['operators'] = [ Operator.get_or_create(name=item) for item in operators ] request_kwargs['type'] = RequestTypeMapping.__members__['rm'].value request = cls(**request_kwargs) request.add_state('in_progress', 'The request was initiated') return request
def patch_request(request_id): """ Modify the given request. :param int request_id: the request ID from the URL :return: a Flask JSON response :rtype: flask.Response :raise Forbidden: If the user trying to patch a request is not an IIB worker :raise NotFound: if the request is not found :raise ValidationError: if the JSON is invalid """ allowed_users = flask.current_app.config['IIB_WORKER_USERNAMES'] # current_user.is_authenticated is only ever False when auth is disabled if current_user.is_authenticated and current_user.username not in allowed_users: raise Forbidden('This API endpoint is restricted to IIB workers') payload = flask.request.get_json() if not isinstance(payload, dict): raise ValidationError('The input data must be a JSON object') if not payload: raise ValidationError( 'At least one key must be specified to update the request') request = Request.query.get_or_404(request_id) invalid_keys = payload.keys() - request.get_mutable_keys() if invalid_keys: raise ValidationError('The following keys are not allowed: {}'.format( ', '.join(invalid_keys))) for key, value in payload.items(): if key == 'arches': Architecture.validate_architecture_json(value) elif key == 'bundle_mapping': exc_msg = f'The "{key}" key must be an object with the values as lists of strings' if not isinstance(value, dict): raise ValidationError(exc_msg) for v in value.values(): if not isinstance(v, list) or any(not isinstance(s, str) for s in v): raise ValidationError(exc_msg) elif not value or not isinstance(value, str): raise ValidationError( f'The value for "{key}" must be a non-empty string') if 'state' in payload and 'state_reason' not in payload: raise ValidationError( 'The "state_reason" key is required when "state" is supplied') elif 'state_reason' in payload and 'state' not in payload: raise ValidationError( 'The "state" key is required when "state_reason" is supplied') state_updated = False if 'state' in payload and 'state_reason' in payload: RequestStateMapping.validate_state(payload['state']) new_state = payload['state'] new_state_reason = payload['state_reason'] # This is to protect against a Celery task getting executed twice and setting the # state each time if request.state.state == new_state and request.state.state_reason == new_state_reason: flask.current_app.logger.info( 'Not adding a new state since it matches the last state') else: request.add_state(new_state, new_state_reason) state_updated = True image_keys = ( 'binary_image_resolved', 'bundle_image', 'from_bundle_image_resolved', 'from_index_resolved', 'index_image', ) for key in image_keys: if key not in payload: continue key_value = payload.get(key, None) key_object = Image.get_or_create(key_value) # SQLAlchemy will not add the object to the database if it's already present setattr(request, key, key_object) for arch in payload.get('arches', []): request.add_architecture(arch) for operator, bundles in payload.get('bundle_mapping', {}).items(): operator_img = Operator.get_or_create(operator) for bundle in bundles: bundle_img = Image.get_or_create(bundle) bundle_img.operator = operator_img db.session.commit() if state_updated: messaging.send_message_for_state_change(request) if current_user.is_authenticated: flask.current_app.logger.info('The user %s patched request %d', current_user.username, request.id) else: flask.current_app.logger.info('An anonymous user patched request %d', request.id) return flask.jsonify(request.to_json()), 200
def add_rm_batch(): """ Submit a batch of requests to add or remove operators from an index image. Note: Any duplicate bundle will be removed from payload when adding operators. :rtype: flask.Response :raise ValidationError: if required parameters are not supplied """ payload = flask.request.get_json() Batch.validate_batch_request_params(payload) batch = Batch(annotations=payload.get('annotations')) db.session.add(batch) requests = [] # Iterate through all the build requests and verify that the requests are valid before # committing them and scheduling the tasks for build_request in payload['build_requests']: try: if build_request.get('operators'): # Check for the validity of a RM request request = RequestRm.from_json(build_request, batch) elif build_request.get('bundles'): build_request_uniq = copy.deepcopy(build_request) build_request_uniq['bundles'] = _get_unique_bundles(build_request_uniq['bundles']) # Check for the validity of an Add request request = RequestAdd.from_json(build_request_uniq, batch) else: raise ValidationError('Build request is not a valid Add/Rm request.') except ValidationError as e: raise ValidationError( f'{str(e).rstrip(".")}. This occurred on the build request in ' f'index {payload["build_requests"].index(build_request)}.' ) db.session.add(request) requests.append(request) db.session.commit() messaging.send_messages_for_new_batch_of_requests(requests) request_jsons = [] # This list will be used for the log message below and avoids the need of having to iterate # through the list of requests another time processed_request_ids = [] for build_request, request in zip(payload['build_requests'], requests): request_jsons.append(request.to_json()) overwrite_from_index = build_request.get('overwrite_from_index') celery_queue = _get_user_queue(serial=overwrite_from_index) if isinstance(request, RequestAdd): args = _get_add_args(build_request, request, overwrite_from_index, celery_queue) elif isinstance(request, RequestRm): args = _get_rm_args(build_request, request, overwrite_from_index) safe_args = _get_safe_args(args, build_request) error_callback = failed_request_callback.s(request.id) try: if isinstance(request, RequestAdd): handle_add_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue, ) else: handle_rm_request.apply_async( args=args, link_error=error_callback, argsrepr=repr(safe_args), queue=celery_queue, ) except kombu.exceptions.OperationalError: unprocessed_requests = [r for r in requests if str(r.id) not in processed_request_ids] handle_broker_batch_error(unprocessed_requests) processed_request_ids.append(str(request.id)) flask.current_app.logger.debug( 'Successfully scheduled the batch %d with requests: %s', batch.id, ', '.join(processed_request_ids), ) return flask.jsonify(request_jsons), 201
def get_builds(): """ Retrieve the paginated build requests. :rtype: flask.Response """ batch_id = flask.request.args.get('batch') state = flask.request.args.get('state') verbose = str_to_bool(flask.request.args.get('verbose')) max_per_page = flask.current_app.config['IIB_MAX_PER_PAGE'] request_type = flask.request.args.get('request_type') user = flask.request.args.get('user') index_image = flask.request.args.get('index_image') query_params = {} # Create an alias class to load the polymorphic classes poly_request = with_polymorphic(Request, '*') query = poly_request.query.options(*get_request_query_options( verbose=verbose)) if state: query_params['state'] = state RequestStateMapping.validate_state(state) state_int = RequestStateMapping.__members__[state].value query = query.join(Request.state) query = query.filter(RequestState.state == state_int) if batch_id is not None: query_params['batch'] = batch_id batch_id = Batch.validate_batch(batch_id) query = query.filter_by(batch_id=batch_id) if request_type: query_params['request_type'] = request_type RequestTypeMapping.validate_type(request_type) request_type = request_type.replace('-', '_') request_type_int = RequestTypeMapping.__members__[request_type].value query = query.filter(Request.type == request_type_int) if user: # join with the user table and then filter on username # request table only has the user_id query_params['user'] = user query = query.join(Request.user).filter(User.username == user) if index_image: query_params['index_image'] = index_image # Get the image id of the image to be searched image_result = Image.query.filter_by( pull_specification=index_image).first() if image_result: # join with the Request* tables to get the response as image_ids are stored there query = (query.outerjoin( RequestCreateEmptyIndex, Request.id == RequestCreateEmptyIndex.id).outerjoin( RequestAdd, Request.id == RequestAdd.id).outerjoin( RequestMergeIndexImage, Request.id == RequestMergeIndexImage.id).outerjoin( RequestRm, Request.id == RequestRm.id)) query = query.filter( or_( RequestCreateEmptyIndex.index_image_id == image_result.id, RequestAdd.index_image_id == image_result.id, RequestMergeIndexImage.index_image_id == image_result.id, RequestRm.index_image_id == image_result.id, )) # if index_image is not found in image table, then raise an error else: raise ValidationError(f'{index_image} is not a valid index image') pagination_query = query.order_by( Request.id.desc()).paginate(max_per_page=max_per_page) requests = pagination_query.items response = { 'items': [request.to_json(verbose=verbose) for request in requests], 'meta': pagination_metadata(pagination_query, **query_params), } return flask.jsonify(response)
def _from_json(request_kwargs, additional_required_params=None, additional_optional_params=None, batch=None): """ Validate and process request agnostic parameters. As part of the processing, the input ``request_kwargs`` parameter is updated to reference database objects where appropriate. :param dict request_kwargs: copy of args provided in API request :param Batch batch: the batch to specify with the request. If one is not specified, one will be created automatically. """ # Validate all required parameters are present required_params = {'binary_image'} | set(additional_required_params or []) optional_params = { 'add_arches', 'overwrite_from_index', 'overwrite_from_index_token', } | set(additional_optional_params or []) validate_request_params( request_kwargs, required_params=required_params, optional_params=optional_params, ) # Check if both `from_index` and `add_arches` are not specified if not request_kwargs.get('from_index') and not request_kwargs.get( 'add_arches'): raise ValidationError( 'One of "from_index" or "add_arches" must be specified') # Verify that `overwrite_from_index` is the correct type overwrite = request_kwargs.pop('overwrite_from_index', False) if not isinstance(overwrite, bool): raise ValidationError( 'The "overwrite_from_index" parameter must be a boolean') # Verify that `overwrite_from_index_token` is the correct type overwrite_token = request_kwargs.pop('overwrite_from_index_token', None) if overwrite_token: if not isinstance(overwrite_token, str): raise ValidationError( 'The "overwrite_from_index_token" parameter must be a string' ) if overwrite_token and not overwrite: raise ValidationError( 'The "overwrite_from_index" parameter is required when' ' the "overwrite_from_index_token" parameter is used') # Verify the user is authorized to use overwrite_from_index # current_user.is_authenticated is only ever False when auth is disabled if current_user.is_authenticated: privileged_users = current_app.config['IIB_PRIVILEGED_USERNAMES'] if overwrite and not overwrite_token and current_user.username not in privileged_users: raise Forbidden( 'You must be a privileged user to set "overwrite_from_index" without' ' setting "overwrite_from_index_token"') # Validate add_arches are correctly provided add_arches = request_kwargs.pop('add_arches', []) Architecture.validate_architecture_json(add_arches) # Validate binary_image is correctly provided binary_image = request_kwargs.pop('binary_image') if not isinstance(binary_image, str): raise ValidationError('"binary_image" must be a string') request_kwargs['binary_image'] = Image.get_or_create( pull_specification=binary_image) if 'from_index' in request_kwargs: if not isinstance(request_kwargs['from_index'], str): raise ValidationError('"from_index" must be a string') request_kwargs['from_index'] = Image.get_or_create( pull_specification=request_kwargs['from_index']) # current_user.is_authenticated is only ever False when auth is disabled if current_user.is_authenticated: request_kwargs['user'] = current_user # Add the request to a new batch batch = batch or Batch() db.session.add(batch) request_kwargs['batch'] = batch