Exemplo n.º 1
0
 def get_workflow_detail(self, helper, workflow_id):
     """
     Retrieve a workflow by id,
     :param helper: The WorkflowHelper constructed for this invocation
     :param workflow_id: a string in {dag_id}__{execution_date} format
                         identifying a workflow
     :returns: a workflow detail dictionary including steps
     """
     if not WorkflowHelper.validate_workflow_id(workflow_id):
         raise ApiError(
             title='Invalid Workflow ID specified',
             description=(
                 'Workflow id must use {dag id}__{execution date} format',
             ),
             status=falcon.HTTP_400,
             retry=False,
         )
     workflow = helper.get_workflow(workflow_id=workflow_id)
     if workflow is None:
         raise ApiError(
             title='Workflow not found',
             description=(
                 'A Workflow with id {} was not found'.format(workflow_id),
             ),
             status=falcon.HTTP_404,
             retry=False,
         )
     return workflow
Exemplo n.º 2
0
 def add_collection(self, collection_id, document_string):
     """
     Triggers a call to Deckhand to add a collection(bucket)
     Documents are assumed to be a string input, not a
     collection.
     Returns the id of the buffer version.
     """
     try:
         self.deckhand.put_bucket(collection_id, document_string)
     except DocumentExistsElsewhereError as deee:
         LOG.info('Deckhand has rejected this input because an included '
                  'document exists in another bucket already')
         raise ApiError(
             title='Documents may not exist in more than one collection',
             description=deee.response_message,
             status=falcon.HTTP_409)
     except DeckhandRejectedInputError as drie:
         LOG.info('Deckhand has rejected this input because: %s',
                  drie.response_message)
         raise ApiError(
             title="Document(s) invalid syntax or otherwise unsuitable",
             description=drie.response_message)
     # reset the revision dict so it regenerates.
     self.revision_dict = None
     return self._get_buffer_rev_id()
Exemplo n.º 3
0
    def get_action_step(self, action_id, step_id):
        """
        Interacts with airflow and the shipyard database to return the
        requested step invoked through shipyard.
        """
        action = self.get_action_db(action_id=action_id)

        if action is None:
            raise ApiError(
                title='Action not found',
                description='Unknown action {}'.format(action_id),
                status=falcon.HTTP_404)

        # resolve the ids for lookup of steps
        dag_id = action['dag_id']
        dag_execution_date = action['dag_execution_date']

        # get the action steps from shipyard db
        steps = self.get_tasks_db(dag_id, dag_execution_date)

        for idx, step in enumerate(steps):
            if step_id == step['task_id']:
                # TODO (Bryan Strassner) more info about the step?
                #                        like logs? Need requirements defined
                step['index'] = idx + 1
                return step

        # if we didn't find it, 404
        raise ApiError(
            title='Step not found',
            description='Unknown step {}'.format(step_id),
            status=falcon.HTTP_404)
    def retrieve_logs(log_endpoint):
        """
        Retrieve Logs
        """

        LOG.debug("Retrieving Airflow logs...")
        try:
            response = requests.get(
                log_endpoint,
                timeout=(CONF.requests_config.airflow_log_connect_timeout,
                         CONF.requests_config.airflow_log_read_timeout))
        except requests.exceptions.RequestException as e:
            LOG.exception(e)
            raise ApiError(
                title='Log retrieval error',
                description='Exception happened during Airflow API request',
                status=falcon.HTTP_500)
        if response.status_code >= 400:
            LOG.info(
                'Airflow endpoint returned error status code %s, '
                'content %s. Response code will be bubbled up',
                response.status_code, response.text)
            raise ApiError(
                title='Log retrieval error',
                description='Airflow endpoint returned error status code',
                status=getattr(falcon, 'HTTP_%d' % response.status_code,
                               falcon.HTTP_500))
        return response.text
Exemplo n.º 5
0
    def get_note_with_access_check(self, context, note_id):
        """Retrieve the note and checks user access to the note

        :param context: the request context
        :param note_id: the id of the note to retrieve.
        :returns: the note
        """
        try:
            note = notes_helper.get_note(note_id)
            note_type = notes_helper.get_note_assoc_id_type(note)
            if note_type not in NOTE_TYPE_RBAC:
                raise ApiError(
                    title="Unable to check permission for note type",
                    description=(
                        "Shipyard is not correctly identifying note type "
                        "for note {}".format(note_id)),
                    status=falcon.HTTP_500,
                    retry=False)
            policy.check_auth(context, NOTE_TYPE_RBAC[note_type])
            return note
        except NoteNotFoundError:
            raise ApiError(
                title="No note found",
                description=("Note {} is not found".format(note_id)),
                status=falcon.HTTP_404)
Exemplo n.º 6
0
    def handle_control(self, action_id, control_verb, context):
        """
        Interacts with airflow to trigger a dag control
        :returns: nothing
        """
        action = self.get_action_db(action_id=action_id)

        if action is None:
            raise ApiError(
                title='Action not found',
                description='Unknown action {}'.format(action_id),
                status=falcon.HTTP_404)

        if control_verb in self.controls:
            self.controls.get(control_verb)(
                dag_id=action['dag_id'],
                execution_date=action['dag_execution_date'])
            self.audit_control_command_db({
                'id': ulid.ulid(),
                'action_id': action_id,
                'command': control_verb,
                'user': context.user
            })
        else:
            raise ApiError(
                title='Control not supported',
                description='Unknown control {}'.format(control_verb),
                status=falcon.HTTP_404)
Exemplo n.º 7
0
    def invoke_airflow_dag(self, dag_id, action, context):
        """
        Call airflow, and invoke a dag
        :param dag_id: the name of the dag to invoke
        :param action: the action structure to invoke the dag with
        """
        # TODO(bryan-strassner) refactor the mechanics of this method to an
        #     airflow api client module

        # Retrieve URL
        web_server_url = CONF.base.web_server
        c_timeout = CONF.base.airflow_api_connect_timeout
        r_timeout = CONF.base.airflow_api_read_timeout

        if 'Error' in web_server_url:
            raise ApiError(
                title='Unable to invoke workflow',
                description=('Airflow URL not found by Shipyard. '
                             'Shipyard configuration is missing web_server '
                             'value'),
                status=falcon.HTTP_503,
                retry=True,
            )
        else:
            conf_value = {'action': action}
            # "conf" - JSON string that gets pickled into the DagRun's
            # conf attribute
            req_url = ('{}admin/rest_api/api?api=trigger_dag&dag_id={}'
                       '&conf={}'.format(web_server_url, dag_id,
                                         self.to_json(conf_value)))

            try:
                resp = requests.get(req_url, timeout=(c_timeout, r_timeout))
                LOG.info('Response code from Airflow trigger_dag: %s',
                         resp.status_code)
                # any 4xx/5xx will be HTTPError, which are RequestException
                resp.raise_for_status()
                response = resp.json()
                LOG.info('Response from Airflow trigger_dag: %s', response)
            except RequestException as rex:
                LOG.error("Request to airflow failed: %s", rex.args)
                raise ApiError(
                    title='Unable to complete request to Airflow',
                    description=(
                        'Airflow could not be contacted properly by Shipyard.'
                    ),
                    status=falcon.HTTP_503,
                    error_list=[{
                        'message': str(type(rex))
                    }],
                    retry=True,
                )

            dag_time = self._exhume_date(dag_id, response['output']['stdout'])
            dag_execution_date = dag_time.strftime('%Y-%m-%dT%H:%M:%S')
            return dag_execution_date
Exemplo n.º 8
0
 def secure_handler(slf, req, resp, *args, **kwargs):
     ctx = req.context
     policy_eng = ctx.policy_engine
     LOG.info("Policy Engine: %s", policy_eng.__class__.__name__)
     # perform auth
     LOG.info("Enforcing policy %s on request %s",
              self.action, ctx.request_id)
     # policy engine must be configured
     if policy_eng is None:
         LOG.error(
             "Error-Policy engine required-action: %s", self.action)
         raise AppError(
             title="Auth is not being handled by any policy engine",
             status=falcon.HTTP_500,
             retry=False
         )
     authorized = False
     try:
         if policy_eng.authorize(self.action, ctx):
             # authorized
             LOG.info("Request is authorized")
             authorized = True
     except:
         # couldn't service the auth request
         LOG.error(
             "Error - Expectation Failed - action: %s", self.action)
         raise ApiError(
             title="Expectation Failed",
             status=falcon.HTTP_417,
             retry=False
         )
     if authorized:
         return f(slf, req, resp, *args, **kwargs)
     else:
         LOG.error("Auth check failed. Authenticated:%s",
                   ctx.authenticated)
         # raise the appropriate response exeception
         if ctx.authenticated:
             LOG.error("Error: Forbidden access - action: %s",
                       self.action)
             raise ApiError(
                 title="Forbidden",
                 status=falcon.HTTP_403,
                 description="Credentials do not permit access",
                 retry=False
             )
         else:
             LOG.error("Error - Unauthenticated access")
             raise ApiError(
                 title="Unauthenticated",
                 status=falcon.HTTP_401,
                 description="Credentials are not established",
                 retry=False
             )
Exemplo n.º 9
0
def check_auth(ctx, rule):
    """Checks the authorization to the requested rule

    :param ctx: the request context for the action being performed
    :param rule: the name of the policy rule to validate the user in the
        context against

    Returns if authorized, otherwise raises an ApiError.
    """
    try:
        policy_eng = ctx.policy_engine
        LOG.info("Policy Engine: %s", policy_eng.__class__.__name__)
        # perform auth
        LOG.info("Enforcing policy %s on request %s", rule, ctx.request_id)
        # policy engine must be configured
        if policy_eng is None:
            LOG.error(
                "Error-Policy engine required-action: %s", rule)
            raise AppError(
                title="Auth is not being handled by any policy engine",
                status=falcon.HTTP_500,
                retry=False
            )
        if policy_eng.authorize(rule, ctx):
            # authorized - log and return
            LOG.info("Request to %s is authorized", rule)
            return
    except Exception as ex:
        # couldn't service the auth request
        LOG.exception("Error - Expectation Failed - action: %s", rule)
        raise ApiError(
            title="Expectation Failed",
            status=falcon.HTTP_417,
            retry=False
        )
    # raise the appropriate response exeception
    if ctx.authenticated:
        # authenticated but not authorized
        LOG.error("Error: Forbidden access - action: %s", rule)
        raise ApiError(
            title="Forbidden",
            status=falcon.HTTP_403,
            description="Credentials do not permit access",
            retry=False
        )
    else:
        LOG.error("Error - Unauthenticated access")
        raise ApiError(
            title="Unauthenticated",
            status=falcon.HTTP_401,
            description="Credentials are not established",
            retry=False
        )
Exemplo n.º 10
0
    def post_collection(self,
                        helper,
                        collection_id,
                        document_data,
                        buffer_mode_param=None,
                        empty_collection=False):
        """
        Ingest the collection after checking preconditions
        """
        buffer_mode = ConfigdocsHelper.get_buffer_mode(buffer_mode_param)

        if helper.is_buffer_valid_for_bucket(collection_id, buffer_mode):
            buffer_revision = helper.add_collection(collection_id,
                                                    document_data)
            if not (empty_collection or helper.is_collection_in_buffer(
                    collection_id)):
                # raise an error if adding the collection resulted in no new
                # revision (meaning it was unchanged) and we're not explicitly
                # clearing the collection
                raise ApiError(
                    title=('Collection {} not added to Shipyard '
                           'buffer'.format(collection_id)),
                    description='Collection created no new revision',
                    status=falcon.HTTP_400,
                    error_list=[{
                        'message':
                        ('The collection {} added no new revision, and has '
                         'been rejected as invalid input. This likely '
                         'means that the collection already exists and '
                         'was reloaded with the same contents'.format(
                             collection_id))
                    }],
                    retry=False,
                )
            else:
                return helper.get_deckhand_validation_status(buffer_revision)
        else:
            raise ApiError(
                title='Invalid collection specified for buffer',
                description='Buffermode : {}'.format(buffer_mode.value),
                status=falcon.HTTP_409,
                error_list=[{
                    'message': ('Buffer is either not empty or the '
                                'collection already exists in buffer. '
                                'Setting a different buffermode may '
                                'provide the desired functionality')
                }],
                retry=False,
            )
Exemplo n.º 11
0
    def post_collection(self,
                        helper,
                        collection_id,
                        document_data,
                        buffer_mode_param=None):
        """
        Ingest the collection after checking preconditions
        """
        if buffer_mode_param is None:
            buffer_mode = BufferMode.REJECTONCONTENTS
        else:
            buffer_mode = ConfigdocsHelper.get_buffer_mode(buffer_mode_param)

        if helper.is_buffer_valid_for_bucket(collection_id, buffer_mode):
            buffer_revision = helper.add_collection(collection_id,
                                                    document_data)
            return helper.get_deckhand_validation_status(buffer_revision)
        else:
            raise ApiError(
                title='Invalid collection specified for buffer',
                description='Buffermode : {}'.format(buffer_mode.value),
                status=falcon.HTTP_409,
                error_list=[{
                    'message': ('Buffer is either not empty or the '
                                'collection already exists in buffer. '
                                'Setting a different buffermode may '
                                'provide the desired functionality')
                }],
                retry=False,
            )
    def validate(self):
        parameters = self.action.get('parameters')
        valid = parameters is not None
        if valid:
            # target_nodes parameter should exist
            nodes = parameters.get('target_nodes')
            valid = nodes is not None
        if valid:
            # should be able to listify the nodes
            try:
                node_list = list(nodes)
                valid = len(node_list) > 0
            except TypeError:
                valid = False
        if valid:
            # each entry should be a string
            for s in node_list:
                if not isinstance(s, str):
                    valid = False
                    break
        if valid:
            # all valid
            return

        # something was invalid
        raise ApiError(
            title='Invalid target_nodes parameter',
            description=('The target_nodes parameter for this action '
                         'should be a list with one or more string values '
                         'representing node names'),
            status=falcon.HTTP_400,
            retry=False)
Exemplo n.º 13
0
    def get_action(self, action_id):
        """
        Interacts with airflow and the shipyard database to return the
        requested action invoked through shipyard.
        """
        # get the action from shipyard db
        action = self.get_action_db(action_id=action_id)
        if action is None:
            raise ApiError(title='Action not found',
                           description='Unknown Action: {}'.format(action_id),
                           status=falcon.HTTP_404)

        # lookup the dag and tasks based on the associated dag_id,
        # execution_date
        dag_id = action['dag_id']
        dag_execution_date = action['dag_execution_date']

        dag = self.get_dag_run_by_id(dag_id, dag_execution_date)
        steps = self.get_tasks_db(dag_id, dag_execution_date)
        if dag is not None:
            # put the values together into an "action" object
            action['dag_status'] = dag['state']
            action['action_lifecycle'] = determine_lifecycle(dag['state'])
            action['steps'] = format_action_steps(action_id, steps)
        action['validations'] = self.get_validations_db(action_id)
        action['command_audit'] = self.get_action_command_audit_db(action_id)
        return action
Exemplo n.º 14
0
def test_create_action_validator_error():
    action_resource = ActionsResource()
    action_resource.get_all_actions_db = actions_db
    action_resource.get_all_dag_runs_db = dag_runs_db
    action_resource.get_all_tasks_db = tasks_db
    action_resource.invoke_airflow_dag = airflow_stub
    action_resource.insert_action = insert_action_stub
    action_resource.audit_control_command_db = audit_control_command_db
    action_resource.get_committed_design_version = lambda: DESIGN_VERSION
    action_resource.check_intermediate_commit_revision = (
        CHECK_INTERMEDIATE_COMMIT)

    # with valid input and some parameters
    with mock.patch('shipyard_airflow.control.action.action_validators'
                    '.validate_site_action',
                    side_effect=ApiError(title='bad')):
        with pytest.raises(ApiError) as apie:
            action = action_resource.create_action(
                action={'name': 'deploy_site',
                        'parameters': {
                            'a': 'aaa'
                        }},
                context=context,
                allow_intermediate_commits=False)
            assert action['timestamp']
            assert action['id']
            assert len(action['id']) == 26
            assert action['dag_execution_date'] == '2017-09-06 14:10:08.528402'
            assert action['dag_status'] == 'SCHEDULED'
            assert action['committed_rev_id'] == 1
        assert apie.value.title == 'bad'
Exemplo n.º 15
0
    def on_post(self, req, resp, collection_id):
        """
        Ingests a collection of documents
        """
        content_length = req.content_length or 0
        if (content_length == 0):
            raise ApiError(
                title=('Content-Length is a required header'),
                description='Content Length is 0 or not specified',
                status=falcon.HTTP_400,
                error_list=[{
                    'message': (
                        'The Content-Length specified is 0 or not set. Check '
                        'that a valid payload is included with this request '
                        'and that your client is properly including a '
                        'Content-Length header. Note that a newline character '
                        'in a prior header can trigger subsequent headers to '
                        'be ignored and trigger this failure.')
                }],
                retry=False, )
        document_data = req.stream.read(content_length)

        buffer_mode = req.get_param('buffermode')

        helper = ConfigdocsHelper(req.context)
        validations = self.post_collection(
            helper=helper,
            collection_id=collection_id,
            document_data=document_data,
            buffer_mode_param=buffer_mode)

        resp.location = '/api/v1.0/configdocs/{}'.format(collection_id)
        resp.body = self.to_json(validations)
        resp.status = falcon.HTTP_201
Exemplo n.º 16
0
 def commit_configdocs(self, helper, force, dryrun):
     """
     Attempts to commit the configdocs
     """
     if helper.is_buffer_empty():
         raise ApiError(
             title=CommitConfigDocsResource.unable_to_commmit,
             description='There are no documents in the buffer to commit',
             status=falcon.HTTP_409,
             retry=True)
     validations = helper.get_validations_for_buffer()
     if dryrun:
         validations['code'] = falcon.HTTP_200
         if 'message' in validations:
             validations['message'] = (
                 validations['message'] + ' DRYRUN')
         else:
             validations['message'] = 'DRYRUN'
     else:
         if force or validations.get('status') == 'Success':
             helper.tag_buffer(configdocs_helper.COMMITTED)
         if force and validations.get('status') == 'Failure':
             # override the status in the response
             validations['code'] = falcon.HTTP_200
             if 'message' in validations:
                 validations['message'] = (
                     validations['message'] + ' FORCED SUCCESS')
             else:
                 validations['message'] = 'FORCED SUCCESS'
     return validations
Exemplo n.º 17
0
    def get_step(self, step_id, try_number=None):
        """
        :param step_id: Step ID - task_id in db
        :param try_number: Number of try - try_number in db

        returns: Step
        """
        # Retrieve step. Note that we will get a list and it will
        # be the content of step[0]
        step_list = [
            x for x in self._get_all_steps() if step_id == x['task_id'] and (
                try_number is None or try_number == x['try_number'])
        ]
        # try_number is needed to get correct task from correct worker
        # the worker host for request URL
        # is referenced in correct task's 'hostname' field

        if not step_list:
            raise ApiError(title='Step Not Found!',
                           description='Unable to retrieve Step',
                           status=falcon.HTTP_404)
        else:
            step = step_list[0]
            LOG.debug("Step Located:")
            LOG.debug(step)

            return step
Exemplo n.º 18
0
 def validate(self):
     if self.action.get('committed_rev_id') is None:
         raise ApiError(
             title='No committed configdocs',
             description=(
                 'Unable to locate a committed revision in Deckhand'),
             status=falcon.HTTP_400,
             retry=False)
Exemplo n.º 19
0
 def _validate_version_parameter(self, version):
     # performs validation of version parameter
     if version.lower() not in VERSION_VALUES:
         raise ApiError(
             title='Invalid version query parameter specified',
             description=(
                 'version must be {}'.format(', '.join(VERSION_VALUES))),
             status=falcon.HTTP_400,
             retry=False, )
Exemplo n.º 20
0
    def _get_dag_info(self):
        """
        returns: Dag Information
        """
        # Retrieve 'dag_id' and 'dag_execution_date'
        dag_id = self.action.get('dag_id')
        dag_execution_date = self.action.get('dag_execution_date')

        if not dag_id:
            raise ApiError(title='Dag ID Not Found!',
                           description='Unable to retrieve Dag ID',
                           status=falcon.HTTP_404)
        elif not dag_execution_date:
            raise ApiError(title='Execution Date Not Found!',
                           description='Unable to retrieve Execution Date',
                           status=falcon.HTTP_404)
        else:
            return dag_id, dag_execution_date
Exemplo n.º 21
0
    def _exhume_date(self, dag_id, log_string):
        # TODO(bryan-strassner) refactor this to an airflow api client module

        # we are unable to use the response time because that
        # does not match the time when the dag was recorded.
        # We have to parse the stdout returned to find the
        # Created <DagRun {dag_id} @ {timestamp}
        # e.g.
        # ...- Created <DagRun deploy_site @ 2017-09-22 22:16:14: man...
        # split on "Created <DagRun deploy_site @ ", then ': "
        # should get to the desired date string.
        #
        # returns the date found in a date object
        log_split = log_string.split('Created <DagRun {} @ '.format(dag_id))
        if len(log_split) < 2:
            raise ApiError(
                title='Unable to determine if workflow has started',
                description=(
                    'Airflow has not responded with parseable output. '
                    'Shipyard is unable to determine run timestamp'),
                status=falcon.HTTP_500,
                error_list=[{
                    'message': log_string
                }],
                retry=True
            )
        else:
            # everything before the ': ' should be a date/time
            date_split = log_split[1].split(': ')[0]
            try:
                return parse(date_split, ignoretz=True)
            except ValueError as valerr:
                raise ApiError(
                    title='Unable to determine if workflow has started',
                    description=('Airflow has not responded with parseable '
                                 'output. Shipyard is unable to determine run '
                                 'timestamp'),
                    status=falcon.HTTP_500,
                    error_list=[{
                        'message': 'value {} has caused {}'.format(date_split,
                                                                   valerr)
                    }],
                    retry=True,
                )
Exemplo n.º 22
0
    def create_action(self, action, context, allow_intermediate_commits=False):
        action_mappings = _action_mappings()
        # use uuid assigned for this request as the id of the action.
        action['id'] = ulid.ulid()
        # the invoking user
        action['user'] = context.user
        # add current timestamp (UTC) to the action.
        action['timestamp'] = str(datetime.utcnow())
        # validate that action is supported.
        LOG.info("Attempting action: %s", action['name'])
        if action['name'] not in action_mappings:
            raise ApiError(
                title='Unable to start action',
                description='Unsupported Action: {}'.format(action['name']))

        dag = action_mappings.get(action['name'])['dag']
        action['dag_id'] = dag

        # Set up configdocs_helper
        self.configdocs_helper = ConfigdocsHelper(context)

        # Retrieve last committed design revision
        action['committed_rev_id'] = self.get_committed_design_version()

        # Check for intermediate commit
        self.check_intermediate_commit_revision(allow_intermediate_commits)

        # populate action parameters if they are not set
        if 'parameters' not in action:
            action['parameters'] = {}

        # validate if there is any validation to do
        for validator in action_mappings.get(action['name'])['validators']:
            # validators will raise ApiError if they are not validated.
            validator(action)

        # invoke airflow, get the dag's date
        dag_execution_date = self.invoke_airflow_dag(
            dag_id=dag, action=action, context=context)
        # set values on the action
        action['dag_execution_date'] = dag_execution_date
        action['dag_status'] = 'SCHEDULED'

        # context_marker is the uuid from the request context
        action['context_marker'] = context.request_id

        # insert the action into the shipyard db
        self.insert_action(action=action)
        self.audit_control_command_db({
            'id': ulid.ulid(),
            'action_id': action['id'],
            'command': 'invoke',
            'user': context.user
        })

        return action
Exemplo n.º 23
0
    def get_action_step(self, action_id, step_id, verbosity=MAX_VERBOSITY):
        """Retrieve a single step

        :param action_id: the action_id containing the target step
        :param step_id: the step to retrieve
        :param verbosity: the level of detail to return for the step. Defaults
            to the highest level of detail.

        Interacts with airflow and the shipyard database to return the
        requested step invoked through shipyard.
        """
        action = self.get_action_db(action_id=action_id)

        if action is None:
            raise ApiError(
                title='Action not found',
                description='Unknown action {}'.format(action_id),
                status=falcon.HTTP_404)

        # resolve the ids for lookup of steps
        dag_id = action['dag_id']
        dag_execution_date = action['dag_execution_date']

        # get the action steps from shipyard db
        steps = self.get_tasks_db(dag_id, dag_execution_date)
        step_notes = notes_helper.get_step_notes(
            action_id=action_id,
            step_id=step_id,
            verbosity=verbosity
        )
        for idx, step in enumerate(steps):
            if step_id == step['task_id']:
                step['index'] = idx + 1
                step['notes'] = []
                for note in step_notes:
                    step['notes'].append(note.view())
                return step

        # if we didn't find it, 404
        raise ApiError(
            title='Step not found',
            description='Unknown step {}'.format(step_id),
            status=falcon.HTTP_404)
Exemplo n.º 24
0
    def post_collection(self,
                        helper,
                        collection_id,
                        document_data,
                        buffer_mode_param=None):
        """
        Ingest the collection after checking preconditions
        """
        buffer_mode = ConfigdocsHelper.get_buffer_mode(buffer_mode_param)

        if helper.is_buffer_valid_for_bucket(collection_id, buffer_mode):
            buffer_revision = helper.add_collection(collection_id,
                                                    document_data)
            if helper.is_collection_in_buffer(collection_id):
                return helper.get_deckhand_validation_status(buffer_revision)
            else:
                raise ApiError(
                    title=('Collection {} not added to Shipyard '
                           'buffer'.format(collection_id)),
                    description='Collection empty or resulted in no revision',
                    status=falcon.HTTP_400,
                    error_list=[{
                        'message': (
                            'Empty collections are not supported. After '
                            'processing, the collection {} added no new '
                            'revision, and has been rejected as invalid '
                            'input'.format(collection_id))
                    }],
                    retry=False,
                )
        else:
            raise ApiError(
                title='Invalid collection specified for buffer',
                description='Buffermode : {}'.format(buffer_mode.value),
                status=falcon.HTTP_409,
                error_list=[{
                    'message': ('Buffer is either not empty or the '
                                'collection already exists in buffer. '
                                'Setting a different buffermode may '
                                'provide the desired functionality')
                }],
                retry=False,
            )
Exemplo n.º 25
0
 def pause_dag(self, dag_id, execution_date):
     """
     Sets the pause flag on this dag/execution
     """
     try:
         AIRFLOW_DB.pause_dag_run(dag_id=dag_id,
                                  execution_date=execution_date)
     except AirflowStateError as state_error:
         raise ApiError(title='Unable to pause action',
                        description=state_error.message,
                        status=falcon.HTTP_409)
Exemplo n.º 26
0
    def get_note_details(self, note):
        """Retrieve the note details from the notes_helper

        :param note: the note with extended information
        """
        try:
            return notes_helper.get_note_details(note)
        except NoteURLNotSpecifiedError:
            raise ApiError(
                title="No further note details are available",
                description=("Note {} has no additional information to "
                             "return".format(note.note_id)),
                status=falcon.HTTP_404)
        except NoteURLRetrievalError:
            raise ApiError(
                title="Unable to retrieve URL information for note",
                description=("Note {} has additional information, but it "
                             "cannot be accessed by Shipyard at this "
                             "time".format(note.note_id)),
                status=falcon.HTTP_500)
Exemplo n.º 27
0
    def parse_step_id(**kwargs):
        """
        Retreive step_id from the input parameters
        """
        step_id = kwargs.get('step_id')
        if step_id is None:
            raise ApiError(title='Missing Step ID!',
                           description='Missing Step ID',
                           status=falcon.HTTP_400)

        LOG.debug("Step ID parsed: %s", step_id)

        return step_id
Exemplo n.º 28
0
    def get_action_validation(self, action_id, validation_id):
        """
        Interacts with the shipyard database to return the requested
        validation information
        :returns: the validation dicitonary object
        """
        action = self.get_action_db(action_id=action_id)

        if action is None:
            raise ApiError(title='Action not found',
                           description='Unknown action {}'.format(action_id),
                           status=falcon.HTTP_404)

        validation = self.get_validation_db(validation_id=validation_id)
        if validation is not None:
            return validation

        # if we didn't find it, 404
        raise ApiError(
            title='Validation not found',
            description='Unknown validation {}'.format(validation_id),
            status=falcon.HTTP_404)
Exemplo n.º 29
0
    def get_rendered_configdocs(self, version=BUFFER, cleartext_secrets=False):
        """
        Returns the rendered configuration documents for the specified
        revision (by name BUFFER, COMMITTED, LAST_SITE_ACTION,
        SUCCESSFUL_SITE_ACTION)
        """
        revision_dict = self._get_revision_dict()

        # Raise Exceptions if we received unexpected version
        if version not in [
                BUFFER, COMMITTED, LAST_SITE_ACTION, SUCCESSFUL_SITE_ACTION
        ]:
            raise ApiError(
                title='Invalid version',
                description='{} is not a valid version'.format(version),
                status=falcon.HTTP_400,
                retry=False)

        if revision_dict.get(version):
            revision_id = revision_dict.get(version).get('id')

            try:
                return self.deckhand.get_rendered_docs_from_revision(
                    cleartext_secrets=cleartext_secrets,
                    revision_id=revision_id)
            except DeckhandError as de:
                raise ApiError(
                    title='Deckhand indicated an error while rendering',
                    description=de.response_message,
                    status=falcon.HTTP_500,
                    retry=False)

        else:
            raise ApiError(
                title='This revision does not exist',
                description='{} version does not exist'.format(version),
                status=falcon.HTTP_404,
                retry=False)
Exemplo n.º 30
0
    def _get_action_info(self):
        """
        :returns: Action Information
        """
        # Retrieve information related to the action id
        action = self._get_action_db(self.action_id)

        if not action:
            raise ApiError(title='Action Not Found!',
                           description='Unknown Action {}'.format(
                               self.action_id),
                           status=falcon.HTTP_404)
        else:
            return action