def get_note_with_access_check(self, context, note_id):
"""Retrieve the note and checks user access to the note
:param context: the request context
:param note_id: the id of the note to retrieve.
:returns: the note
"""
try:
note = notes_helper.get_note(note_id)
note_type = notes_helper.get_note_assoc_id_type(note)
if note_type not in NOTE_TYPE_RBAC:
raise ApiError(
title="Unable to check permission for note type",
description=(
"Shipyard is not correctly identifying note type "
"for note {}".format(note_id)),
status=falcon.HTTP_500,
retry=False)
policy.check_auth(context, NOTE_TYPE_RBAC[note_type])
return note
except NoteNotFoundError:
raise ApiError(
title="No note found",
description=("Note {} is not found".format(note_id)),
status=falcon.HTTP_404)
def get_action(self, action_id, verbosity):
"""
Interacts with airflow and the shipyard database to return the
requested action invoked through shipyard.
:param action_id: the action_id to look up
:param verbosity: the maximum verbosity for the associated action.
note that the associated steps will only support a verbosity
of 1 when retrieving an action (but support more verbosity when
retreiving the step itself)
"""
# get the action from shipyard db
action = self.get_action_db(action_id=action_id)
if action is None:
raise ApiError(
title='Action not found',
description='Unknown Action: {}'.format(action_id),
status=falcon.HTTP_404)
# lookup the dag and tasks based on the associated dag_id,
# execution_date
dag_id = action['dag_id']
dag_execution_date = action['dag_execution_date']
dag = self.get_dag_run_by_id(dag_id, dag_execution_date)
steps = self.get_tasks_db(dag_id, dag_execution_date)
if dag is not None:
# put the values together into an "action" object
action['dag_status'] = dag['state']
action['action_lifecycle'] = determine_lifecycle(dag['state'])
step_verbosity = MIN_VERBOSITY if (
verbosity > MIN_VERBOSITY) else verbosity
action['steps'] = format_action_steps(
action_id=action_id,
steps=steps,
verbosity=step_verbosity
)
action['validations'] = self.get_validations_db(action_id)
action['command_audit'] = self.get_action_command_audit_db(action_id)
notes = notes_helper.get_action_notes(
action_id=action_id,
verbosity=verbosity
)
action['notes'] = []
for note in notes:
action['notes'].append(note.view())
return action
def get_all_actions(self, verbosity):
"""Retrieve all actions known to Shipyard
:param verbosity: Integer 0-5, the level of verbosity applied to the
response's notes.
Interacts with airflow and the shipyard database to return the list of
actions invoked through shipyard.
"""
# fetch actions from the shipyard db
all_actions = self.get_action_map()
# fetch the associated dags, steps from the airflow db
all_dag_runs = self.get_dag_run_map()
all_tasks = self.get_all_tasks_db()
notes = notes_helper.get_all_action_notes(verbosity=verbosity)
# correlate the actions and dags into a list of action entites
actions = []
for action_id, action in all_actions.items():
dag_key = action['dag_id'] + action['dag_execution_date']
dag_key_id = action['dag_id']
dag_key_date = action['dag_execution_date']
# locate the dag run associated
dag_state = all_dag_runs.get(dag_key, {}).get('state', None)
# get the dag status from the dag run state
action['dag_status'] = dag_state
action['action_lifecycle'] = determine_lifecycle(dag_state)
# get the steps summary
action_tasks = [
step for step in all_tasks
if step['dag_id'].startswith(dag_key_id)
and step['execution_date'].strftime(
'%Y-%m-%dT%H:%M:%S') == dag_key_date
]
action['steps'] = format_action_steps(action_id=action_id,
steps=action_tasks,
verbosity=0)
action['notes'] = []
for note in notes.get(action_id, []):
action['notes'].append(note.view())
actions.append(action)
return actions
def get_action_step(self, action_id, step_id, verbosity=MAX_VERBOSITY):
"""Retrieve a single step
:param action_id: the action_id containing the target step
:param step_id: the step to retrieve
:param verbosity: the level of detail to return for the step. Defaults
to the highest level of detail.
Interacts with airflow and the shipyard database to return the
requested step invoked through shipyard.
"""
action = self.get_action_db(action_id=action_id)
if action is None:
raise ApiError(
title='Action not found',
description='Unknown action {}'.format(action_id),
status=falcon.HTTP_404)
# resolve the ids for lookup of steps
dag_id = action['dag_id']
dag_execution_date = action['dag_execution_date']
# get the action steps from shipyard db
steps = self.get_tasks_db(dag_id, dag_execution_date)
step_notes = notes_helper.get_step_notes(
action_id=action_id,
step_id=step_id,
verbosity=verbosity
)
for idx, step in enumerate(steps):
if step_id == step['task_id']:
step['index'] = idx + 1
step['notes'] = []
for note in step_notes:
step['notes'].append(note.view())
return step
# if we didn't find it, 404
raise ApiError(
title='Step not found',
description='Unknown step {}'.format(step_id),
status=falcon.HTTP_404)
def get_note_details(self, note):
"""Retrieve the note details from the notes_helper
:param note: the note with extended information
"""
try:
return notes_helper.get_note_details(note)
except NoteURLNotSpecifiedError:
raise ApiError(
title="No further note details are available",
description=("Note {} has no additional information to "
"return".format(note.note_id)),
status=falcon.HTTP_404)
except NoteURLRetrievalError:
raise ApiError(
title="Unable to retrieve URL information for note",
description=("Note {} has additional information, but it "
"cannot be accessed by Shipyard at this "
"time".format(note.note_id)),
status=falcon.HTTP_500)
def format_action_steps(action_id, steps, verbosity=MIN_VERBOSITY):
""" Converts a list of action step db records to desired format
:param action_id: the action containing steps
:param steps: the list of dictionaries of step info, in database format
:param verbosity: the verbosity level of notes to retrieve, defaults to 1.
if set to a value less than 1, notes will not be retrieved.
"""
if not steps:
return []
steps_response = []
step_notes_dict = notes_helper.get_all_step_notes_for_action(
action_id=action_id, verbosity=verbosity)
for idx, step in enumerate(steps):
step_task_id = step.get('task_id')
steps_response.append(
format_step(action_id=action_id,
step=step,
index=idx + 1,
notes=[
note.view()
for note in step_notes_dict.get(step_task_id, [])
]))
return steps_response
def create_action(self, action, context, allow_intermediate_commits=False):
# use uuid assigned for this request as the id of the action.
action['id'] = ulid.ulid()
# the invoking user
action['user'] = context.user
# add current timestamp (UTC) to the action.
action['timestamp'] = str(datetime.utcnow())
# add external marker that is the passed with request context
action['context_marker'] = context.request_id
# validate that action is supported.
LOG.info("Attempting action: %s", action['name'])
action_mappings = _action_mappings()
if action['name'] not in action_mappings:
raise ApiError(title='Unable to start action',
description='Unsupported Action: {}'.format(
action['name']))
action_cfg = action_mappings.get(action['name'])
# check access to specific actions - lack of access will exception out
policy.check_auth(context, action_cfg['rbac_policy'])
dag = action_cfg['dag']
action['dag_id'] = dag
# Set up configdocs_helper
self.configdocs_helper = ConfigdocsHelper(context)
# Retrieve last committed design revision
action['committed_rev_id'] = self.get_committed_design_version()
# Set if intermediate commits are ignored
action['allow_intermediate_commits'] = allow_intermediate_commits
# populate action parameters if they are not set
if 'parameters' not in action:
action['parameters'] = {}
for validator in action_cfg['validators']:
# validators will raise ApiError if they fail validation.
# validators are expected to accept action as a parameter, but
# handle all other kwargs (e.g. def vdtr(action, **kwargs): even if
# they don't use that parameter.
validator(action=action, configdocs_helper=self.configdocs_helper)
# invoke airflow, get the dag's date
dag_execution_date = self.invoke_airflow_dag(dag_id=dag,
action=action,
context=context)
# set values on the action
action['dag_execution_date'] = dag_execution_date
action['dag_status'] = 'SCHEDULED'
# insert the action into the shipyard db
# TODO(b-str): When invoke_airflow_dag triggers a DAG but fails to
# respond properly, no record is inserted, so there is a running
# process with no tracking in the Shipyard database. This is not
# ideal.
self.insert_action(action=action)
notes_helper.make_action_note(action_id=action['id'],
note_val="Configdoc revision {}".format(
action['committed_rev_id']))
self.audit_control_command_db({
'id': ulid.ulid(),
'action_id': action['id'],
'command': 'invoke',
'user': context.user
})
return action