def test_determine_lifecycle():
dag_statuses = [
{
'input': 'queued',
'expected': 'Pending'
},
{
'input': 'ShUTdown',
'expected': 'Failed'
},
{
'input': 'RUNNING',
'expected': 'Processing'
},
{
'input': 'None',
'expected': 'Pending'
},
{
'input': None,
'expected': 'Pending'
},
{
'input': 'bogusBroken',
'expected': 'Unknown (bogusBroken)'
},
]
for status_pair in dag_statuses:
assert (status_pair['expected'] == action_helper.determine_lifecycle(
status_pair['input']))
def get_all_actions(self):
"""
Interacts with airflow and the shipyard database to return the list of
actions invoked through shipyard.
"""
# fetch actions from the shipyard db
all_actions = self.get_action_map()
# fetch the associated dags, steps from the airflow db
all_dag_runs = self.get_dag_run_map()
all_tasks = self.get_all_tasks_db()
# correlate the actions and dags into a list of action entites
actions = []
for action_id, action in all_actions.items():
dag_key = action['dag_id'] + action['dag_execution_date']
dag_key_id = action['dag_id']
dag_key_date = action['dag_execution_date']
# locate the dag run associated
dag_state = all_dag_runs.get(dag_key, {}).get('state', None)
# get the dag status from the dag run state
action['dag_status'] = dag_state
action['action_lifecycle'] = determine_lifecycle(dag_state)
# get the steps summary
action_tasks = [
step for step in all_tasks
if step['dag_id'].startswith(dag_key_id)
and step['execution_date'].strftime(
'%Y-%m-%dT%H:%M:%S') == dag_key_date
]
action['steps'] = format_action_steps(action_id, action_tasks)
actions.append(action)
return actions
def get_action(self, action_id):
"""
Interacts with airflow and the shipyard database to return the
requested action invoked through shipyard.
"""
# get the action from shipyard db
action = self.get_action_db(action_id=action_id)
if action is None:
raise ApiError(title='Action not found',
description='Unknown Action: {}'.format(action_id),
status=falcon.HTTP_404)
# lookup the dag and tasks based on the associated dag_id,
# execution_date
dag_id = action['dag_id']
dag_execution_date = action['dag_execution_date']
dag = self.get_dag_run_by_id(dag_id, dag_execution_date)
steps = self.get_tasks_db(dag_id, dag_execution_date)
if dag is not None:
# put the values together into an "action" object
action['dag_status'] = dag['state']
action['action_lifecycle'] = determine_lifecycle(dag['state'])
action['steps'] = format_action_steps(action_id, steps)
action['validations'] = self.get_validations_db(action_id)
action['command_audit'] = self.get_action_command_audit_db(action_id)
return action
def get_action(self, action_id, verbosity):
"""
Interacts with airflow and the shipyard database to return the
requested action invoked through shipyard.
:param action_id: the action_id to look up
:param verbosity: the maximum verbosity for the associated action.
note that the associated steps will only support a verbosity
of 1 when retrieving an action (but support more verbosity when
retreiving the step itself)
"""
# get the action from shipyard db
action = self.get_action_db(action_id=action_id)
if action is None:
raise ApiError(
title='Action not found',
description='Unknown Action: {}'.format(action_id),
status=falcon.HTTP_404)
# lookup the dag and tasks based on the associated dag_id,
# execution_date
dag_id = action['dag_id']
dag_execution_date = action['dag_execution_date']
dag = self.get_dag_run_by_id(dag_id, dag_execution_date)
steps = self.get_tasks_db(dag_id, dag_execution_date)
if dag is not None:
# put the values together into an "action" object
action['dag_status'] = dag['state']
action['action_lifecycle'] = determine_lifecycle(dag['state'])
step_verbosity = MIN_VERBOSITY if (
verbosity > MIN_VERBOSITY) else verbosity
action['steps'] = format_action_steps(
action_id=action_id,
steps=steps,
verbosity=step_verbosity
)
action['validations'] = self.get_validations_db(action_id)
action['command_audit'] = self.get_action_command_audit_db(action_id)
notes = notes_helper.get_action_notes(
action_id=action_id,
verbosity=verbosity
)
action['notes'] = []
for note in notes:
action['notes'].append(note.view())
return action
def get_all_actions(self, verbosity):
"""Retrieve all actions known to Shipyard
:param verbosity: Integer 0-5, the level of verbosity applied to the
response's notes.
Interacts with airflow and the shipyard database to return the list of
actions invoked through shipyard.
"""
# fetch actions from the shipyard db
all_actions = self.get_action_map()
# fetch the associated dags, steps from the airflow db
all_dag_runs = self.get_dag_run_map()
all_tasks = self.get_all_tasks_db()
notes = notes_helper.get_all_action_notes(verbosity=verbosity)
# correlate the actions and dags into a list of action entites
actions = []
for action_id, action in all_actions.items():
dag_key = action['dag_id'] + action['dag_execution_date']
dag_key_id = action['dag_id']
dag_key_date = action['dag_execution_date']
# locate the dag run associated
dag_state = all_dag_runs.get(dag_key, {}).get('state', None)
# get the dag status from the dag run state
action['dag_status'] = dag_state
action['action_lifecycle'] = determine_lifecycle(dag_state)
# get the steps summary
action_tasks = [
step for step in all_tasks
if step['dag_id'].startswith(dag_key_id)
and step['execution_date'].strftime(
'%Y-%m-%dT%H:%M:%S') == dag_key_date
]
action['steps'] = format_action_steps(action_id=action_id,
steps=action_tasks,
verbosity=0)
action['notes'] = []
for note in notes.get(action_id, []):
action['notes'].append(note.view())
actions.append(action)
return actions
