def test_commit_configdocs(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_revision = lambda x: {
'status': 'Success'
}
helper.get_revision_id = lambda x: 1
commit_resp = ccdr.commit_configdocs(helper, False, False)
mock_method.assert_called_once_with('committed')
assert commit_resp['status'] == 'Success'
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_revision = (
lambda x: {
'status': 'Failure',
'code': '400 Bad Request',
'message': 'this is a mock response'
})
helper.get_revision_id = lambda x: 1
commit_resp = ccdr.commit_configdocs(helper, False, False)
assert '400' in commit_resp['code']
assert commit_resp['message'] is not None
assert commit_resp['status'] == 'Failure'
def test_commit_configdocs_dryrun(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_revision = lambda x: {
'status': 'Success'
}
helper.get_revision_id = lambda x: 1
commit_resp = ccdr.commit_configdocs(helper, False, True)
assert '200' in commit_resp['code']
assert commit_resp['message'] == 'DRYRUN'
assert commit_resp['status'] == 'Success'
def test_commit_configdocs_force(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_revision = lambda x: {
'status': 'Failure'
}
helper.get_revision_id = lambda x: 1
commit_resp = ccdr.commit_configdocs(helper, True, False)
mock_method.assert_called_once_with('committed')
assert '200' in commit_resp['code']
assert 'FORCED' in commit_resp['message']
assert commit_resp['status'] == 'Failure'
class ActionsResource(BaseResource):
"""
The actions resource represent the asyncrhonous invocations of shipyard
"""
@policy.ApiEnforcer(policy.LIST_ACTIONS)
def on_get(self, req, resp, **kwargs):
"""
Return actions that have been invoked through shipyard.
:returns: a json array of action entities
"""
resp.body = self.to_json(
self.get_all_actions(verbosity=req.context.verbosity))
resp.status = falcon.HTTP_200
@policy.ApiEnforcer(policy.CREATE_ACTION)
def on_post(self, req, resp, **kwargs):
"""
Accept an action into shipyard
"""
# The 'allow-intermediate-commits' query parameter is set to False
# unless explicitly set to True
allow_intermediate_commits = (req.get_param_as_bool(
name='allow-intermediate-commits'))
input_action = self.req_json(req, validate_json_schema=ACTION)
action = self.create_action(
action=input_action,
context=req.context,
allow_intermediate_commits=allow_intermediate_commits)
LOG.info("Id %s generated for action %s", action['id'], action['name'])
# respond with the action and location for checking status
resp.status = falcon.HTTP_201
resp.body = self.to_json(action)
resp.location = '/api/v1.0/actions/{}'.format(action['id'])
def create_action(self, action, context, allow_intermediate_commits=False):
# use uuid assigned for this request as the id of the action.
action['id'] = ulid.ulid()
# the invoking user
action['user'] = context.user
# add current timestamp (UTC) to the action.
action['timestamp'] = str(datetime.utcnow())
# add external marker that is the passed with request context
action['context_marker'] = context.request_id
# validate that action is supported.
LOG.info("Attempting action: %s", action['name'])
action_mappings = _action_mappings()
if action['name'] not in action_mappings:
raise ApiError(title='Unable to start action',
description='Unsupported Action: {}'.format(
action['name']))
action_cfg = action_mappings.get(action['name'])
# check access to specific actions - lack of access will exception out
policy.check_auth(context, action_cfg['rbac_policy'])
dag = action_cfg['dag']
action['dag_id'] = dag
# Set up configdocs_helper
self.configdocs_helper = ConfigdocsHelper(context)
# Retrieve last committed design revision
action['committed_rev_id'] = self.get_committed_design_version()
# Set if intermediate commits are ignored
action['allow_intermediate_commits'] = allow_intermediate_commits
# populate action parameters if they are not set
if 'parameters' not in action:
action['parameters'] = {}
for validator in action_cfg['validators']:
# validators will raise ApiError if they fail validation.
# validators are expected to accept action as a parameter, but
# handle all other kwargs (e.g. def vdtr(action, **kwargs): even if
# they don't use that parameter.
validator(action=action, configdocs_helper=self.configdocs_helper)
# invoke airflow, get the dag's date
dag_execution_date = self.invoke_airflow_dag(dag_id=dag,
action=action,
context=context)
# set values on the action
action['dag_execution_date'] = dag_execution_date
action['dag_status'] = 'SCHEDULED'
# insert the action into the shipyard db
# TODO(b-str): When invoke_airflow_dag triggers a DAG but fails to
# respond properly, no record is inserted, so there is a running
# process with no tracking in the Shipyard database. This is not
# ideal.
self.insert_action(action=action)
notes_helper.make_action_note(action_id=action['id'],
note_val="Configdoc revision {}".format(
action['committed_rev_id']))
self.audit_control_command_db({
'id': ulid.ulid(),
'action_id': action['id'],
'command': 'invoke',
'user': context.user
})
return action
def get_all_actions(self, verbosity):
"""Retrieve all actions known to Shipyard
:param verbosity: Integer 0-5, the level of verbosity applied to the
response's notes.
Interacts with airflow and the shipyard database to return the list of
actions invoked through shipyard.
"""
# fetch actions from the shipyard db
all_actions = self.get_action_map()
# fetch the associated dags, steps from the airflow db
all_dag_runs = self.get_dag_run_map()
all_tasks = self.get_all_tasks_db()
notes = notes_helper.get_all_action_notes(verbosity=verbosity)
# correlate the actions and dags into a list of action entites
actions = []
for action_id, action in all_actions.items():
dag_key = action['dag_id'] + action['dag_execution_date']
dag_key_id = action['dag_id']
dag_key_date = action['dag_execution_date']
# locate the dag run associated
dag_state = all_dag_runs.get(dag_key, {}).get('state', None)
# get the dag status from the dag run state
action['dag_status'] = dag_state
action['action_lifecycle'] = determine_lifecycle(dag_state)
# get the steps summary
action_tasks = [
step for step in all_tasks
if step['dag_id'].startswith(dag_key_id)
and step['execution_date'].strftime(
'%Y-%m-%dT%H:%M:%S') == dag_key_date
]
action['steps'] = format_action_steps(action_id=action_id,
steps=action_tasks,
verbosity=0)
action['notes'] = []
for note in notes.get(action_id, []):
action['notes'].append(note.view())
actions.append(action)
return actions
def get_action_map(self):
"""
maps an array of dictionaries to a dictonary of the same results by id
:returns: a dictionary of dictionaries keyed by action id
"""
return {action['id']: action for action in self.get_all_actions_db()}
def get_all_actions_db(self):
"""
Wrapper for call to the shipyard database to get all actions
:returns: a list of dictionaries keyed by action id
"""
return SHIPYARD_DB.get_all_submitted_actions()
def get_dag_run_map(self):
"""
Maps an array of dag runs to a keyed dictionary
:returns: a dictionary of dictionaries keyed by dag_id and
execution_date
"""
return {
run['dag_id'] +
run['execution_date'].strftime('%Y-%m-%dT%H:%M:%S'): run
for run in self.get_all_dag_runs_db()
}
def get_all_dag_runs_db(self):
"""
Wrapper for call to the airflow db to get all dag runs
:returns: a list of dictionaries representing dag runs in airflow
"""
return AIRFLOW_DB.get_all_dag_runs()
def get_all_tasks_db(self):
"""
Wrapper for call to the airflow db to get all tasks
:returns: a list of task dictionaries
"""
return AIRFLOW_DB.get_all_tasks()
def insert_action(self, action):
"""
Wrapper for call to the shipyard db to insert an action
"""
return SHIPYARD_DB.insert_action(action)
def audit_control_command_db(self, action_audit):
"""
Wrapper for the shipyard db call to record an audit of the
action control taken
"""
return SHIPYARD_DB.insert_action_command_audit(action_audit)
def invoke_airflow_dag(self, dag_id, action, context):
"""
Call airflow, and invoke a dag
:param dag_id: the name of the dag to invoke
:param action: the action structure to invoke the dag with
"""
# TODO(bryan-strassner) refactor the mechanics of this method to an
# airflow api client module
# Retrieve URL
web_server_url = CONF.base.web_server
api_path = 'api/experimental/dags/{}/dag_runs'
req_url = os.path.join(web_server_url, api_path.format(dag_id))
c_timeout = CONF.base.airflow_api_connect_timeout
r_timeout = CONF.base.airflow_api_read_timeout
if 'Error' in web_server_url:
raise ApiError(
title='Unable to invoke workflow',
description=('Airflow URL not found by Shipyard. '
'Shipyard configuration is missing web_server '
'value'),
status=falcon.HTTP_503,
retry=True,
)
else:
# No cache please
headers = {'Cache-Control': 'no-cache'}
# "conf" - JSON string that gets pickled into the DagRun's
# conf attribute. The conf is passed as as a string of escaped
# json inside the json payload accepted by the API.
conf_value = self.to_json({'action': action})
payload = {'run_id': action['id'], 'conf': conf_value}
try:
resp = requests.post(req_url,
timeout=(c_timeout, r_timeout),
headers=headers,
json=payload)
LOG.info('Response code from Airflow trigger_dag: %s',
resp.status_code)
# any 4xx/5xx will be HTTPError, which are RequestException
resp.raise_for_status()
response = resp.json()
LOG.info('Response from Airflow trigger_dag: %s', response)
except RequestException as rex:
LOG.error("Request to airflow failed: %s", rex.args)
raise ApiError(
title='Unable to complete request to Airflow',
description=(
'Airflow could not be contacted properly by Shipyard.'
),
status=falcon.HTTP_503,
error_list=[{
'message': str(type(rex))
}],
retry=True,
)
dag_time = self._exhume_date(dag_id, response['message'])
dag_execution_date = dag_time.strftime('%Y-%m-%dT%H:%M:%S')
return dag_execution_date
def _exhume_date(self, dag_id, log_string):
# we are unable to use the response time because that
# does not match the time when the dag was recorded.
# We have to parse the returned message to find the
# Created <DagRun {dag_id} @ {timestamp}
# e.g.
# ...- Created <DagRun deploy_site @ 2017-09-22 22:16:14: man...
# split on "Created <DagRun deploy_site @ ", then ': "
# should get to the desired date string.
#
# returns the date found in a date object
log_split = log_string.split('Created <DagRun {} @ '.format(dag_id))
if len(log_split) < 2:
raise ApiError(
title='Unable to determine if workflow has started',
description=(
'Airflow has not responded with parseable output. '
'Shipyard is unable to determine run timestamp'),
status=falcon.HTTP_500,
error_list=[{
'message': log_string
}],
retry=True)
else:
# everything before the ': ' should be a date/time
date_split = log_split[1].split(': ')[0]
try:
return parse(date_split, ignoretz=True)
except ValueError as valerr:
raise ApiError(
title='Unable to determine if workflow has started',
description=('Airflow has not responded with parseable '
'output. Shipyard is unable to determine run '
'timestamp'),
status=falcon.HTTP_500,
error_list=[{
'message':
'value {} has caused {}'.format(date_split, valerr)
}],
retry=True,
)
def get_committed_design_version(self):
"""Retrieves the committed design version from Deckhand.
Returns None if there is no committed version
"""
committed_rev_id = self.configdocs_helper.get_revision_id(
configdocs_helper.COMMITTED)
if committed_rev_id:
LOG.info("The committed revision in Deckhand is %d",
committed_rev_id)
return committed_rev_id
LOG.info("No committed revision found in Deckhand")
return None
class ActionsResource(BaseResource):
"""
The actions resource represent the asyncrhonous invocations of shipyard
"""
@policy.ApiEnforcer('workflow_orchestrator:list_actions')
def on_get(self, req, resp, **kwargs):
"""
Return actions that have been invoked through shipyard.
:returns: a json array of action entities
"""
resp.body = self.to_json(self.get_all_actions())
resp.status = falcon.HTTP_200
@policy.ApiEnforcer('workflow_orchestrator:create_action')
def on_post(self, req, resp, **kwargs):
"""
Accept an action into shipyard
"""
# The 'allow-intermediate-commits' query parameter is set to False
# unless explicitly set to True
allow_intermediate_commits = (
req.get_param_as_bool(name='allow-intermediate-commits'))
input_action = self.req_json(req, validate_json_schema=ACTION)
action = self.create_action(
action=input_action,
context=req.context,
allow_intermediate_commits=allow_intermediate_commits)
LOG.info("Id %s generated for action %s", action['id'], action['name'])
# respond with the action and location for checking status
resp.status = falcon.HTTP_201
resp.body = self.to_json(action)
resp.location = '/api/v1.0/actions/{}'.format(action['id'])
def create_action(self, action, context, allow_intermediate_commits=False):
action_mappings = _action_mappings()
# use uuid assigned for this request as the id of the action.
action['id'] = ulid.ulid()
# the invoking user
action['user'] = context.user
# add current timestamp (UTC) to the action.
action['timestamp'] = str(datetime.utcnow())
# validate that action is supported.
LOG.info("Attempting action: %s", action['name'])
if action['name'] not in action_mappings:
raise ApiError(
title='Unable to start action',
description='Unsupported Action: {}'.format(action['name']))
dag = action_mappings.get(action['name'])['dag']
action['dag_id'] = dag
# Set up configdocs_helper
self.configdocs_helper = ConfigdocsHelper(context)
# Retrieve last committed design revision
action['committed_rev_id'] = self.get_committed_design_version()
# Check for intermediate commit
self.check_intermediate_commit_revision(allow_intermediate_commits)
# populate action parameters if they are not set
if 'parameters' not in action:
action['parameters'] = {}
# validate if there is any validation to do
for validator in action_mappings.get(action['name'])['validators']:
# validators will raise ApiError if they are not validated.
validator(action)
# invoke airflow, get the dag's date
dag_execution_date = self.invoke_airflow_dag(
dag_id=dag, action=action, context=context)
# set values on the action
action['dag_execution_date'] = dag_execution_date
action['dag_status'] = 'SCHEDULED'
# context_marker is the uuid from the request context
action['context_marker'] = context.request_id
# insert the action into the shipyard db
self.insert_action(action=action)
self.audit_control_command_db({
'id': ulid.ulid(),
'action_id': action['id'],
'command': 'invoke',
'user': context.user
})
return action
def get_all_actions(self):
"""
Interacts with airflow and the shipyard database to return the list of
actions invoked through shipyard.
"""
# fetch actions from the shipyard db
all_actions = self.get_action_map()
# fetch the associated dags, steps from the airflow db
all_dag_runs = self.get_dag_run_map()
all_tasks = self.get_all_tasks_db()
# correlate the actions and dags into a list of action entites
actions = []
for action_id, action in all_actions.items():
dag_key = action['dag_id'] + action['dag_execution_date']
dag_key_id = action['dag_id']
dag_key_date = action['dag_execution_date']
# locate the dag run associated
dag_state = all_dag_runs.get(dag_key, {}).get('state', None)
# get the dag status from the dag run state
action['dag_status'] = dag_state
action['action_lifecycle'] = determine_lifecycle(dag_state)
# get the steps summary
action_tasks = [
step for step in all_tasks
if step['dag_id'].startswith(dag_key_id) and
step['execution_date'].strftime(
'%Y-%m-%dT%H:%M:%S') == dag_key_date
]
action['steps'] = format_action_steps(action_id, action_tasks)
actions.append(action)
return actions
def get_action_map(self):
"""
maps an array of dictionaries to a dictonary of the same results by id
:returns: a dictionary of dictionaries keyed by action id
"""
return {action['id']: action for action in self.get_all_actions_db()}
def get_all_actions_db(self):
"""
Wrapper for call to the shipyard database to get all actions
:returns: a list of dictionaries keyed by action id
"""
return SHIPYARD_DB.get_all_submitted_actions()
def get_dag_run_map(self):
"""
Maps an array of dag runs to a keyed dictionary
:returns: a dictionary of dictionaries keyed by dag_id and
execution_date
"""
return {
run['dag_id'] +
run['execution_date'].strftime('%Y-%m-%dT%H:%M:%S'): run
for run in self.get_all_dag_runs_db()
}
def get_all_dag_runs_db(self):
"""
Wrapper for call to the airflow db to get all dag runs
:returns: a list of dictionaries representing dag runs in airflow
"""
return AIRFLOW_DB.get_all_dag_runs()
def get_all_tasks_db(self):
"""
Wrapper for call to the airflow db to get all tasks
:returns: a list of task dictionaries
"""
return AIRFLOW_DB.get_all_tasks()
def insert_action(self, action):
"""
Wrapper for call to the shipyard db to insert an action
"""
return SHIPYARD_DB.insert_action(action)
def audit_control_command_db(self, action_audit):
"""
Wrapper for the shipyard db call to record an audit of the
action control taken
"""
return SHIPYARD_DB.insert_action_command_audit(action_audit)
def invoke_airflow_dag(self, dag_id, action, context):
"""
Call airflow, and invoke a dag
:param dag_id: the name of the dag to invoke
:param action: the action structure to invoke the dag with
"""
# TODO(bryan-strassner) refactor the mechanics of this method to an
# airflow api client module
# Retrieve URL
web_server_url = CONF.base.web_server
c_timeout = CONF.base.airflow_api_connect_timeout
r_timeout = CONF.base.airflow_api_read_timeout
if 'Error' in web_server_url:
raise ApiError(
title='Unable to invoke workflow',
description=('Airflow URL not found by Shipyard. '
'Shipyard configuration is missing web_server '
'value'),
status=falcon.HTTP_503,
retry=True, )
else:
conf_value = {'action': action}
# "conf" - JSON string that gets pickled into the DagRun's
# conf attribute
req_url = ('{}admin/rest_api/api?api=trigger_dag&dag_id={}'
'&conf={}'.format(web_server_url,
dag_id, self.to_json(conf_value)))
try:
resp = requests.get(req_url, timeout=(c_timeout, r_timeout))
LOG.info('Response code from Airflow trigger_dag: %s',
resp.status_code)
# any 4xx/5xx will be HTTPError, which are RequestException
resp.raise_for_status()
response = resp.json()
LOG.info('Response from Airflow trigger_dag: %s', response)
except RequestException as rex:
LOG.error("Request to airflow failed: %s", rex.args)
raise ApiError(
title='Unable to complete request to Airflow',
description=(
'Airflow could not be contacted properly by Shipyard.'
),
status=falcon.HTTP_503,
error_list=[{
'message': str(type(rex))
}],
retry=True, )
dag_time = self._exhume_date(dag_id,
response['output']['stdout'])
dag_execution_date = dag_time.strftime('%Y-%m-%dT%H:%M:%S')
return dag_execution_date
def _exhume_date(self, dag_id, log_string):
# TODO(bryan-strassner) refactor this to an airflow api client module
# we are unable to use the response time because that
# does not match the time when the dag was recorded.
# We have to parse the stdout returned to find the
# Created <DagRun {dag_id} @ {timestamp}
# e.g.
# ...- Created <DagRun deploy_site @ 2017-09-22 22:16:14: man...
# split on "Created <DagRun deploy_site @ ", then ': "
# should get to the desired date string.
#
# returns the date found in a date object
log_split = log_string.split('Created <DagRun {} @ '.format(dag_id))
if len(log_split) < 2:
raise ApiError(
title='Unable to determine if workflow has started',
description=(
'Airflow has not responded with parseable output. '
'Shipyard is unable to determine run timestamp'),
status=falcon.HTTP_500,
error_list=[{
'message': log_string
}],
retry=True
)
else:
# everything before the ': ' should be a date/time
date_split = log_split[1].split(': ')[0]
try:
return parse(date_split, ignoretz=True)
except ValueError as valerr:
raise ApiError(
title='Unable to determine if workflow has started',
description=('Airflow has not responded with parseable '
'output. Shipyard is unable to determine run '
'timestamp'),
status=falcon.HTTP_500,
error_list=[{
'message': 'value {} has caused {}'.format(date_split,
valerr)
}],
retry=True,
)
def get_committed_design_version(self):
LOG.info("Checking for committed revision in Deckhand...")
committed_rev_id = self.configdocs_helper.get_revision_id(
configdocs_helper.COMMITTED
)
if committed_rev_id:
LOG.info("The committed revision in Deckhand is %d",
committed_rev_id)
return committed_rev_id
else:
raise ApiError(
title='Unable to locate any committed revision in Deckhand',
description='No committed version found in Deckhand',
status=falcon.HTTP_404,
retry=False)
def check_intermediate_commit_revision(self,
allow_intermediate_commits=False):
LOG.info("Checking for intermediate committed revision in Deckhand...")
intermediate_commits = (
self.configdocs_helper.check_intermediate_commit())
if intermediate_commits and not allow_intermediate_commits:
raise ApiError(
title='Intermediate Commit Detected',
description=(
'The current committed revision of documents has '
'other prior commits that have not been used as '
'part of a site action, e.g. update_site. If you '
'are aware and these other commits are intended, '
'please rerun this action with the option '
'`allow-intermediate-commits=True`'),
status=falcon.HTTP_409,
retry=False
)