def test_get_committed_design_version_missing(*args):
with pytest.raises(ApiError) as apie:
act_resource = ActionsResource()
act_resource.configdocs_helper = ConfigdocsHelper(
ShipyardRequestContext())
act_resource.get_committed_design_version()
assert apie.value.status == falcon.HTTP_404
assert apie.value.title == ('Unable to locate any committed revision in '
'Deckhand')
def test_on_get(mock_get_all_actions, mock_authorize):
act_resource = ActionsResource()
context.policy_engine = ShipyardPolicy()
req = create_req(context, None)
resp = create_resp()
act_resource.on_get(req, resp)
mock_authorize.assert_called_once_with(
'workflow_orchestrator:list_actions', context)
assert mock_get_all_actions.call_count == 1
assert resp.body is not None
assert resp.status == '200 OK'
def _gen_action_resource_stubbed():
# TODO(bryan-strassner): mabye subclass this instead?
action_resource = ActionsResource()
action_resource.get_all_actions_db = actions_db
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
action_resource.invoke_airflow_dag = airflow_stub
action_resource.insert_action = insert_action_stub
action_resource.audit_control_command_db = audit_control_command_db
action_resource.get_committed_design_version = lambda: DESIGN_VERSION
return action_resource
def test_invoke_airflow_dag_errors(mock_info):
act_resource = ActionsResource()
dag_id = 'test_dag_id'
action = 'test_action'
web_server_url = CONF.base.web_server
conf_value = {'action': action}
responses.add(
method='GET',
url='{}admin/rest_api/api?api=trigger_dag&dag_id={}'
'&conf={}'.format(web_server_url, dag_id,
act_resource.to_json(conf_value)),
body=json.dumps({
"error": "not found"
}),
status=404,
content_type='application/json')
with pytest.raises(ApiError) as expected_exc:
act_resource.invoke_airflow_dag(dag_id, action, context)
mock_info.assert_called_with('Response code from Airflow trigger_dag: %s',
404)
assert 'Unable to complete request to Airflow' in str(expected_exc)
assert 'Airflow could not be contacted properly by Shipyard' in str(
expected_exc)
with mock.patch.object(actions_api, 'CONF') as mock_conf:
mock_conf.base.web_server = 'Error'
with pytest.raises(ApiError) as expected_exc:
act_resource.invoke_airflow_dag(dag_id, action, context)
assert 'Unable to invoke workflow' in str(expected_exc)
assert ('Airflow URL not found by Shipyard. Shipyard configuration is '
'missing web_server value') in str(expected_exc)
def test_invoke_airflow_dag_success(mock_info, mock_exhume_date):
act_resource = ActionsResource()
dag_id = 'test_dag_id'
action = {'id': '123', 'user': '******'}
CONF = cfg.CONF
web_server_url = CONF.base.web_server
conf_value = {'action': action}
log_string = 'Created <DagRun deploy_site @ 2017-09-22 22:16:14: man'
responses.add(method='POST',
url='{}api/experimental/dags/{}/dag_runs'.format(
web_server_url, dag_id),
body=json.dumps({'message': log_string}),
status=200,
content_type='application/json')
result = act_resource.invoke_airflow_dag(dag_id, action, context)
mock_exhume_date.assert_called_with(dag_id, log_string)
assert result == '2017-09-22T22:16:14'
def start_api():
middlewares = [
AuthMiddleware(),
ContextMiddleware(),
LoggingMiddleware(),
CommonParametersMiddleware()
]
control_api = falcon.API(
request_type=ShipyardRequest, middleware=middlewares)
control_api.add_route('/versions', VersionsResource())
# v1.0 of Shipyard API
v1_0_routes = [
# API for managing region data
('/health', HealthResource()),
('/actions', ActionsResource()),
('/actions/{action_id}', ActionsIdResource()),
('/actions/{action_id}/control/{control_verb}',
ActionsControlResource()),
('/actions/{action_id}/steps/{step_id}',
ActionsStepsResource()),
('/actions/{action_id}/steps/{step_id}/logs',
ActionsStepsLogsResource()),
('/actions/{action_id}/validations/{validation_id}',
ActionsValidationsResource()),
('/configdocs', ConfigDocsStatusResource()),
('/configdocs/{collection_id}', ConfigDocsResource()),
('/commitconfigdocs', CommitConfigDocsResource()),
('/notedetails/{note_id}', NoteDetailsResource()),
('/renderedconfigdocs', RenderedConfigDocsResource()),
('/workflows', WorkflowResource()),
('/workflows/{workflow_id}', WorkflowIdResource()),
('/site_statuses', StatusResource()),
]
# Set up the 1.0 routes
route_v1_0_prefix = '/api/v1.0'
for path, res in v1_0_routes:
route = '{}{}'.format(route_v1_0_prefix, path)
LOG.info(
'Adding route: %s Handled by %s',
route,
res.__class__.__name__
)
control_api.add_route(route, res)
# Error handlers (FILO handling)
control_api.add_error_handler(Exception, default_exception_handler)
control_api.add_error_handler(AppError, AppError.handle)
# built-in error serializer
control_api.set_error_serializer(default_error_serializer)
return control_api
def test_invoke_airflow_dag_success(mock_info, mock_exhume_date):
act_resource = ActionsResource()
dag_id = 'test_dag_id'
action = 'test_action'
CONF = cfg.CONF
web_server_url = CONF.base.web_server
conf_value = {'action': action}
log_string = 'Created <DagRun deploy_site @ 2017-09-22 22:16:14: man'
responses.add(
method='GET',
url='{}admin/rest_api/api?api=trigger_dag&dag_id={}&conf={}'.format(
web_server_url, dag_id, act_resource.to_json(conf_value)),
body=json.dumps({'output': {
'stdout': log_string
}}),
status=200,
content_type='application/json')
result = act_resource.invoke_airflow_dag(dag_id, action, context)
mock_exhume_date.assert_called_with(dag_id, log_string)
assert result == '2017-09-22T22:16:14'
def test_on_post(mock_info, mock_create_action, mock_authorize, *args):
act_resource = ActionsResource()
context.policy_engine = ShipyardPolicy()
json_body = json.dumps({
'user': "******",
'req_id': "test_req_id",
'external_ctx': "test_ext_ctx",
'name': "test_name"
}).encode('utf-8')
req = create_req(context, json_body)
resp = create_resp()
act_resource.on_post(req, resp)
mock_authorize.assert_called_once_with(
'workflow_orchestrator:create_action', context)
mock_create_action.assert_called_once_with(action=json.loads(
json_body.decode('utf-8')),
context=context,
allow_intermediate_commits=None)
mock_info.assert_called_with("Id %s generated for action %s", 'test_id',
'test_name')
assert resp.status == '201 Created'
assert resp.body is not None
assert '/api/v1.0/actions/' in resp.location
def test_get_all_actions(*args):
"""
Tests the main response from get all actions
"""
action_resource = ActionsResource()
action_resource.get_all_actions_db = actions_db
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
result = action_resource.get_all_actions(verbosity=1)
assert len(result) == len(actions_db())
for action in result:
if action['name'] == 'dag_it':
assert len(action['steps']) == 1
assert action['dag_status'] == 'FAILED'
if action['name'] == 'dag2':
assert len(action['steps']) == 3
assert action['dag_status'] == 'SUCCESS'
def test_exhume_date():
act_resource = ActionsResource()
dag_id = 'test_dag_id'
log_string = 'test_log_string'
with pytest.raises(ApiError) as expected_exc:
act_resource._exhume_date(dag_id, log_string)
assert 'Unable to determine if workflow has started' in str(expected_exc)
assert ('Airflow has not responded with parseable output. Shipyard is '
'unable to determine run timestamp') in str(expected_exc)
dag_id = 'deploy_site'
log_string = 'Created <DagRun deploy_site @ 2017-09-22 22:16:14: man'
result = act_resource._exhume_date(dag_id, log_string)
assert result == datetime(2017, 9, 22, 22, 16, 14)
log_string = 'Created <DagRun deploy_site @ test'
with pytest.raises(ApiError) as expected_exc:
act_resource._exhume_date(dag_id, log_string)
assert 'Unable to determine if workflow has started' in str(expected_exc)
assert (
'Airflow has not responded with parseable output. Shipyard is unable '
'to determine run timestamp') in str(expected_exc)
def test_get_all_actions():
"""
Tests the main response from get all actions
"""
action_resource = ActionsResource()
action_resource.get_all_actions_db = actions_db
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
os.environ['DB_CONN_AIRFLOW'] = 'nothing'
os.environ['DB_CONN_SHIPYARD'] = 'nothing'
result = action_resource.get_all_actions()
assert len(result) == len(actions_db())
for action in result:
if action['name'] == 'dag_it':
assert len(action['steps']) == 1
assert action['dag_status'] == 'FAILED'
if action['name'] == 'dag2':
assert len(action['steps']) == 3
assert action['dag_status'] == 'SUCCESS'
def test_get_all_actions_notes(*args):
"""
Tests the main response from get all actions
"""
action_resource = ActionsResource()
action_resource.get_all_actions_db = actions_db
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
# inject some notes
nh.make_action_note('aaaaaa', "hello from aaaaaa1")
nh.make_action_note('aaaaaa', "hello from aaaaaa2")
nh.make_action_note('bbbbbb', "hello from bbbbbb")
result = action_resource.get_all_actions(verbosity=1)
assert len(result) == len(actions_db())
for action in result:
if action['id'] == 'aaaaaa':
assert len(action['notes']) == 2
if action['id'] == 'bbbbbb':
assert len(action['notes']) == 1
assert action['notes'][0]['note_val'] == 'hello from bbbbbb'
def test_get_committed_design_version_missing(*args):
act_resource = ActionsResource()
act_resource.configdocs_helper = ConfigdocsHelper(ShipyardRequestContext())
assert act_resource.get_committed_design_version() is None
def test_create_action_validator_error():
action_resource = ActionsResource()
action_resource.get_all_actions_db = actions_db
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
action_resource.invoke_airflow_dag = airflow_stub
action_resource.insert_action = insert_action_stub
action_resource.audit_control_command_db = audit_control_command_db
action_resource.get_committed_design_version = lambda: DESIGN_VERSION
action_resource.check_intermediate_commit_revision = (
CHECK_INTERMEDIATE_COMMIT)
# with valid input and some parameters
with mock.patch('shipyard_airflow.control.action.action_validators'
'.validate_site_action',
side_effect=ApiError(title='bad')):
with pytest.raises(ApiError) as apie:
action = action_resource.create_action(
action={'name': 'deploy_site',
'parameters': {
'a': 'aaa'
}},
context=context,
allow_intermediate_commits=False)
assert action['timestamp']
assert action['id']
assert len(action['id']) == 26
assert action['dag_execution_date'] == '2017-09-06 14:10:08.528402'
assert action['dag_status'] == 'SCHEDULED'
assert action['committed_rev_id'] == 1
assert apie.value.title == 'bad'
def test_create_action():
action_resource = ActionsResource()
action_resource.get_all_actions_db = actions_db
action_resource.get_all_dag_runs_db = dag_runs_db
action_resource.get_all_tasks_db = tasks_db
action_resource.invoke_airflow_dag = airflow_stub
action_resource.insert_action = insert_action_stub
action_resource.audit_control_command_db = audit_control_command_db
action_resource.get_committed_design_version = lambda: DESIGN_VERSION
action_resource.check_intermediate_commit_revision = (
CHECK_INTERMEDIATE_COMMIT)
# with invalid input. fail.
with mock.patch('shipyard_airflow.control.action.action_validators'
'.validate_site_action') as validator:
try:
action = action_resource.create_action(
action={'name': 'broken',
'parameters': {
'a': 'aaa'
}},
context=context,
allow_intermediate_commits=False)
assert False, 'Should throw an ApiError'
except ApiError:
# expected
pass
assert not validator.called
# with valid input and some parameters
with mock.patch('shipyard_airflow.control.action.action_validators'
'.validate_site_action') as validator:
try:
action = action_resource.create_action(
action={'name': 'deploy_site',
'parameters': {
'a': 'aaa'
}},
context=context,
allow_intermediate_commits=False)
assert action['timestamp']
assert action['id']
assert len(action['id']) == 26
assert action['dag_execution_date'] == '2017-09-06 14:10:08.528402'
assert action['dag_status'] == 'SCHEDULED'
assert action['committed_rev_id'] == 1
except ApiError:
assert False, 'Should not raise an ApiError'
validator.assert_called_once_with(action)
# with valid input and no parameters
with mock.patch('shipyard_airflow.control.action.action_validators'
'.validate_site_action') as validator:
try:
action = action_resource.create_action(
action={'name': 'deploy_site'},
context=context,
allow_intermediate_commits=False)
assert action['timestamp']
assert action['id']
assert len(action['id']) == 26
assert action['dag_execution_date'] == '2017-09-06 14:10:08.528402'
assert action['dag_status'] == 'SCHEDULED'
assert action['committed_rev_id'] == 1
except ApiError:
assert False, 'Should not raise an ApiError'
validator.assert_called_once_with(action)
def test_audit_control_command_db(mock_insert_action_audit):
act_resource = ActionsResource()
action_audit = 'test_action_audit'
act_resource.audit_control_command_db(action_audit)
mock_insert_action_audit.assert_called_with(action_audit)
def test_insert_action(mock_insert_action):
act_resource = ActionsResource()
action = 'test_action'
act_resource.insert_action(action)
mock_insert_action.assert_called_with(action)
def test_get_all_tasks_db(mock_get_all_tasks):
act_resource = ActionsResource()
act_resource.get_all_tasks_db()
assert mock_get_all_tasks.called
def test_get_all_dag_runs_db(mock_get_all_dag_runs):
act_resource = ActionsResource()
act_resource.get_all_dag_runs_db()
assert mock_get_all_dag_runs.called
def test_get_all_actions_db(mock_get_all_submitted_actions):
act_resource = ActionsResource()
act_resource.get_all_actions_db()
assert mock_get_all_submitted_actions.called