def test_commit_configdocs(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_revision = lambda x: {
'status': 'Success'
}
helper.get_revision_id = lambda x: 1
commit_resp = ccdr.commit_configdocs(helper, False, False)
mock_method.assert_called_once_with('committed')
assert commit_resp['status'] == 'Success'
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_revision = (
lambda x: {
'status': 'Failure',
'code': '400 Bad Request',
'message': 'this is a mock response'
})
helper.get_revision_id = lambda x: 1
commit_resp = ccdr.commit_configdocs(helper, False, False)
assert '400' in commit_resp['code']
assert commit_resp['message'] is not None
assert commit_resp['status'] == 'Failure'
def test__get_revision_dict_commit_and_buff():
"""
Tests the processing of revision dict response from deckhand
with a committed and a buffer revision
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = lambda: yaml.load("""
---
- id: 1
url: https://deckhand/api/v1.0/revisions/1
createdAt: 2017-07-14T21:23Z
buckets: [mop]
tags: [a, b, committed]
validationPolicies:
site-deploy-validation:
status: failed
- id: 2
url: https://deckhand/api/v1.0/revisions/2
createdAt: 2017-07-16T01:15Z
buckets: [flop, mop]
tags: [b]
validationPolicies:
site-deploy-validation:
status: succeeded
...
""")
rev_dict = helper._get_revision_dict()
committed = rev_dict.get(configdocs_helper.COMMITTED)
buffer = rev_dict.get(configdocs_helper.BUFFER)
latest = rev_dict.get(configdocs_helper.LATEST)
count = rev_dict.get(configdocs_helper.REVISION_COUNT)
assert committed.get('id') == 1
assert buffer.get('id') == 2
assert latest.get('id') == 2
assert count == 2
def test_get_revision_dict_last_site_action_and_successful_site_action():
"""
Tests the processing of revision dict response from deckhand
for last_site_action and successful_site_action revision
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = lambda: yaml.load("""
---
- id: 1
url: https://deckhand/api/v1.0/revisions/1
createdAt: 2018-04-30T21:23Z
buckets: [mop]
tags: [committed, site-action-success]
validationPolicies:
site-deploy-validation:
status: succeeded
- id: 2
url: https://deckhand/api/v1.0/revisions/2
createdAt: 2018-04-30T23:35Z
buckets: [flop, mop]
tags: [committed, site-action-failure]
validationPolicies:
site-deploy-validation:
status: succeeded
...
""")
rev_dict = helper._get_revision_dict()
successful_site_action = rev_dict.get(
configdocs_helper.SUCCESSFUL_SITE_ACTION)
last_site_action = rev_dict.get(configdocs_helper.LAST_SITE_ACTION)
assert successful_site_action.get('id') == 1
assert last_site_action.get('id') == 2
def test_get_collection_docs():
"""
Returns the representation of the yaml docs from deckhand
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_docs_from_revision = (
lambda revision_id, bucket_id: "{'yaml': 'yaml'}")
helper._get_revision_dict = lambda: REV_EMPTY_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_EMPTY_DICT)
with pytest.raises(ApiError):
helper.get_collection_docs(configdocs_helper.BUFFER, 'mop')
with pytest.raises(ApiError):
helper.get_collection_docs(configdocs_helper.COMMITTED, 'mop')
helper._get_revision_dict = lambda: REV_COMMIT_AND_BUFFER_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_COMMIT_AND_BUFFER_DICT)
yaml_str = helper.get_collection_docs(configdocs_helper.BUFFER, 'mop')
assert len(yaml_str) == 16
yaml_str = helper.get_collection_docs(configdocs_helper.COMMITTED, 'mop')
assert len(yaml_str) == 16
def test__get_revision_dict_errs():
"""
tests getting a revision dictionary method when the deckhand
client has raised an exception
"""
def _raise_dre():
raise DeckhandResponseError(status_code=9000,
response_message='This is bogus')
def _raise_nree():
raise NoRevisionsExistError()
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = _raise_dre
with pytest.raises(AppError):
helper._get_revision_dict()
helper.deckhand.get_revision_list = _raise_nree
rev_dict = helper._get_revision_dict()
committed = rev_dict.get(configdocs_helper.COMMITTED)
buffer = rev_dict.get(configdocs_helper.BUFFER)
latest = rev_dict.get(configdocs_helper.LATEST)
count = rev_dict.get(configdocs_helper.REVISION_COUNT)
assert committed is None
assert buffer is None
assert latest is None
assert count == 0
def test__get_deckhand_validation_errors_empty_results():
"""
Tets the functionality of processing a response from deckhand
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand._get_base_validation_resp = (
lambda revision_id: FK_VAL_BASE_RESP_EMPTY)
assert len(helper._get_deckhand_validation_errors(5)) == 0
def create_action(self, action, context, allow_intermediate_commits=False):
action_mappings = _action_mappings()
# use uuid assigned for this request as the id of the action.
action['id'] = ulid.ulid()
# the invoking user
action['user'] = context.user
# add current timestamp (UTC) to the action.
action['timestamp'] = str(datetime.utcnow())
# validate that action is supported.
LOG.info("Attempting action: %s", action['name'])
if action['name'] not in action_mappings:
raise ApiError(
title='Unable to start action',
description='Unsupported Action: {}'.format(action['name']))
dag = action_mappings.get(action['name'])['dag']
action['dag_id'] = dag
# Set up configdocs_helper
self.configdocs_helper = ConfigdocsHelper(context)
# Retrieve last committed design revision
action['committed_rev_id'] = self.get_committed_design_version()
# Check for intermediate commit
self.check_intermediate_commit_revision(allow_intermediate_commits)
# populate action parameters if they are not set
if 'parameters' not in action:
action['parameters'] = {}
# validate if there is any validation to do
for validator in action_mappings.get(action['name'])['validators']:
# validators will raise ApiError if they are not validated.
validator(action)
# invoke airflow, get the dag's date
dag_execution_date = self.invoke_airflow_dag(
dag_id=dag, action=action, context=context)
# set values on the action
action['dag_execution_date'] = dag_execution_date
action['dag_status'] = 'SCHEDULED'
# context_marker is the uuid from the request context
action['context_marker'] = context.request_id
# insert the action into the shipyard db
self.insert_action(action=action)
self.audit_control_command_db({
'id': ulid.ulid(),
'action_id': action['id'],
'command': 'invoke',
'user': context.user
})
return action
def test_get_validations_for_revision(*args):
"""
Tests the functionality of the get_validations_for_revision method
"""
helper = ConfigdocsHelper(CTX)
val_status = helper.get_validations_for_revision(3)
err_count = val_status['details']['errorCount']
err_list_count = len(val_status['details']['messageList'])
assert err_list_count == 6
assert val_status['details']['errorCount'] == 4
def test_tag_buffer():
"""
Tests that the tag buffer method attempts to tag the right version
"""
with patch.object(ConfigdocsHelper, 'tag_revision') as mock_method:
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
helper.tag_buffer('artful')
mock_method.assert_called_once_with(5, 'artful')
def test_add_collection():
"""
Tests the adding of a collection to deckhand - primarily
error handling
"""
with patch.object(DeckhandClient, 'put_bucket') as mock_method:
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
assert helper.add_collection('mop', 'yaml:yaml') == 5
mock_method.assert_called_once_with('mop', 'yaml:yaml')
def test__get_deckhand_validations_empty_errors():
"""
Tets the functionality of processing a response from deckhand
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand._get_base_validation_resp = (
lambda revision_id: FK_VAL_BASE_RESP)
helper.deckhand._get_subset_validation_response = (
lambda reivsion_id, subset_name: FK_VAL_SUBSET_RESP)
helper.deckhand._get_entry_validation_response = (
lambda reivsion_id, subset_name, entry_id: FK_VAL_ENTRY_RESP_EMPTY)
assert len(helper._get_deckhand_validation_errors(5)) == 0
def test_get_validations_for_revision_dh_render(dh_client):
"""
Tests the functionality of the get_validations_for_revision method
"""
helper = ConfigdocsHelper(CTX)
hold_ve = configdocs_helper._get_validation_endpoints
helper._get_deckhand_validation_errors = lambda revision_id: []
val_status = helper.get_validations_for_revision(3)
err_count = val_status['details']['errorCount']
err_list_count = len(val_status['details']['messageList'])
assert err_count == err_list_count
assert val_status['details']['errorCount'] == 1
assert val_status['details']['messageList'][0]['message'] == 'broken!'
def test_commit_configdocs_buffer_err(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
with pytest.raises(ApiError):
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: True
helper.get_validations_for_revision = lambda x: {
'status': 'Success'
}
ccdr.commit_configdocs(helper, False, False)
def test__get_revision_dict_empty():
"""
Tests the processing of revision dict response from deckhand
where the response is an empty list
"""
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = lambda: []
rev_dict = helper._get_revision_dict()
committed = rev_dict.get(configdocs_helper.COMMITTED)
buffer = rev_dict.get(configdocs_helper.BUFFER)
latest = rev_dict.get(configdocs_helper.LATEST)
count = rev_dict.get(configdocs_helper.REVISION_COUNT)
assert committed is None
assert buffer is None
assert latest is None
assert count == 0
def test_is_buffer_emtpy():
"""
Test the method to check if the configdocs buffer is empty
"""
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
assert not helper.is_buffer_empty()
helper._get_revision_dict = lambda: REV_BUFF_EMPTY_DICT
assert helper.is_buffer_empty()
helper._get_revision_dict = lambda: REV_NO_COMMIT_DICT
assert not helper.is_buffer_empty()
helper._get_revision_dict = lambda: REV_EMPTY_DICT
assert helper.is_buffer_empty()
def test_construct_configdocs_helper():
"""
Creates a configdoc helper, tests that the context
is passed to the sub-helper
"""
helper = ConfigdocsHelper(CTX)
assert helper.ctx == CTX
def on_post(self, req, resp, collection_id):
"""
Ingests a collection of documents
"""
# Determine if this request is clearing the collection's contents.
empty_coll = req.get_param_as_bool('empty-collection') or False
if empty_coll:
document_data = ""
LOG.debug("Collection %s is being emptied", collection_id)
else:
# Note, a newline in a prior header can trigger subsequent
# headers to be "missing" (and hence cause this code to think
# that the content length is missing)
content_length = self.validate_content_length(req.content_length)
document_data = req.stream.read(content_length)
buffer_mode = req.get_param('buffermode')
helper = ConfigdocsHelper(req.context)
validations = self.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data,
buffer_mode_param=buffer_mode,
empty_collection=empty_coll)
resp.status = falcon.HTTP_201
if validations and validations['status'] == 'Success':
validations['code'] = resp.status
resp.location = '/api/v1.0/configdocs/{}'.format(collection_id)
resp.body = self.to_json(validations)
def on_post(self, req, resp, collection_id):
"""
Ingests a collection of documents
"""
content_length = req.content_length or 0
if (content_length == 0):
raise ApiError(
title=('Content-Length is a required header'),
description='Content Length is 0 or not specified',
status=falcon.HTTP_400,
error_list=[{
'message':
('The Content-Length specified is 0 or not set. Check '
'that a valid payload is included with this request '
'and that your client is properly including a '
'Content-Length header. Note that a newline character '
'in a prior header can trigger subsequent headers to '
'be ignored and trigger this failure.')
}],
retry=False,
)
document_data = req.stream.read(content_length)
buffer_mode = req.get_param('buffermode')
helper = ConfigdocsHelper(req.context)
validations = self.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data,
buffer_mode_param=buffer_mode)
resp.location = '/api/v1.0/configdocs/{}'.format(collection_id)
resp.body = self.to_json(validations)
resp.status = falcon.HTTP_201
def test_get_committed_design_version_missing(*args):
with pytest.raises(ApiError) as apie:
act_resource = ActionsResource()
act_resource.configdocs_helper = ConfigdocsHelper(
ShipyardRequestContext())
act_resource.get_committed_design_version()
assert apie.value.status == falcon.HTTP_404
assert apie.value.title == ('Unable to locate any committed revision in '
'Deckhand')
def test_parse_received_doc_data():
helper = ConfigdocsHelper(CTX)
yaml = """
---
document1:
- a
- b
- c
---
document2:
- 1
- 2
- 3
...
"""
parsed = helper.parse_received_doc_data(yaml)
assert type(parsed) == list
assert len(parsed) == 2
def test_post_collection_not_valid_for_buffer(self):
"""
Tests the post collection method of the ConfigdocsResource
"""
helper = None
collection_id = 'trees'
document_data = 'lots of info'
with pytest.raises(ApiError) as apie:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
# not valid for bucket
helper.get_deckhand_validation_status = (
lambda a: configdocs_helper._format_validations_to_status([], 0
))
cdr.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data)
assert apie.value.status == '409 Conflict'
def test_commit_configdocs_dryrun(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_revision = lambda x: {
'status': 'Success'
}
helper.get_revision_id = lambda x: 1
commit_resp = ccdr.commit_configdocs(helper, False, True)
assert '200' in commit_resp['code']
assert commit_resp['message'] == 'DRYRUN'
assert commit_resp['status'] == 'Success'
def test_get_collection(self):
helper = None
with patch.object(ConfigdocsHelper,
'get_collection_docs') as mock_method:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
cdr.get_collection(helper, 'apples')
mock_method.assert_called_once_with('buffer', 'apples')
def test_post_collection(self):
"""
Tests the post collection method of the ConfigdocsResource
"""
helper = None
collection_id = 'trees'
document_data = 'lots of info'
with patch.object(ConfigdocsHelper, 'add_collection') as mock_method:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
helper.is_buffer_valid_for_bucket = lambda a, b: True
helper.get_deckhand_validation_status = (
lambda a: configdocs_helper._format_validations_to_status([], 0
))
cdr.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data)
mock_method.assert_called_once_with(collection_id, document_data)
def test_commit_configdocs_force(self):
"""
Tests the CommitConfigDocsResource method commit_configdocs
"""
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_revision = lambda x: {
'status': 'Failure'
}
helper.get_revision_id = lambda x: 1
commit_resp = ccdr.commit_configdocs(helper, True, False)
mock_method.assert_called_once_with('committed')
assert '200' in commit_resp['code']
assert 'FORCED' in commit_resp['message']
assert commit_resp['status'] == 'Failure'
def on_get(self, req, resp):
"""
Returns the whole set of rendered documents
"""
version = (req.params.get('version') or 'buffer')
self._validate_version_parameter(version)
helper = ConfigdocsHelper(req.context)
resp.body = self.get_rendered_configdocs(helper=helper,
version=version)
resp.append_header('Content-Type', 'application/x-yaml')
resp.status = falcon.HTTP_200
def test_post_collection_not_added(self):
"""
Tests the post collection method of the ConfigdocsResource
"""
helper = None
collection_id = 'trees'
document_data = 'lots of info'
with patch.object(ConfigdocsHelper, 'add_collection') as mock_method:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper(CTX)
helper.get_deckhand_validation_status = (
lambda a: configdocs_helper._format_validations_to_status([], 0
))
with pytest.raises(ApiError) as apie:
cdr.post_collection(helper=helper,
collection_id=collection_id,
document_data=document_data)
assert apie.value.status == '400 Bad Request'
mock_method.assert_called_once_with(collection_id, document_data)
def test_get_rendered_configdocs():
"""
Tests the RenderedConfigDocsResource method get_rendered_configdocs
"""
rcdr = RenderedConfigDocsResource()
with patch.object(ConfigdocsHelper,
'get_rendered_configdocs') as mock_method:
helper = ConfigdocsHelper(CTX)
rcdr.get_rendered_configdocs(helper, version='buffer')
mock_method.assert_called_once_with('buffer')
def on_post(self, req, resp):
"""
Get validations from all Airship components
Functionality does not exist yet
"""
# force and dryrun query parameter is False unless explicitly true
force = req.get_param_as_bool(name='force') or False
dryrun = req.get_param_as_bool(name='dryrun') or False
helper = ConfigdocsHelper(req.context)
validations = self.commit_configdocs(helper, force, dryrun)
resp.body = self.to_json(validations)
resp.status = validations.get('code', falcon.HTTP_200)
def test_add_messages_to_validation_status():
helper = ConfigdocsHelper(CTX)
status = {
"kind": "Status",
"apiVersion": "v1.0",
"metadata": {},
"status": "Success",
"message": "Validations succeeded",
"reason": "Validation",
"details": {
"errorCount": 0,
"messageList": [],
},
"code": falcon.HTTP_200
}
info_messages = ['message 1', 'message 2']
warn_messages = ['message 3']
error_messages = ['message 4', 'message 5']
add_messages_to_validation_status(status, info_messages, 'info')
assert status['details']['errorCount'] == 0
assert len(status['details']['messageList']) == 2
assert type(status['details']['messageList'][0]) == dict
assert status['details']['messageList'][0]['code'] == falcon.HTTP_200
assert status['details']['messageList'][0]['message'] == 'message 1'
assert status['details']['messageList'][0]['status'] == 'Info'
assert status['details']['messageList'][0]['level'] == 'info'
assert status["status"] == "Success"
assert status["message"] == "Validations succeeded"
assert status["code"] == falcon.HTTP_200
add_messages_to_validation_status(status, warn_messages, 'warning')
assert status['details']['errorCount'] == 0
assert len(status['details']['messageList']) == 3
assert status['details']['messageList'][2]['code'] == falcon.HTTP_200
assert status['details']['messageList'][2]['message'] == 'message 3'
assert status['details']['messageList'][2]['status'] == 'Warning'
assert status['details']['messageList'][2]['level'] == 'warning'
assert status["status"] == "Success"
assert status["message"] == "Validations succeeded"
assert status["code"] == falcon.HTTP_200
add_messages_to_validation_status(status, error_messages, 'error')
assert status['details']['errorCount'] == 2
assert len(status['details']['messageList']) == 5
assert status['details']['messageList'][3]['code'] == falcon.HTTP_400
assert status['details']['messageList'][3]['message'] == 'message 4'
assert status['details']['messageList'][3]['status'] == 'Error'
assert status['details']['messageList'][3]['level'] == 'error'
assert status["status"] == "Failure"
assert status["message"] == "Validations failed"
assert status["code"] == falcon.HTTP_400
