def test_load_yaml_raises_error_if_wrong_format(self, yamlMock, osMock):
yamlMock.safe_load.side_effect = YAMLError("error")
with self.assertRaises(YAMLError):
self.inv._load_yaml(self.source_data_path)
self.assertEqual(
str(self.logging.getLogger.return_value.error.mock_calls),
str([call(YAMLError("error"))]),
)
def yaml_load(obj):
'''Safely loads the YAML-formatted object `obj` into a dict and returns
that dict. Note that being YAML a superset of json, all properly
json-formatted strings are also correctly loaded and the quote character
' is also allowed (in pure json, only " is allowed).
:param obj: (dict, stream, string denoting an existing file path, or
string denoting the file content in YAML syntax): If stream (i.e.,
an object with the `read` attribute), uses it for reading and parsing
its content into dict. If dict, this method is no-op and the dict is
returned, if string denoting an existing file, a stream is opened
from the file and processed as explained above (the stream will be
closed in this case). If string, the string is treated as YAML
content and parsed: in this case, the output must be a dict otherwise
a YAMLError is thrown
:raises: YAMLError
'''
if isinstance(obj, dict):
return obj
close_stream = False
if isinstance(obj, str):
close_stream = True
if isfile(obj): # file input
stream = open(obj, 'r')
else:
stream = StringIO(obj) # YAML content input
elif not hasattr(obj, 'read'):
# raise a general message meaningful for a Rest framework:
raise YAMLError('Invalid input, expected data as string in YAML or '
'JSON syntax, found %s' % str(obj.__class__.__name__))
else:
stream = obj
try:
ret = safe_load(stream)
# for some weird reason, in case of a string ret is the string itself,
# and no error is raised. Let's do it here:
if not isinstance(ret, dict):
if isinstance(obj, (str, bytes)):
raise YAMLError('The given string input is neither a valid '
'YAML content nor the path of an existing '
'YAML file')
raise YAMLError('Unable to load input (%s) as YAML' %
(obj.__class__.__name__))
return ret
except YAMLError as _:
raise
finally:
if close_stream:
stream.close()
def __load_config(config_path: str):
"""Loads config_path with pyyaml
Args:
config_path: Path to config file
"""
try:
with open(config_path, "r") as config_file:
return yaml.load(stream=config_file, Loader=Loader)
except FileNotFoundError:
raise FileNotFoundError(f"No such File: '{config_path}'")
except YAMLError:
raise YAMLError("Error Parsing Config File")
def _constructor_env_variables(loader, node):
"""
Extracts the environment variable from the node's value.
:param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml
:return: value of the environment variable
"""
raw_value = loader.construct_scalar(node)
new_value = os.environ.get(raw_value)
if new_value is None:
msg = "Cannot construct value from {node}: {value}".format(
node=node, value=new_value)
raise YAMLError(msg)
return new_value
def yaml_load_clean(data):
"""Read YAML.
Handles dependencies.
Raises:
YAMLError
Returns:
dict: Data.
"""
from yaml import load, YAMLError
try:
return load(read_contents(data))
except YAMLError:
raise YAMLError('YAMLError: An unexpected error occurred when '
'attempting to read supplied YAML.')
def _constructor_envfile_variables(loader, node):
"""
Extracts the environment variable from the node's value.
:param yaml.Loader loader: the yaml loader
:param node: the current node in the yaml
:return: value read from file pointed to by environment variable
"""
raw_value = loader.construct_scalar(node)
filepath = os.environ.get(raw_value)
try:
with open(filepath, "r") as fd:
new_value = fd.read()
except (TypeError, IOError) as e:
msg = "Cannot construct value from {node}: {path}".format(
node=node, path=filepath)
raise YAMLError(msg) from e
else:
return new_value
def __init__(self, tasks=None):
if tasks is None:
self.tasks = []
elif isinstance(tasks, str):
try:
self.tasks = [Task(title=x[0], state=x[1], estimate=x[2]) for x in get_dataset(tasks)]
except YAMLError:
raise YAMLError("Error in format")
except ValueError as e:
raise ValueError("Error in data")
except OSError as e:
raise OSError("Error in file")
except Exception as e:
raise Exception("Unknown")
elif isinstance(tasks, list):
if not all(isinstance(n, Task) for n in tasks):
raise ValueError('Bad type of tasks')
self.tasks = tasks
else:
raise ValueError("Not supported object provided")
def _raise_yamlerror(*args):
raise YAMLError()
def test_read_yaml_err(self, mock_obj):
yaml_fn = self._fu.qualified_path(self.path, self.yaml)
self.create_text_file(yaml_fn)
mock_obj.side_effect = YAMLError('mock error')
actual = self._fu.read_yaml(yamlFile=yaml_fn)
self.assertIsNone(actual)
class TestWatcher(object):
@pytest.fixture
def spec_factory(self):
with mock.patch(
"fiaas_deploy_daemon.specs.factory.SpecFactory") as mockk:
yield mockk
@pytest.fixture
def deploy_queue(self):
return Queue()
@pytest.fixture
def watcher(self):
return mock.create_autospec(spec=Watcher, spec_set=True, instance=True)
@pytest.fixture
def lifecycle(self):
return mock.create_autospec(spec=Lifecycle,
spec_set=True,
instance=True)
@pytest.fixture
def crd_watcher(self, spec_factory, deploy_queue, watcher, lifecycle):
crd_watcher = CrdWatcher(spec_factory, deploy_queue, Configuration([]),
lifecycle)
crd_watcher._watcher = watcher
return crd_watcher
@pytest.fixture(autouse=True)
def status_get(self):
with mock.patch(
"fiaas_deploy_daemon.crd.status.FiaasApplicationStatus.get",
spec_set=True) as m:
m.side_effect = NotFound
yield m
def test_creates_custom_resource_definition_if_not_exists_when_watching_it(
self, get, post, crd_watcher, watcher):
get.side_effect = NotFound("Something")
watcher.watch.side_effect = NotFound("Something")
expected_application = {
'metadata': {
'namespace': 'default',
'name': 'applications.fiaas.schibsted.io',
'ownerReferences': [],
'finalizers': [],
},
'spec': {
'version': 'v1',
'group': 'fiaas.schibsted.io',
'names': {
'shortNames': ['app', 'fa'],
'kind': 'Application',
'plural': 'applications'
}
}
}
expected_status = {
'metadata': {
'namespace': 'default',
'name': 'application-statuses.fiaas.schibsted.io',
'ownerReferences': [],
'finalizers': [],
},
'spec': {
'version': 'v1',
'group': 'fiaas.schibsted.io',
'names': {
'shortNames': ['status', 'appstatus', 'fs'],
'kind': 'ApplicationStatus',
'plural': 'application-statuses'
}
}
}
def make_response(data):
mock_response = mock.create_autospec(Response)
mock_response.json.return_value = data
return mock_response
post.side_effect = [
make_response(expected_application),
make_response(expected_status)
]
crd_watcher._watch(None)
calls = [
mock.call(
"/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/",
expected_application),
mock.call(
"/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/",
expected_status)
]
assert post.call_args_list == calls
def test_is_able_to_watch_custom_resource_definition(
self, crd_watcher, deploy_queue, watcher):
watcher.watch.return_value = [WatchEvent(ADD_EVENT, FiaasApplication)]
assert deploy_queue.qsize() == 0
crd_watcher._watch(None)
assert deploy_queue.qsize() == 1
@pytest.mark.parametrize(
"event,deployer_event_type,annotations,repository", [
(ADD_EVENT, "UPDATE", None, None),
(ADD_EVENT, "UPDATE", {
"deployment": {
"fiaas/source-repository": "xyz"
}
}, "xyz"),
(MODIFIED_EVENT, "UPDATE", None, None),
(MODIFIED_EVENT, "UPDATE", {
"deployment": {
"fiaas/source-repository": "xyz"
}
}, "xyz"),
(DELETED_EVENT, "DELETE", None, None),
])
def test_deploy(self, crd_watcher, deploy_queue, spec_factory, watcher,
app_spec, event, deployer_event_type, lifecycle,
annotations, repository):
event["object"]["spec"]["config"]["annotations"] = annotations
watcher.watch.return_value = [WatchEvent(event, FiaasApplication)]
spec = event["object"]["spec"]
app_name = spec["application"]
namespace = event["object"]["metadata"]["namespace"]
deployment_id = (
event["object"]["metadata"]["labels"]["fiaas/deployment_id"]
if deployer_event_type != "DELETE" else "deletion")
app_spec = app_spec._replace(name=app_name,
namespace=namespace,
deployment_id=deployment_id)
spec_factory.return_value = app_spec
lifecycle_subject = Subject(app_name, namespace, deployment_id,
repository, app_spec.labels.status,
app_spec.annotations.status)
lifecycle.initiate.return_value = lifecycle_subject
crd_watcher._watch(None)
if event in [ADD_EVENT, MODIFIED_EVENT]:
lifecycle.initiate.assert_called_once_with(
app_name=event["object"]["spec"]["application"],
namespace=event["object"]["metadata"]["namespace"],
deployment_id='deployment_id',
repository=repository,
labels=None,
annotations=None)
app_config = spec["config"]
additional_labels = AdditionalLabelsOrAnnotations()
additional_annotations = AdditionalLabelsOrAnnotations()
spec_factory.assert_called_once_with(
name=app_name,
image=spec["image"],
app_config=app_config,
teams=[],
tags=[],
deployment_id=deployment_id,
namespace=namespace,
additional_labels=additional_labels,
additional_annotations=additional_annotations)
assert deploy_queue.qsize() == 1
deployer_event = deploy_queue.get_nowait()
if event in [ADD_EVENT, MODIFIED_EVENT]:
assert deployer_event == DeployerEvent(deployer_event_type,
app_spec, lifecycle_subject)
else:
assert deployer_event == DeployerEvent(deployer_event_type,
app_spec, None)
assert deploy_queue.empty()
@pytest.mark.parametrize("namespace", [None, "default"])
def test_watch_namespace(self, crd_watcher, watcher, namespace):
crd_watcher._watch(namespace)
watcher.watch.assert_called_once_with(namespace=namespace)
@pytest.mark.parametrize(
"event,deployer_event_type,error,annotations,repository", [
(ADD_EVENT, "UPDATE", YAMLError("invalid yaml"), {}, None),
(ADD_EVENT, "UPDATE", InvalidConfiguration("invalid config"), {},
None),
(MODIFIED_EVENT, "UPDATE", YAMLError("invalid yaml"), {}, None),
(MODIFIED_EVENT, "UPDATE", InvalidConfiguration("invalid config"),
{}, None),
(MODIFIED_EVENT, "UPDATE", InvalidConfiguration("invalid config"),
{
"deployment": {
"fiaas/source-repository": "xyz"
}
}, "xyz"),
])
def test_deploy_reports_failure_on_exception(
self, crd_watcher, deploy_queue, spec_factory, watcher, event,
deployer_event_type, error, lifecycle, annotations, repository):
event["object"]["metadata"]["annotations"] = annotations
watcher.watch.return_value = [WatchEvent(event, FiaasApplication)]
spec_factory.side_effect = error
lifecycle_subject = Subject(
app_name=event["object"]["spec"]["application"],
namespace=event["object"]["metadata"]["namespace"],
deployment_id='deployment_id',
repository=repository,
labels=None,
annotations=None)
lifecycle.initiate.return_value = lifecycle_subject
crd_watcher._watch(None)
lifecycle.failed.assert_called_once_with(lifecycle_subject)
assert deploy_queue.empty()
@pytest.mark.parametrize("result, count", (
("SUCCESS", 0),
("FAILED", 1),
("RUNNING", 1),
("INITIATED", 1),
("ANY_OTHER_VALUE_THAN_SUCCESS", 1),
))
def test_deploy_based_on_status_result(self, crd_watcher, deploy_queue,
watcher, status_get, result, count):
watcher.watch.return_value = [WatchEvent(ADD_EVENT, FiaasApplication)]
status_get.side_effect = lambda *args, **kwargs: mock.DEFAULT # disable default behavior of raising NotFound
status_get.return_value = FiaasApplicationStatus(new=False,
result=result)
assert deploy_queue.qsize() == 0
crd_watcher._watch(None)
assert deploy_queue.qsize() == count
class TestTprWatcher(object):
@pytest.fixture
def spec_factory(self):
with mock.patch(
"fiaas_deploy_daemon.specs.factory.SpecFactory") as mockk:
yield mockk
@pytest.fixture
def deploy_queue(self):
return Queue()
@pytest.fixture
def watcher(self):
return mock.create_autospec(spec=Watcher, spec_set=True, instance=True)
@pytest.fixture
def lifecycle(self):
return mock.create_autospec(spec=Lifecycle,
spec_set=True,
instance=True)
@pytest.fixture
def tpr_watcher(self, spec_factory, deploy_queue, watcher, lifecycle):
mock_watcher = TprWatcher(spec_factory, deploy_queue,
Configuration([]), lifecycle)
mock_watcher._watcher = watcher
return mock_watcher
def test_creates_third_party_resource_if_not_exists_when_watching_it(
self, get, post, tpr_watcher, watcher):
get.side_effect = NotFound("Something")
watcher.watch.side_effect = NotFound("Something")
expected_application = {
'metadata': {
'namespace': 'default',
'name': 'paasbeta-application.schibsted.io',
'ownerReferences': [],
'finalizers': [],
},
'description': 'A paas application definition',
'versions': [{
'name': 'v1beta'
}]
}
expected_status = {
'metadata': {
'namespace': 'default',
'name': 'paasbeta-status.schibsted.io',
'ownerReferences': [],
'finalizers': [],
},
'description': 'A paas application status',
'versions': [{
'name': 'v1beta'
}]
}
def make_response(data):
mock_response = mock.create_autospec(Response)
mock_response.json.return_value = data
return mock_response
post.side_effect = [
make_response(expected_application),
make_response(expected_status)
]
tpr_watcher._watch(None)
calls = [
mock.call("/apis/extensions/v1beta1/thirdpartyresources/",
expected_application),
mock.call("/apis/extensions/v1beta1/thirdpartyresources/",
expected_status)
]
assert post.call_args_list == calls
def test_is_able_to_watch_third_party_resource(self, tpr_watcher,
deploy_queue, watcher):
watcher.watch.return_value = [
WatchEvent(ADD_EVENT, PaasbetaApplication)
]
assert deploy_queue.qsize() == 0
tpr_watcher._watch(None)
assert deploy_queue.qsize() == 1
@pytest.mark.parametrize(
"event,deployer_event_type,annotations,repository", [
(ADD_EVENT, "UPDATE", None, None),
(ADD_EVENT, "UPDATE", {
"deployment": {
"fiaas/source-repository": "xyz"
}
}, "xyz"),
(MODIFIED_EVENT, "UPDATE", None, None),
(MODIFIED_EVENT, "UPDATE", {
"deployment": {
"fiaas/source-repository": "xyz"
}
}, "xyz"),
(DELETED_EVENT, "DELETE", None, None),
])
def test_deploy(self, tpr_watcher, deploy_queue, spec_factory, watcher,
app_spec, event, deployer_event_type, lifecycle,
annotations, repository):
event["object"]["metadata"]["annotations"] = annotations
watcher.watch.return_value = [WatchEvent(event, PaasbetaApplication)]
spec = event["object"]["spec"]
app_name = spec["application"]
namespace = event["object"]["metadata"]["namespace"]
deployment_id = (
event["object"]["metadata"]["labels"]["fiaas/deployment_id"]
if deployer_event_type != "DELETE" else "deletion")
app_spec = app_spec._replace(name=app_name,
namespace=namespace,
deployment_id=deployment_id)
spec_factory.return_value = app_spec
tpr_watcher._watch(None)
if event in [ADD_EVENT, MODIFIED_EVENT]:
lifecycle.initiate.assert_called_once_with(
app_name=event["object"]["spec"]["application"],
namespace=event["object"]["metadata"]["namespace"],
deployment_id='deployment_id',
repository=repository)
app_config = spec["config"]
spec_factory.assert_called_once_with(name=app_name,
image=spec["image"],
app_config=app_config,
teams=[],
tags=[],
deployment_id=deployment_id,
namespace=namespace)
assert deploy_queue.qsize() == 1
deployer_event = deploy_queue.get_nowait()
assert deployer_event == DeployerEvent(deployer_event_type, app_spec)
assert deploy_queue.empty()
@pytest.mark.parametrize("namespace", [None, "default"])
def test_watch_namespace(self, tpr_watcher, watcher, namespace):
tpr_watcher._watch(namespace)
watcher.watch.assert_called_once_with(namespace=namespace)
@pytest.mark.parametrize(
"event,deployer_event_type,error,annotations,repository", [
(ADD_EVENT, "UPDATE", YAMLError("invalid yaml"), None, None),
(ADD_EVENT, "UPDATE", InvalidConfiguration("invalid config"), None,
None),
(MODIFIED_EVENT, "UPDATE", YAMLError("invalid yaml"), None, None),
(MODIFIED_EVENT, "UPDATE", InvalidConfiguration("invalid config"),
None, None),
(MODIFIED_EVENT, "UPDATE", InvalidConfiguration("invalid config"),
{
"deployment": {
"fiaas/source-repository": "xyz"
}
}, "xyz"),
])
def test_deploy_reports_failure_on_exception(
self, tpr_watcher, deploy_queue, spec_factory, watcher, event,
deployer_event_type, error, lifecycle, annotations, repository):
event["object"]["metadata"]["annotations"] = annotations
watcher.watch.return_value = [WatchEvent(event, PaasbetaApplication)]
spec_factory.side_effect = error
tpr_watcher._watch(None)
lifecycle.failed.assert_called_once_with(
app_name=event["object"]["spec"]["application"],
namespace=event["object"]["metadata"]["namespace"],
deployment_id='deployment_id',
repository=repository)
assert deploy_queue.empty()
def test_main_raises_ConfigurationParsingError_on_load_YAMLError(
self, mockOpen, mockYamlLoad):
mockYamlLoad.side_effect = YAMLError('boom')
with self.assertRaises(ConfigurationParsingError):
main('file')
def test_load_config(self, mock):
mock.side_effect = [self.config, IOError(), YAMLError()]
self.assertEqual(main.load_config(), self.config)
self.assertEqual(main.load_config(), {})
self.assertEqual(main.load_config(), {})
self.assertIn("ooni", self.config)
def fails(*args, **kwargs):
raise YAMLError()
