def deploy_env_logic(config: dict) -> ApplicationVersion:
branch = BranchName(config).get()
tag = get_tag()
if tag:
return ApplicationVersion("PRD", str(tag), branch)
elif branch == "master":
return ApplicationVersion("ACP", "SNAPSHOT", branch)
else:
logger.info("Not deploying feature branches")
exit(0)
def call_victim(self, config):
env = ApplicationVersion("DEV", "04fab6", "my-branch")
with mock.patch(
"takeoff.azure.credentials.keyvault.ServicePrincipalCredentials.credentials",
return_value="mylittlepony") as m_creds:
victim.vault_and_client(config, env)
m_creds.assert_called_once_with(config, "dev")
def victim():
conf = {**takeoff_config(), **BASE_CONF}
with mock.patch("takeoff.step.KeyVaultClient.vault_and_client",
return_value=(None, None)):
return CreateApplicationInsights(
ApplicationVersion("dev", "0.0.0", "my-branch"), conf)
def victim():
with mock.patch.dict(os.environ, env_variables), \
mock.patch("takeoff.step.ApplicationName.get", return_value="my_little_pony"), \
mock.patch("takeoff.azure.deploy_to_kubernetes.KeyVaultClient.vault_and_client", return_value=(None, None)):
conf = {**takeoff_config(), **BASE_CONF}
conf['azure'].update({"kubernetes_naming": "kubernetes{env}"})
return DeployToKubernetes(ApplicationVersion("dev", "v", "branch"), conf)
def victim():
m_jobs_api_client = mock.MagicMock()
m_runs_api_client = mock.MagicMock()
m_jobs_api_client.list_jobs.return_value = {
"jobs": [
{"job_id": "id1", "settings": {"name": "job1"}},
{"job_id": "id2", "settings": {"name": "job2"}},
]
}
m_jobs_api_client.delete_job.return_value = True
m_jobs_api_client.create_job.return_value = {"job_id": "job1"}
m_jobs_api_client.run_now.return_value = {"run_id": "run1"}
m_runs_api_client.list_runs.return_value = {
"runs": [{"run_id": "run1"}, {"run_id": "run2"}]
}
with mock.patch("takeoff.azure.deploy_to_databricks.KeyVaultClient.vault_and_client", return_value=(None, None)), \
mock.patch("takeoff.step.ApplicationName.get", return_value="my_app"), \
mock.patch("takeoff.azure.deploy_to_databricks.Databricks", return_value=MockDatabricksClient()), \
mock.patch("takeoff.azure.deploy_to_databricks.JobsApi", return_value=m_jobs_api_client), \
mock.patch("takeoff.azure.deploy_to_databricks.RunsApi", return_value=m_runs_api_client):
conf = {**takeoff_config(), **BASE_CONF}
return DeployToDatabricks(ApplicationVersion('ACP', 'bar', 'foo'), conf)
def victim():
m_client = mock.MagicMock()
m_client.consumer_groups.list_by_event_hub.return_value = {
MockEventHubClientResponse("group1"),
MockEventHubClientResponse("group2")
}
m_client.consumer_groups.create_or_update.return_value = {}
m_client.event_hubs.list_by_namespace.return_value = {
MockEventHubClientResponse("hub1"),
MockEventHubClientResponse("hub2")
}
m_client.event_hubs.list_authorization_rules.return_value = {
MockEventHubClientResponse("rule1"),
MockEventHubClientResponse("rule2")
}
m_client.event_hubs.list_keys.return_value = MockEventHubClientResponse(
'potatoes1', 'potato-connection')
with mock.patch("takeoff.step.ApplicationName.get", return_value="my_little_pony"), \
mock.patch("takeoff.azure.configure_eventhub.ConfigureEventHub._get_eventhub_client", return_value=m_client), \
mock.patch("takeoff.azure.configure_eventhub.KeyVaultClient.vault_and_client", return_value=(None, None)):
conf = {**takeoff_config(), **BASE_CONF}
conf['azure'].update({"eventhub_naming": "eventhub{env}"})
return ConfigureEventHub(ApplicationVersion('DEV', 'local', 'foo'),
conf)
def test_application_insights_with_databricks_secret(self, m1, m2, m3):
conf = {
**takeoff_config(),
**BASE_CONF, 'create_databricks_secret': True
}
target = CreateApplicationInsights(
ApplicationVersion("dev", "0.0.0", "my-branch"), conf)
m_client = mock.MagicMock()
m_client.components.create_or_update.return_value = MockApplicationInsights(
"something", "my-key")
with mock.patch(
"takeoff.azure.create_application_insights.CreateApplicationInsights._create_client",
return_value=m_client):
with mock.patch(
"takeoff.azure.create_application_insights.ApplicationInsightsComponent"
) as m_app_insights_component:
with mock.patch(
"takeoff.azure.create_application_insights.CreateApplicationInsights.create_databricks_secret"
) as m_create_databricks_secret:
target.create_application_insights()
m_app_insights_component.assert_called_once_with(
application_type='other', kind='other', location='west europe')
m_create_databricks_secret.assert_called_once_with(
'my_little_pony', Secret("instrumentation-key", "my-key"))
def test_validate_schema_invalid_target(self, _):
conf = {**takeoff_config(), **BASE_CONF, "target": ["ivy"]}
with pytest.raises(vol.Invalid):
victim(ApplicationVersion("dev", "v", "branch"), conf)
conf = {
**takeoff_config(),
**BASE_CONF,
**{
"language": "sbt",
"target": ["pypi"]
}
}
with pytest.raises(vol.Invalid):
victim(ApplicationVersion("dev", "v", "branch"), conf)
def test_validate_minimal_schema(self, _, __):
conf = {**takeoff_config(), **BASE_CONF}
conf['azure'].update({"kubernetes_naming": "kubernetes{env}"})
res = DeployToKubernetes(ApplicationVersion("dev", "v", "branch"),
conf)
assert res.config[
'kubernetes_config_path'] == "kubernetes_config/k8s.yml.j2"
def test_validate_invalid_schema(self):
INVALID_CONF = {
'task': 'create_application_insights',
'application_type': 'invalid',
'kind': 'invalid'
}
conf = {**takeoff_config(), **INVALID_CONF}
with pytest.raises(MultipleInvalid):
CreateApplicationInsights(ApplicationVersion("dev", "v", "branch"), conf)
def test_get_custom_values_invalid_env(self, _):
custom_conf = {'custom_values': {'invalid_env': {'my_custom_value': 'hello'}}, **BASE_CONF}
conf = {**takeoff_config(), **custom_conf}
conf['azure'].update({"kubernetes_naming": "kubernetes{env}"})
res = DeployToKubernetes(ApplicationVersion("dev", "v", "branch"), conf)
with pytest.raises(ValueError):
res._get_custom_values()
def test_validate_full_schema(self, _):
conf = {**takeoff_config(),
**BASE_CONF, **{
"dockerfiles": [{
"file": "Dockerfile_custom",
"postfix": "Dave",
"custom_image_name": "Mustaine"
}]}}
DockerImageBuilder(ApplicationVersion("dev", "v", "branch"), conf)
def deploy_env_logic(config: dict) -> ApplicationVersion:
"""Returns the version of this application based on provided Takeoff config.
Args:
config: Schiphol takeoff configuration
Returns:
Information about the version of the application and to which environment it should be deployed
"""
branch = BranchName(config=config).get()
tag = get_tag()
git_hash = get_short_hash()
if tag:
return ApplicationVersion("PRD", str(tag), branch)
elif branch == "master":
return ApplicationVersion("ACP", "SNAPSHOT", branch)
else:
return ApplicationVersion("DEV", git_hash, branch)
def test_publish_to_pypi(self, m1, m2, m3):
conf = {
**takeoff_config(),
**BASE_CONF, "language": "python",
"target": ["pypi"]
}
env = ApplicationVersion('prd', '1.0.0', 'branch')
with mock.patch("takeoff.azure.publish_artifact.upload") as m:
victim(env, conf).publish_to_pypi()
m.assert_called_once_with(upload_settings="foo", dists=["dist/*"])
def setup_victim(add_secrets: bool):
secrets_conf = {}
if add_secrets:
secrets_conf = {
'task':
'create_databricks_secrets_from_vault',
'dev': [
{
'FOO': 'foo_value'
},
{
'BAR': 'bar_value'
},
],
'acp': [
{
'FOO': 'fooacc_value'
},
{
'BAR': 'baracc_value'
},
{
'BAZ': 'baz_value'
},
]
}
m_client = mock.MagicMock()
m_client.consumer_groups.list_by_event_hub.return_value = {}
m_client.list_scopes.return_value = {
"scopes": [{
"name": "scope1"
}, {
"name": " scope2"
}]
}
m_client.create_scope.return_value = True
m_client.put_secret.return_value = True
with mock.patch("takeoff.step.ApplicationName.get", return_value="my_little_pony"), \
mock.patch("takeoff.azure.create_databricks_secrets.KeyVaultClient.vault_and_client", return_value=(None, None)), \
mock.patch("takeoff.azure.create_databricks_secrets.Databricks", return_value=MockDatabricksClient()), \
mock.patch("takeoff.azure.create_databricks_secrets.SecretApi", return_value=m_client):
conf = {
**takeoff_config(),
**BASE_CONF,
**{
"common": {
"databricks_library_path": "/path"
}
},
**secrets_conf
}
return CreateDatabricksSecretsFromVault(
ApplicationVersion('ACP', '0.0.0', 'my-branch'), conf)
def test_get_custom_values(self, _):
custom_conf = {'custom_values': {'dev': {'my_custom_value': 'hello'}}, **BASE_CONF}
conf = {**takeoff_config(), **custom_conf}
conf['azure'].update({"kubernetes_naming": "kubernetes{env}"})
res = DeployToKubernetes(ApplicationVersion("dev", "v", "branch"), conf)
result = res._get_custom_values()
expected_result = {"my_custom_value": "hello"}
assert result == expected_result
def test_publish_to_ivy_with_tag(self, m1, m2, m3):
conf = {
**takeoff_config(),
**BASE_CONF, "language": "scala",
"target": ["ivy"]
}
env = ApplicationVersion('prd', '1.0.0', 'branch')
with mock.patch("takeoff.azure.publish_artifact.run_shell_command",
return_value=(0, ['output_lines'])) as m:
victim(env, conf).publish_to_ivy()
m.assert_called_once_with(["sbt", 'set version := "1.0.0"', "publish"])
def test_validate_await_invalid_resource_name(self, _, __):
custom_conf = {
'wait_for_rollout': {
'resource_name': 'invalid_name',
'resource_namespace': 'my_space'
},
**BASE_CONF
}
conf = {**takeoff_config(), **custom_conf}
with pytest.raises(voluptuous.error.MultipleInvalid):
DeployToKubernetes(ApplicationVersion("dev", "v", "branch"), conf)
def test_create_secrets_from_yaml_file(self):
with open("tests/test_deployment.yml", "r") as f:
takeoff_config = yaml.safe_load(f.read())
for task_config in takeoff_config["steps"]:
if task_config[
"task"] == "create_databricks_secrets_from_vault":
res = victim(ApplicationVersion("ACP", "foo", "bar"),
task_config).get_deployment_secrets()
assert res == [
Secret("FOO", "acp_bar"),
Secret("BAR", "acp_foo")
]
def test_validate_minimal_schema(self, _):
conf = {**takeoff_config(), **BASE_CONF}
res = DockerImageBuilder(ApplicationVersion("dev", "v", "branch"),
conf)
assert res.config['dockerfiles'] == [{
"file": "Dockerfile",
"postfix": None,
"prefix": None,
"custom_image_name": None,
'tag_release_as_latest': True
}]
def test_get_secrets_from_config_with_empty_env(self):
env = ApplicationVersion("DEV", "foo", "bar")
config = {
"task": "create_databricks_secrets_from_vault",
"dev": [],
"acc": [{
"FOO": "fooacc_value"
}, {
"BAR": "baracc_value"
}],
}
res = victim(env, config).get_deployment_secrets()
assert res == []
def test_validate_minimal_schema(self, _, __):
conf = {**takeoff_config(), **BASE_CONF}
conf['azure'].update({"eventhub_naming": "eventhub{env}"})
ConfigureEventHub(ApplicationVersion("dev", "v", "branch"), conf)
def test_get_secrets_from_empty_config(self):
env = ApplicationVersion("DEV", "foo", "bar")
config = {"task": "create_databricks_secrets_from_vault"}
res = victim(env, config).get_deployment_secrets()
assert res == []
import os
import sys
import mock
from takeoff.application_version import ApplicationVersion
from takeoff.azure import util as victim
from takeoff.deploy import add_takeoff_plugin_paths
ENV = ApplicationVersion("dev", "local", "master")
def test_get_default_resource_group():
res = victim.get_resource_group_name(
{'azure': {
'resource_group_naming': 'rg{env}'
}}, ENV)
assert res == "rgdev"
@mock.patch("takeoff.util.DEFAULT_TAKEOFF_PLUGIN_PREFIX", "_takeoff_")
def test_get_custom_resource_group():
paths = [os.path.dirname(os.path.realpath(__file__))]
add_takeoff_plugin_paths(paths)
res = victim.get_resource_group_name({}, ENV)
assert res == "Dave"
sys.path.remove(paths[0])
def test_get_default_keyvault():
def victim_release() -> DockerImageBuilder:
with mock.patch("takeoff.build_docker_image.DockerRegistry.credentials", return_value=CREDS), \
mock.patch("takeoff.step.ApplicationName.get", return_value="myapp"):
conf = {**takeoff_config(), **BASE_CONF}
return DockerImageBuilder(ApplicationVersion('PRD', '2.1.0', 'master'),
conf)
def test_validate_minimal_schema_missing_key(self, _, __):
conf = {**takeoff_config(), 'task': 'createEventHubConsumerGroups'}
conf['azure'].update({"eventhub_naming": "eventhub{env}"})
with pytest.raises(vol.MultipleInvalid):
ConfigureEventHub(ApplicationVersion("dev", "v", "branch"), conf)
def test_invalid_config_empty_jobs(self, _):
config = {**takeoff_config(), **BASE_CONF, "jobs": []}
with pytest.raises(vol.MultipleInvalid):
DeployToDatabricks(ApplicationVersion("DEV", "local", "foo"), config)
import os
import unittest
from unittest import mock
import pytest
from takeoff.application_version import ApplicationVersion
from takeoff.build_artifact import BuildArtifact as victim
from tests.azure import takeoff_config
BASE_CONF = {"task": "build_artifact", "build_tool": "python"}
FAKE_ENV = ApplicationVersion('env', 'v', 'branch')
class TestBuildArtifact(unittest.TestCase):
@mock.patch.dict(os.environ, {"CI_PROJECT_NAME": "Elon"})
def test_validate_minimal_schema(self):
conf = {**takeoff_config(), **BASE_CONF}
victim(FAKE_ENV, conf)
@mock.patch.dict(os.environ, {"CI_PROJECT_NAME": "Elon"})
def test_build_python(self):
conf = {**takeoff_config(), **BASE_CONF}
with mock.patch.object(victim, "build_python_wheel") as m:
victim(FAKE_ENV, conf).run()
m.assert_called_once()
@mock.patch.dict(os.environ, {"CI_PROJECT_NAME": "Elon"})
def call_victim(self, config):
return victim(config, ApplicationVersion("env", "", "")).credentials()
def test_version_is_feature():
env = ApplicationVersion("DEV", "108fba3", 'some-branch')
assert env.on_feature_branch
