def test_create_eventhub_producer_policies_secrets(self, victim):
policies = [EventHubProducerPolicy('entity1', False), EventHubProducerPolicy('entity2', False)]
victim.create_eventhub_producer_policies(policies)
assert Context().get(ContextKey.EVENTHUB_PRODUCER_POLICY_SECRETS) == [Secret('entity1-connection-string', 'potato-connection'),
Secret('entity2-connection-string', 'potato-connection')]
def test_add_secrets(self, victim):
secrets = [Secret("foo", "oof"), Secret("bar", "rab")]
victim._add_secrets("my-scope", secrets)
calls = [
mock.call("my-scope", "foo", "oof", None),
mock.call("my-scope", "bar", "rab", None)
]
victim.secret_api.put_secret.assert_has_calls(calls)
def test_create_secrets_from_yaml_file(self):
with open("tests/test_deployment.yml", "r") as f:
takeoff_config = yaml.safe_load(f.read())
for task_config in takeoff_config["steps"]:
if task_config[
"task"] == "create_databricks_secrets_from_vault":
res = victim(ApplicationVersion("ACP", "foo", "bar"),
task_config).get_deployment_secrets()
assert res == [
Secret("FOO", "acp_bar"),
Secret("BAR", "acp_foo")
]
def _create_image_pull_secret(self, application_name: str) -> str:
pull_secrets_yaml = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "assets", "kubernetes_image_pull_secrets.yml.j2"
)
return self._render_and_write_kubernetes_config(
kubernetes_config_path=pull_secrets_yaml,
application_name=application_name,
secrets=[Secret("pull_secret", self._get_docker_registry_secret())],
custom_values={
"namespace": Secret("namespace", self.config["image_pull_secret"]["namespace"]).val,
"secret_name": Secret("secret_name", self.config["image_pull_secret"]["secret_name"]).val,
},
)
def _create_consumer_group(self, group: EventHubConsumerGroup) -> Secret:
"""Creates given consumer groups on EventHub. Optionally constructs Databricks secret
containing the connection string for the consumer group.
Args:
group: Object containing names of EventHub namespace and entity
"""
try:
logger.info(f"Creating consumer group {group}")
self.eventhub_client.consumer_groups.create_or_update(
group.eventhub.resource_group,
group.eventhub.namespace,
group.eventhub.name,
group.consumer_group,
)
connection_string = self._create_connection_string(group.eventhub)
except CloudError as e:
logger.error("Something went wrong during creating consumer group")
raise e
suffix = ""
if group.append_env_to_databricks_secret_name:
suffix = "-{env}"
secret_name = get_databricks_secret_name(
f"{group.eventhub.name}-connection-string{suffix}", self.env)
secret = Secret(secret_name, connection_string.connection_string)
if group.create_databricks_secret:
self.create_databricks_secrets([secret])
return secret
def test_application_insights_with_databricks_secret(self, m1, m2, m3):
conf = {
**takeoff_config(),
**BASE_CONF, 'create_databricks_secret': True
}
target = CreateApplicationInsights(
ApplicationVersion("dev", "0.0.0", "my-branch"), conf)
m_client = mock.MagicMock()
m_client.components.create_or_update.return_value = MockApplicationInsights(
"something", "my-key")
with mock.patch(
"takeoff.azure.create_application_insights.CreateApplicationInsights._create_client",
return_value=m_client):
with mock.patch(
"takeoff.azure.create_application_insights.ApplicationInsightsComponent"
) as m_app_insights_component:
with mock.patch(
"takeoff.azure.create_application_insights.CreateApplicationInsights.create_databricks_secret"
) as m_create_databricks_secret:
target.create_application_insights()
m_app_insights_component.assert_called_once_with(
application_type='other', kind='other', location='west europe')
m_create_databricks_secret.assert_called_once_with(
'my_little_pony', Secret("instrumentation-key", "my-key"))
def test_get_secrets_from_config(self):
env = ApplicationVersion("DEV", "foo", "bar")
config = {
"task": "create_databricks_secrets_from_vault",
"dev": [{
"FOO": "foo_value"
}, {
"BAR": "bar_value"
}],
"acc": [{
"FOO": "fooacc_value"
}, {
"BAR": "baracc_value"
}],
}
res = victim(env, config).get_deployment_secrets()
assert res == [Secret("FOO", "foo_value"), Secret("BAR", "bar_value")]
def get_deployment_secrets(self) -> List[Secret]:
"""
Returns:
List[Secret]: The list of all secrets from the environment part in the config
"""
conf_secrets = self.config.get(self.app_version.environment.lower(),
[])
app_secrets = [
Secret(k, v) for secret in conf_secrets for k, v in secret.items()
]
return app_secrets
def _retrieve_secrets(self, client: AzureKeyVaultClient, vault: str,
prefix: Optional[str]) -> List[Secret]:
secrets = list(client.get_secrets(vault))
secrets_ids = self._extract_keyvault_ids_from(secrets)
secrets_filtered = self._filter_keyvault_ids(secrets_ids, prefix)
app_secrets = [
Secret(_.databricks_secret_key,
client.get_secret(vault, _.keyvault_id, "").value)
for _ in secrets_filtered
]
return app_secrets
def _create_producer_policy(
self,
policy: EventHubProducerPolicy,
resource_group: str,
eventhub_namespace: str,
application_name: str,
) -> Secret:
"""Creates given producer policy on EventHub. Optionally constructs Databricks secret
containing the connection string for the policy.
Args:
policy: Name of the EventHub entity
resource_group: The name of the resource group
eventhub_namespace: The name of the EventHub namespace
application_name: The name of this application
"""
common_azure_parameters = {
"resource_group_name":
resource_group,
"namespace_name":
eventhub_namespace,
"event_hub_name":
get_eventhub_entity_name(policy.eventhub_entity_name, self.env),
"authorization_rule_name":
f"{application_name}-send-policy",
}
try:
logger.info(
f"Creating producer policy with values {pprint.pformat(common_azure_parameters)}"
)
self.eventhub_client.event_hubs.create_or_update_authorization_rule(
**common_azure_parameters, rights=[AccessRights.send])
connection_string = self.eventhub_client.event_hubs.list_keys(
**common_azure_parameters).primary_connection_string
except Exception as e:
logger.error(
"Could not create connection String. Make sure the EventHub exists."
)
raise e
secret = Secret(f"{policy.eventhub_entity_name}-connection-string",
connection_string)
if policy.create_databricks_secret:
self.create_databricks_secrets([secret])
return secret
def create_application_insights(self):
client = self._create_client()
insight = self._find_existing_instance(client, self.application_name)
if not insight:
logger.info("Creating new Application Insights...")
# Create a new Application Insights
comp = ApplicationInsightsComponent(
location=self.config["azure"]["location"],
kind=self.config["kind"],
application_type=self.config["application_type"],
)
insight = client.components.create_or_update(
get_resource_group_name(self.config, self.env),
self.application_name, comp)
if self.config["create_databricks_secret"]:
instrumentation_secret = Secret("instrumentation-key",
insight.instrumentation_key)
self.create_databricks_secret(self.application_name,
instrumentation_secret)
class TestCreateDatabricksSecretsFromVault(object):
@mock.patch("takeoff.step.ApplicationName.get",
return_value="my_little_pony")
@mock.patch(
"takeoff.azure.create_databricks_secrets.KeyVaultClient.vault_and_client",
return_value=(None, None))
@mock.patch("takeoff.azure.create_databricks_secrets.Databricks",
return_value=MockDatabricksClient())
@mock.patch("takeoff.azure.create_databricks_secrets.SecretApi",
return_value={})
def test_validate_minimal_schema(self, m1, m2, m3, m4):
conf = {**takeoff_config(), **BASE_CONF}
CreateDatabricksSecretsFromVault(
ApplicationVersion('ACP', 'bar', 'foo'), conf)
def test_scope_exists(self, victim):
scopes = {"scopes": [{"name": "foo"}, {"name": "bar"}]}
assert victim._scope_exists(scopes, "foo")
assert not victim._scope_exists(scopes, "foobar")
@mock.patch(
'takeoff.azure.create_databricks_secrets.KeyVaultCredentialsMixin.get_keyvault_secrets',
return_value=[Secret('key1', 'foo'),
Secret('key2', 'bar')])
def test_combine_secrets_without_deployment_secrets(
self, _, victim_without_secrets):
combined_secrets = victim_without_secrets._combine_secrets()
assert len(combined_secrets) == 2
@mock.patch(
'takeoff.azure.create_databricks_secrets.KeyVaultCredentialsMixin.get_keyvault_secrets',
return_value=[])
def test_combine_secrets_without_deployment_and_keyvault_secrets(
self, _, victim_without_secrets):
combined_secrets = victim_without_secrets._combine_secrets()
assert len(combined_secrets) == 0
@mock.patch(
'takeoff.azure.create_databricks_secrets.KeyVaultCredentialsMixin.get_keyvault_secrets',
return_value=[])
def test_combine_secrets_without_keyvault_secrets(self, _, victim):
combined_secrets = victim._combine_secrets()
assert len(combined_secrets) == 3
@mock.patch(
'takeoff.azure.create_databricks_secrets.KeyVaultCredentialsMixin.get_keyvault_secrets',
return_value=[Secret('key1', 'foo'),
Secret('key2', 'bar')])
def test_combine_secrets_with_deployment_and_keyvault_secrets(
self, _, victim):
combined_secrets = victim._combine_secrets()
assert len(combined_secrets) == 5
@mock.patch(
'takeoff.azure.create_databricks_secrets.KeyVaultCredentialsMixin.get_keyvault_secrets',
return_value=[Secret('FOO', 'foo'),
Secret('BAR', 'bar')])
def test_combine_secrets_with_duplicate_deployment_and_keyvault_secrets(
self, _, victim):
combined_secrets = victim._combine_secrets()
assert len(combined_secrets) == 3
def test_create_scope(self, victim):
victim._create_scope("my-awesome-scope")
victim.secret_api.create_scope.assert_called_once_with(
"my-awesome-scope", None)
def test_create_scope_already_exists(self, victim):
victim._create_scope("scope1")
victim.secret_api.create_scope.assert_not_called()
def test_add_secrets(self, victim):
secrets = [Secret("foo", "oof"), Secret("bar", "rab")]
victim._add_secrets("my-scope", secrets)
calls = [
mock.call("my-scope", "foo", "oof", None),
mock.call("my-scope", "bar", "rab", None)
]
victim.secret_api.put_secret.assert_has_calls(calls)