def test_get_data_store_secrets_with_updated_settings(self):
with self.assertRaises(VolumeNotFoundError):
get_data_store_secrets(['data'], {})
secrets, _ = get_data_store_secrets(['data1'], {'data1': '/path/to/data'})
self.assertEqual(len(secrets), 0)
secrets, secret_keys = get_data_store_secrets(['data3'], {'data3': '/path/to/data3'})
assert len(secrets) == 1
assert len(secret_keys) == 1
assert list(secrets)[0] == ('data3-secret-name', 'data3-secret-key')
assert secret_keys == {
'/path/to/data3': {'secret_key': 'data3-secret-key', 'store': 'gcs'}}
secrets, secret_keys = get_data_store_secrets(
['data3', 'data4'],
{'data3': '/path/to/data3', 'data4': '/path/to/data4'})
assert len(secrets) == 2
assert len(secret_keys) == 2
assert secrets == {
('data3-secret-name', 'data3-secret-key'),
('data4-secret-name', 'data4-secret-key')}
assert secret_keys == {
'/path/to/data3': {'secret_key': 'data3-secret-key', 'store': 'gcs'},
'/path/to/data4': {'secret_key': 'data4-secret-key', 'store': 'gcs'}
}
def test_get_data_store_secrets(self):
secrets, _ = get_data_store_secrets(['data'], {})
self.assertEqual(len(secrets), 0)
secrets, _ = get_data_store_secrets(None, None)
self.assertEqual(len(secrets), 0)
with self.assertRaises(StoreNotFoundError):
get_data_store_secrets(['data1'], {})
def get_job_stores_secrets_env_vars(persistence_outputs,
outputs_path,
persistence_data,
data_paths,
outputs_refs_jobs=None,
outputs_refs_experiments=None):
env_vars = []
# Stores' secrets
secrets = set([])
secret_keys = {}
data_secrets, data_secret_keys = get_data_store_secrets(
persistence_data=persistence_data, data_paths=data_paths)
secrets |= data_secrets
secret_keys.update(data_secret_keys)
outputs_secrets, outputs_secret_keys = get_outputs_store_secrets(
persistence_outputs=persistence_outputs, outputs_path=outputs_path)
secrets |= outputs_secrets
secret_keys.update(outputs_secret_keys)
jobs_refs_secrets, jobs_refs_secret_keys = get_outputs_refs_store_secrets(
specs=outputs_refs_jobs)
secrets |= jobs_refs_secrets
secret_keys.update(jobs_refs_secret_keys)
experiments_refs_secrets, experiments_refs_secret_keys = get_outputs_refs_store_secrets(
specs=outputs_refs_experiments)
secrets |= experiments_refs_secrets
secret_keys.update(experiments_refs_secret_keys)
# Expose secret keys from all secrets
for (secret, secret_key) in secrets:
env_vars.append(
get_from_secret(key_name=secret_key,
secret_key_name=secret_key,
secret_ref_name=secret))
# Add paths' secret env vars
if secret_keys:
env_vars.append(
get_env_var(name=constants.CONFIG_MAP_RUN_STORES_ACCESS_KEYS,
value=secret_keys))
return env_vars