def test_get_data_paths_works_as_expected(self):
with self.assertRaises(StoreNotFoundError):
stores.get_data_paths(['path1', 'path2'])
assert stores.get_data_paths(['data2']) == {'data2': '/data/2'}
assert stores.get_data_paths(['data3']) == {
'data3': 'gs://data-bucket'
}
assert stores.get_data_paths(['data2', 'data3']) == {
'data2': '/data/2',
'data3': 'gs://data-bucket'
}
def get_job_env_vars(persistence_outputs,
outputs_path,
persistence_data,
log_level=None,
logs_path=None,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
ephemeral_token=None):
env_vars = [
get_env_var(name=API_HTTP_URL, value=get_settings_http_api_url()),
get_env_var(name=API_WS_HOST, value=get_settings_ws_api_url()),
get_env_var(name=constants.CONFIG_MAP_IN_CLUSTER, value=True),
get_env_var(name=constants.CONFIG_MAP_API_VERSION, value=VERSION_V1),
get_env_var(name=constants.CONFIG_MAP_INTERNAL_HEADER,
value=settings.HEADERS_INTERNAL.replace('_', '-')),
get_env_var(name=constants.CONFIG_MAP_INTERNAL_HEADER_SERVICE,
value=InternalServices.RUNNER),
]
if log_level:
env_vars.append(
get_env_var(name=constants.CONFIG_MAP_LOG_LEVEL_KEY_NAME, value=log_level))
if logs_path:
env_vars.append(
get_env_var(name=constants.CONFIG_MAP_RUN_LOGS_PATH_KEY_NAME, value=logs_path))
# Data and outputs paths
data_paths = stores.get_data_paths(persistence_data)
env_vars += [
get_env_var(name=constants.CONFIG_MAP_RUN_OUTPUTS_PATH_KEY_NAME, value=outputs_path),
get_env_var(name=constants.CONFIG_MAP_RUN_DATA_PATHS_KEY_NAME, value=data_paths)
]
refs_outputs = {}
outputs_jobs_paths = get_paths_from_specs(specs=outputs_refs_jobs)
if outputs_jobs_paths:
refs_outputs['jobs'] = outputs_jobs_paths
outputs_experiments_paths = get_paths_from_specs(specs=outputs_refs_experiments)
if outputs_experiments_paths:
refs_outputs['experiments'] = outputs_experiments_paths
if refs_outputs:
env_vars.append(
get_env_var(name=constants.CONFIG_MAP_REFS_OUTPUTS_PATHS_KEY_NAME,
value=refs_outputs))
env_vars += get_job_stores_secrets_env_vars(persistence_outputs=persistence_outputs,
outputs_path=outputs_path,
persistence_data=persistence_data,
data_paths=data_paths,
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments)
if ephemeral_token:
env_vars.append(
get_env_var(name=constants.SECRET_EPHEMERAL_TOKEN,
value=ephemeral_token))
return env_vars
def get_job_env_vars(namespace,
persistence_outputs,
outputs_path,
persistence_data,
log_level=None,
logs_path=None,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
ephemeral_token=None):
env_vars = get_internal_env_vars(
service_internal_header=InternalServices.RUNNER, namespace=namespace)
if log_level:
env_vars.append(
get_env_var(name=constants.CONFIG_MAP_LOG_LEVEL_KEY_NAME,
value=log_level))
if logs_path:
env_vars.append(
get_env_var(name=constants.CONFIG_MAP_RUN_LOGS_PATH_KEY_NAME,
value=logs_path))
# Data and outputs paths
data_paths = stores.get_data_paths(persistence_data)
env_vars += [
get_env_var(name=constants.CONFIG_MAP_RUN_OUTPUTS_PATH_KEY_NAME,
value=outputs_path),
get_env_var(name=constants.CONFIG_MAP_RUN_DATA_PATHS_KEY_NAME,
value=data_paths)
]
refs_outputs = {}
outputs_jobs_paths = get_paths_from_specs(specs=outputs_refs_jobs)
if outputs_jobs_paths:
refs_outputs['jobs'] = outputs_jobs_paths
outputs_experiments_paths = get_paths_from_specs(
specs=outputs_refs_experiments)
if outputs_experiments_paths:
refs_outputs['experiments'] = outputs_experiments_paths
if refs_outputs:
env_vars.append(
get_env_var(name=constants.CONFIG_MAP_REFS_OUTPUTS_PATHS_KEY_NAME,
value=refs_outputs))
env_vars += get_job_stores_secrets_env_vars(
persistence_outputs=persistence_outputs,
outputs_path=outputs_path,
persistence_data=persistence_data,
data_paths=data_paths,
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments)
if ephemeral_token:
env_vars.append(
get_env_var(name=constants.SECRET_EPHEMERAL_TOKEN,
value=ephemeral_token))
return env_vars
def test_get_data_paths_raises_for_unrecognised_paths(self):
with self.assertRaises(StoreNotFoundError):
stores.get_data_paths(['path1', 'path2'])