def read_storage_providers(self):
"""Reads the global variables to create the providers needed.
Variable schema: STORAGE_AUTH_$1_$2_$3
$1: MINIO | S3 | ONEDATA
$2: USER | PASS | TOKEN | SPACE | HOST
$3: STORAGE_ID (Specified in the function definition file,
is unique for each storage defined)
e.g.: STORAGE_AUTH_MINIO_USER_12345
"""
# Remove the prefix 'STORAGE_AUTH_'
env_vars = SysUtils.get_filtered_env_vars("STORAGE_AUTH_")
# type = MINIO | S3 | ONEDATA ...
# cred = USER | PASS | TOKEN ...
provider_info = namedtuple('provider_info', ['type', 'cred', 'id'])
for env_key, env_val in env_vars.items():
# Don't split past the id
# MINIO_USER_123_45 -> *[MINIO, USER, 123_45]
prov_info = provider_info(*env_key.split("_", 2))
# Link ID with TYPE
if prov_info.id not in self.auth_id:
self.auth_id[prov_info.id] = prov_info.type
if prov_info.type not in self.auth_type:
# Link TYPE with AUTH data
self.auth_type[prov_info.type] = AuthData(
prov_info.id, prov_info.type)
self.auth_type[prov_info.type].set_credential(
prov_info.cred, env_val)
def test_get_filtered_env_vars(self):
with mock.patch.dict('os.environ', {
"K1": "V1",
"F1_C1": "VC1",
"F1_C2": "VC2"
},
clear=True):
self.assertEqual(SysUtils.get_filtered_env_vars("F1_"), {
"C1": "VC1",
"C2": "VC2"
})
def get_output_paths():
"""Returns the defined output providers.
Reads the global variables to create the providers needed.
Variable schema: STORAGE_PATH_$1_$2
$1: INPUT | OUTPUT
$2: STORAGE_ID (Specified in the function definition file,
is unique for each storage defined)
e.g.: STORAGE_PATH_INPUT_12345
"""
get_logger().info("Reading output path variables")
env_vars = SysUtils.get_filtered_env_vars("STORAGE_PATH_")
storage_path = namedtuple('storage_path', ['id', 'path'])
# Remove the storage type 'OUTPUT_' and store only the id and the path
# Store a tuple, so the information can't be modified
return [storage_path(env_key[7:], env_val) for env_key, env_val in env_vars.items()
if env_key.startswith('OUTPUT_')]
