def read_storage_providers(self): """Reads the global variables to create the providers needed. Variable schema: STORAGE_AUTH_$1_$2_$3 $1: MINIO | S3 | ONEDATA $2: USER | PASS | TOKEN | SPACE | HOST $3: STORAGE_ID (Specified in the function definition file, is unique for each storage defined) e.g.: STORAGE_AUTH_MINIO_USER_12345 """ # Remove the prefix 'STORAGE_AUTH_' env_vars = SysUtils.get_filtered_env_vars("STORAGE_AUTH_") # type = MINIO | S3 | ONEDATA ... # cred = USER | PASS | TOKEN ... provider_info = namedtuple('provider_info', ['type', 'cred', 'id']) for env_key, env_val in env_vars.items(): # Don't split past the id # MINIO_USER_123_45 -> *[MINIO, USER, 123_45] prov_info = provider_info(*env_key.split("_", 2)) # Link ID with TYPE if prov_info.id not in self.auth_id: self.auth_id[prov_info.id] = prov_info.type if prov_info.type not in self.auth_type: # Link TYPE with AUTH data self.auth_type[prov_info.type] = AuthData( prov_info.id, prov_info.type) self.auth_type[prov_info.type].set_credential( prov_info.cred, env_val)
def test_get_filtered_env_vars(self): with mock.patch.dict('os.environ', { "K1": "V1", "F1_C1": "VC1", "F1_C2": "VC2" }, clear=True): self.assertEqual(SysUtils.get_filtered_env_vars("F1_"), { "C1": "VC1", "C2": "VC2" })
def get_output_paths(): """Returns the defined output providers. Reads the global variables to create the providers needed. Variable schema: STORAGE_PATH_$1_$2 $1: INPUT | OUTPUT $2: STORAGE_ID (Specified in the function definition file, is unique for each storage defined) e.g.: STORAGE_PATH_INPUT_12345 """ get_logger().info("Reading output path variables") env_vars = SysUtils.get_filtered_env_vars("STORAGE_PATH_") storage_path = namedtuple('storage_path', ['id', 'path']) # Remove the storage type 'OUTPUT_' and store only the id and the path # Store a tuple, so the information can't be modified return [storage_path(env_key[7:], env_val) for env_key, env_val in env_vars.items() if env_key.startswith('OUTPUT_')]