def test_get_minio_auth(self):
with mock.patch.dict('os.environ',
{'FUNCTION_CONFIG': StrUtils.utf8_to_base64_string(CONFIG_FILE_OK)},
clear=True):
minio_auth = StorageConfig().get_auth_data_by_stg_type('MINIO')
self.assertEqual(minio_auth.type, 'MINIO')
self.assertEqual(minio_auth.get_credential('access_key'),
'test_minio_access')
self.assertEqual(minio_auth.get_credential('secret_key'),
'test_minio_secret')
def test_get_onedata_auth(self):
with mock.patch.dict('os.environ',
{'FUNCTION_CONFIG': StrUtils.utf8_to_base64_string(CONFIG_FILE_OK)},
clear=True):
onedata_auth = StorageConfig().get_auth_data_by_stg_type('ONEDATA')
self.assertEqual(onedata_auth.type, 'ONEDATA')
self.assertEqual(onedata_auth.get_credential('oneprovider_host'),
'test_oneprovider.host')
self.assertEqual(onedata_auth.get_credential('token'),
'test_onedata_token')
self.assertEqual(onedata_auth.get_credential('space'),
'test_onedata_space')
def test_parse_config_no_storage_provider(self):
with mock.patch.dict('os.environ',
{'FUNCTION_CONFIG': StrUtils.utf8_to_base64_string(CONFIG_FILE_NO_STORAGE_PROVIDER)},
clear=True):
StorageConfig()
self.assertLogs('There is no storage provider defined for this function.',
level='WARNING')
def test_parse_config_no_output(self):
with mock.patch.dict('os.environ',
{'FUNCTION_CONFIG': StrUtils.utf8_to_base64_string(CONFIG_FILE_NO_OUTPUT)},
clear=True):
StorageConfig()
self.assertLogs('There is no output defined for this function.',
level='WARNING')
def test_parse_config_valid(self):
with mock.patch.dict('os.environ',
{'FUNCTION_CONFIG': StrUtils.utf8_to_base64_string(CONFIG_FILE_OK)},
clear=True):
config = StorageConfig()
expected_output = [
{
'storage_provider': 's3',
'path': 'bucket/folder'
}, {
'storage_provider': 'minio',
'path': 'bucket',
'suffix': ['txt', 'jpg'],
'prefix': ['result-']
}
]
self.assertEqual(config.output, expected_output)
self.assertEqual(config.minio_auth.type, 'MINIO')
self.assertEqual(config.minio_auth.get_credential('access_key'), 'test_minio_access')
self.assertEqual(config.minio_auth.get_credential('secret_key'), 'test_minio_secret')
self.assertEqual(config.onedata_auth.type, 'ONEDATA')
self.assertEqual(config.onedata_auth.get_credential('oneprovider_host'), 'test_oneprovider.host')
self.assertEqual(config.onedata_auth.get_credential('token'), 'test_onedata_token')
self.assertEqual(config.onedata_auth.get_credential('space'), 'test_onedata_space')
self.assertEqual(config.s3_auth.type, 'S3')
def test_parse_config_invalid_onedata(self):
with mock.patch.dict('os.environ',
{'FUNCTION_CONFIG': StrUtils.utf8_to_base64_string(CONFIG_FILE_INVALID_ONEDATA)},
clear=True):
with self.assertRaises(SystemExit):
StorageConfig()
self.assertLogs('The storage authentication of \'ONEDATA\' is not well-defined.',
level='ERROR')
def test_upload_output(self, mock_minio, mock_s3, mock_get_files):
with mock.patch.dict('os.environ',
{'FUNCTION_CONFIG': StrUtils.utf8_to_base64_string(CONFIG_FILE_OK)},
clear=True):
files = [
'/tmp/test/file1.txt',
'/tmp/test/file1.jpg',
'/tmp/test/result-file.txt',
'/tmp/test/result-file.out',
'/tmp/test/file2.txt',
'/tmp/test/file2.out'
]
mock_get_files.return_value = files
StorageConfig().upload_output('/tmp/test')
self.assertEqual(mock_minio.call_count, 1)
self.assertEqual(mock_minio.call_args,
call('/tmp/test/result-file.txt',
'result-file.txt',
'bucket'))
self.assertEqual(mock_s3.call_count, 6)
for i, f in enumerate(files):
self.assertEqual(mock_s3.call_args_list[i],
call(f, f.split('/')[3], 'bucket/folder'))
def _read_storage_config(self):
get_logger().info("Reading storage configuration")
self.stg_config = StorageConfig()
class Supervisor():
"""Generic supervisor used to create the required supervisors
based on the environment variable 'SUPERVISOR_TYPE'."""
# pylint: disable=too-few-public-methods
def __init__(self, event, context=None):
self._create_tmp_dirs()
# Parse the event_info data
self.parsed_event = parse_event(event)
# Read storage config
self._read_storage_config()
# Create the supervisor
self.supervisor = _create_supervisor(event, context)
def _create_tmp_dirs(self):
"""Creates the temporal directories where the
input/output data is going to be stored.
The folders are deleted automatically
when the execution finishes.
"""
self.input_tmp_dir = FileUtils.create_tmp_dir()
self.output_tmp_dir = FileUtils.create_tmp_dir()
SysUtils.set_env_var("TMP_INPUT_DIR", self.input_tmp_dir.name)
SysUtils.set_env_var("TMP_OUTPUT_DIR", self.output_tmp_dir.name)
def _read_storage_config(self):
get_logger().info("Reading storage configuration")
self.stg_config = StorageConfig()
@exception()
def _parse_input(self):
"""Download input data from storage provider
or save data from POST request.
A function can have information from several storage providers
but one event always represents only one file (so far), so only
one provider is going to be used for each event received.
"""
input_file_path = self.stg_config.download_input(
self.parsed_event, self.input_tmp_dir.name)
if input_file_path and FileUtils.is_file(input_file_path):
SysUtils.set_env_var('INPUT_FILE_PATH', input_file_path)
get_logger().info('INPUT_FILE_PATH variable set to \'%s\'',
input_file_path)
@exception()
def _parse_output(self):
self.stg_config.upload_output(self.output_tmp_dir.name)
@exception()
def run(self):
"""Generic method to launch the supervisor execution."""
try:
if is_batch_execution() and SysUtils.is_lambda_environment():
# Only delegate to batch
self.supervisor.execute_function()
else:
self._parse_input()
self.supervisor.execute_function()
self._parse_output()
get_logger().info('Creating response')
return self.supervisor.create_response()
except FaasSupervisorError as fse:
get_logger().exception(fse)
get_logger().error('Creating error response')
return self.supervisor.create_error_response()
def test_download_input(self, mock_download_file):
event = UnknownEvent('')
StorageConfig().download_input(event, '/tmp/test')
mock_download_file.assert_called_once_with(event, '/tmp/test')
def test_get_invalid_auth(self):
invalid_auth = StorageConfig().get_auth_data_by_stg_type('INVALID_TYPE')
self.assertIsNone(invalid_auth)
def test_get_s3_auth(self):
with mock.patch.dict('os.environ',
{'FUNCTION_CONFIG': StrUtils.utf8_to_base64_string(CONFIG_FILE_OK)},
clear=True):
s3_auth = StorageConfig().get_auth_data_by_stg_type('S3')
self.assertEqual(s3_auth.type, 'S3')
def _read_storage_config(self):
self.stg_config = StorageConfig()