def test_local_downloader_runner(self):
with tempfile.TemporaryDirectory() as basedir_name:
os.makedirs(os.path.join(basedir_name, 'filepath'))
f_data = 'Hello, world!'
with open(os.path.join(basedir_name, 'filepath', 'filename.ext'),
mode='w') as f:
f.write(f_data)
with open(os.path.join(basedir_name, 'filepath', 'filename.ext'),
mode='r') as f:
self.assertEqual(f_data, f.read())
downloader_runner = LocalDownloaderRunner(basedir_name)
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
openers = downloader_runner.download(
downloader_tempdir_name,
files=[File(name='filename.ext', subdir='filepath')])
self.assertEqual(1, len(openers))
with openers[0]() as f:
self.assertEqual(f_data, f.read())
return
def test_file_path_error(self):
inst = File()
with self.assertRaises(AttributeError):
path = inst.path
return
def test_file_from_cloudevents_model_ok(self):
inst_list = File.from_cloudevents_model(self._event_ok)
self.assertEqual(1, len(inst_list))
for name in ['_id', 'name', 'subdir']:
self.assertEqual(self._file_data.get(name, None), getattr(inst_list[0], name, None))
return
def test_file_from_cloudevents_model_error_invalid_source(self):
with self.assertRaises(InvalidSourceValueError):
inst_list = File.from_cloudevents_model(self._event_error_invalid_source)
return
def test_file_path_ok_name_subdir(self):
inst = File(name=self._file_data.get('name', None), subdir=self._file_data.get('subdir', None))
self.assertEqual(os.path.join(self._file_data.get('subdir', None), self._file_data.get('name', None)), inst.path)
return
def test_file_path_ok_name(self):
inst = File(name=self._file_data.get('name', None))
self.assertEqual(self._file_data.get('name', None), inst.path)
return
def handle(self, event: Event) -> None:
transaction_inst = Transaction.from_cloudevents_model(event)
transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(
event)
file_insts = File.from_cloudevents_model(event)
config_by_config_id = _to_proxymod_config_by_config_id(
transaction_key_values=transaction_key_value_insts)
for config_id in ['config_1', 'config_2', 'config_3']:
if config_id not in config_by_config_id:
raise ConfigNotFoundProxEventHandlerError(event, config_id)
config = config_by_config_id[config_id]
if not _is_valid_proxymod_config(config):
raise InvalidConfigProxEventHandlerError(
event, config_id, config)
input_file_insts = []
model_file_insts = []
_in_dir = config_by_config_id.get('config_1',
{}).get('INPUTS',
{}).get('in_dir', None)
_in_file_one = config_by_config_id.get('config_1', {}).get(
'INPUTS', {}).get('in_file_one', None)
_in_file_two = config_by_config_id.get('config_1', {}).get(
'INPUTS', {}).get('in_file_two', None)
for file_inst in file_insts:
if ('text/csv' == file_inst.mimetype) and (
file_inst.subdir
is not None) and (_in_dir == file_inst.subdir) and (
file_inst.name
is not None) and ((_in_file_one == file_inst.name) or
(_in_file_two == file_inst.name)):
input_file_insts.append(file_inst)
elif ('text/x-python'
== file_inst.mimetype) and ('models/' == file_inst.subdir):
model_file_insts.append(file_inst)
else:
# NOTE Ignore other files.
pass
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
# model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts)
with open(os.path.join(uploader_tempdir_name,
'download-stdout.log'),
mode='w') as downloader_stdout_file:
with open(os.path.join(uploader_tempdir_name,
'download-stderr.log'),
mode='w') as downloader_stderr_file:
with contextlib.redirect_stdout(
downloader_stdout_file):
with contextlib.redirect_stderr(
downloader_stderr_file):
model_file_openers = self.downloader_runner.download(
downloader_tempdir_name, model_file_insts)
model_file_funcs = []
for model_file_inst, model_file_opener in zip(
model_file_insts, model_file_openers):
with model_file_opener() as file:
try:
name = os.path.splitext(model_file_inst.name)[0]
spec = importlib.util.spec_from_file_location(
name, file.name)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# NOTE Deliberately raise `AttributeError` if `name` does not exist.
func = getattr(module, name)
if callable(func):
model_file_funcs.append(func)
else:
# NOTE Deliberately raise `TypeError` by calling an uncallable.
func()
except Exception as reason:
raise InvalidModelProxEventHandlerError(
event, model_file_inst, reason)
# input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts)
with open(os.path.join(uploader_tempdir_name,
'download-stdout.log'),
mode='a') as downloader_stdout_file:
with open(os.path.join(uploader_tempdir_name,
'download-stderr.log'),
mode='a') as downloader_stderr_file:
with contextlib.redirect_stdout(
downloader_stdout_file):
with contextlib.redirect_stderr(
downloader_stderr_file):
input_file_openers = self.downloader_runner.download(
downloader_tempdir_name, input_file_insts)
abspath_config_by_config_id = copy.deepcopy(
config_by_config_id)
for config_id, config in abspath_config_by_config_id.items():
if 'INPUTS' in config:
if 'in_dir' in config['INPUTS']:
for input_file_inst, opener in zip(
input_file_insts, input_file_openers):
with opener() as file:
config['INPUTS'][
'in_dir'] = os.path.abspath(
os.path.dirname(file.name))
break
if 'OUTPUTS' in config:
if 'out_dir' in config['OUTPUTS']:
config['OUTPUTS']['out_dir'] = os.path.abspath(
os.path.join(uploader_tempdir_name,
config['OUTPUTS']['out_dir']))
for config_id, config in config_by_config_id.items():
with open(os.path.join(uploader_tempdir_name,
'{0}.ini'.format(config_id)),
mode='w') as config_file:
config_file.write(_format_proxymod_config(config))
with tempfile.NamedTemporaryFile(
suffix='.ini') as config_1_file:
config_1_file.write(
bytes(
_format_proxymod_config(
abspath_config_by_config_id['config_1']),
'utf-8'))
config_1_file.seek(0)
with tempfile.NamedTemporaryFile(
suffix='.ini') as config_2_file:
config_2_file.write(
bytes(
_format_proxymod_config(
abspath_config_by_config_id['config_2']),
'utf-8'))
config_2_file.seek(0)
with tempfile.NamedTemporaryFile(
suffix='.ini') as config_3_file:
config_3_file.write(
bytes(
_format_proxymod_config(
abspath_config_by_config_id['config_3']
), 'utf-8'))
config_3_file.seek(0)
# for model_file_inst, model_file_func in zip(model_file_insts, model_file_funcs):
# try:
# model_file_func(config_1_file.name, config_2_file.name, config_3_file.name)
# except Exception as reason:
# raise InvalidModelProxEventHandlerError(event, model_file_inst, reason)
with open(os.path.join(uploader_tempdir_name,
'stdout.log'),
mode='w') as stdout_file:
with open(os.path.join(uploader_tempdir_name,
'stderr.log'),
mode='w') as stderr_file:
with contextlib.redirect_stdout(
stdout_file):
with contextlib.redirect_stderr(
stderr_file):
for model_file_inst, model_file_func in zip(
model_file_insts,
model_file_funcs):
try:
model_file_func(
config_1_file.name,
config_2_file.name,
config_3_file.name)
except Exception as reason:
raise InvalidModelProxEventHandlerError(
event, model_file_inst,
reason)
# (bundle, job_id, state) = self.uploader_runner.upload(uploader_tempdir_name, transaction=Transaction(submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, proposal=transaction_inst.proposal), transaction_key_values=[TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)])
with open(os.path.join(uploader_tempdir_name,
'upload-stdout.log'),
mode='w') as uploader_stdout_file:
with open(os.path.join(uploader_tempdir_name,
'upload-stderr.log'),
mode='w') as uploader_stderr_file:
with contextlib.redirect_stdout(uploader_stdout_file):
with contextlib.redirect_stderr(
uploader_stderr_file):
(bundle, job_id,
state) = self.uploader_runner.upload(
uploader_tempdir_name,
transaction=Transaction(
submitter=transaction_inst.submitter,
instrument=transaction_inst.
instrument,
proposal=transaction_inst.proposal),
transaction_key_values=[
TransactionKeyValue(
key='Transactions._id',
value=transaction_inst._id)
])
pass
pass
pass
def test_local_runners(self):
transaction = Transaction(submitter=1, instrument=1, proposal=1)
transaction_key_values = [
TransactionKeyValue(key='Transactions._id', value=1),
]
files = [File(name='filename.ext', subdir='filepath')]
file_strs = ['Hello, world!']
self.assertEqual(len(files), len(file_strs))
with tempfile.TemporaryDirectory() as basedir_name:
for file, file_str in zip(files, file_strs):
os.makedirs(
os.path.join(basedir_name, os.path.dirname(file.path)))
with open(os.path.join(basedir_name, file.path),
mode='w') as f:
f.write(file_str)
with open(os.path.join(basedir_name, file.path),
mode='r') as f:
self.assertEqual(file_str, f.read())
downloader_runner = LocalDownloaderRunner(basedir_name)
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
openers = downloader_runner.download(downloader_tempdir_name,
files=files)
self.assertEqual(len(files), len(openers))
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
uploader_runner = LocalUploaderRunner()
for file, opener in zip(files, openers):
os.makedirs(
os.path.join(uploader_tempdir_name,
os.path.dirname(file.path)))
with opener() as orig_f:
with open(os.path.join(uploader_tempdir_name,
file.path),
mode='w') as new_f:
new_f.write(orig_f.read().upper())
with opener() as orig_f:
with open(os.path.join(uploader_tempdir_name,
file.path),
mode='r') as new_f:
self.assertEqual(orig_f.read().upper(),
new_f.read())
(bundle, job_id, state) = uploader_runner.upload(
uploader_tempdir_name,
transaction=transaction,
transaction_key_values=transaction_key_values)
self.assertTrue(bundle.md_obj.is_valid())
self.assertEqual(len(files), len(bundle.file_data))
for file, file_data in zip(files, bundle.file_data):
self.assertEqual(os.path.join('data', file.path),
file_data.get('name', None))
self.assertEqual(None, job_id)
self.assertEqual({}, state)
return