def test_file_from_cloudevents_model_ok(self): inst_list = File.from_cloudevents_model(self._event_ok) self.assertEqual(1, len(inst_list)) for name in ['_id', 'name', 'subdir']: self.assertEqual(self._file_data.get(name, None), getattr(inst_list[0], name, None)) return
def test_file_from_cloudevents_model_error_invalid_source(self): with self.assertRaises(InvalidSourceValueError): inst_list = File.from_cloudevents_model(self._event_error_invalid_source) return
def handle(self, event: Event) -> None: transaction_inst = Transaction.from_cloudevents_model(event) transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model( event) file_insts = File.from_cloudevents_model(event) config_by_config_id = _to_proxymod_config_by_config_id( transaction_key_values=transaction_key_value_insts) for config_id in ['config_1', 'config_2', 'config_3']: if config_id not in config_by_config_id: raise ConfigNotFoundProxEventHandlerError(event, config_id) config = config_by_config_id[config_id] if not _is_valid_proxymod_config(config): raise InvalidConfigProxEventHandlerError( event, config_id, config) input_file_insts = [] model_file_insts = [] _in_dir = config_by_config_id.get('config_1', {}).get('INPUTS', {}).get('in_dir', None) _in_file_one = config_by_config_id.get('config_1', {}).get( 'INPUTS', {}).get('in_file_one', None) _in_file_two = config_by_config_id.get('config_1', {}).get( 'INPUTS', {}).get('in_file_two', None) for file_inst in file_insts: if ('text/csv' == file_inst.mimetype) and ( file_inst.subdir is not None) and (_in_dir == file_inst.subdir) and ( file_inst.name is not None) and ((_in_file_one == file_inst.name) or (_in_file_two == file_inst.name)): input_file_insts.append(file_inst) elif ('text/x-python' == file_inst.mimetype) and ('models/' == file_inst.subdir): model_file_insts.append(file_inst) else: # NOTE Ignore other files. pass with tempfile.TemporaryDirectory() as downloader_tempdir_name: with tempfile.TemporaryDirectory() as uploader_tempdir_name: # model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts) with open(os.path.join(uploader_tempdir_name, 'download-stdout.log'), mode='w') as downloader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'download-stderr.log'), mode='w') as downloader_stderr_file: with contextlib.redirect_stdout( downloader_stdout_file): with contextlib.redirect_stderr( downloader_stderr_file): model_file_openers = self.downloader_runner.download( downloader_tempdir_name, model_file_insts) model_file_funcs = [] for model_file_inst, model_file_opener in zip( model_file_insts, model_file_openers): with model_file_opener() as file: try: name = os.path.splitext(model_file_inst.name)[0] spec = importlib.util.spec_from_file_location( name, file.name) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # NOTE Deliberately raise `AttributeError` if `name` does not exist. func = getattr(module, name) if callable(func): model_file_funcs.append(func) else: # NOTE Deliberately raise `TypeError` by calling an uncallable. func() except Exception as reason: raise InvalidModelProxEventHandlerError( event, model_file_inst, reason) # input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts) with open(os.path.join(uploader_tempdir_name, 'download-stdout.log'), mode='a') as downloader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'download-stderr.log'), mode='a') as downloader_stderr_file: with contextlib.redirect_stdout( downloader_stdout_file): with contextlib.redirect_stderr( downloader_stderr_file): input_file_openers = self.downloader_runner.download( downloader_tempdir_name, input_file_insts) abspath_config_by_config_id = copy.deepcopy( config_by_config_id) for config_id, config in abspath_config_by_config_id.items(): if 'INPUTS' in config: if 'in_dir' in config['INPUTS']: for input_file_inst, opener in zip( input_file_insts, input_file_openers): with opener() as file: config['INPUTS'][ 'in_dir'] = os.path.abspath( os.path.dirname(file.name)) break if 'OUTPUTS' in config: if 'out_dir' in config['OUTPUTS']: config['OUTPUTS']['out_dir'] = os.path.abspath( os.path.join(uploader_tempdir_name, config['OUTPUTS']['out_dir'])) for config_id, config in config_by_config_id.items(): with open(os.path.join(uploader_tempdir_name, '{0}.ini'.format(config_id)), mode='w') as config_file: config_file.write(_format_proxymod_config(config)) with tempfile.NamedTemporaryFile( suffix='.ini') as config_1_file: config_1_file.write( bytes( _format_proxymod_config( abspath_config_by_config_id['config_1']), 'utf-8')) config_1_file.seek(0) with tempfile.NamedTemporaryFile( suffix='.ini') as config_2_file: config_2_file.write( bytes( _format_proxymod_config( abspath_config_by_config_id['config_2']), 'utf-8')) config_2_file.seek(0) with tempfile.NamedTemporaryFile( suffix='.ini') as config_3_file: config_3_file.write( bytes( _format_proxymod_config( abspath_config_by_config_id['config_3'] ), 'utf-8')) config_3_file.seek(0) # for model_file_inst, model_file_func in zip(model_file_insts, model_file_funcs): # try: # model_file_func(config_1_file.name, config_2_file.name, config_3_file.name) # except Exception as reason: # raise InvalidModelProxEventHandlerError(event, model_file_inst, reason) with open(os.path.join(uploader_tempdir_name, 'stdout.log'), mode='w') as stdout_file: with open(os.path.join(uploader_tempdir_name, 'stderr.log'), mode='w') as stderr_file: with contextlib.redirect_stdout( stdout_file): with contextlib.redirect_stderr( stderr_file): for model_file_inst, model_file_func in zip( model_file_insts, model_file_funcs): try: model_file_func( config_1_file.name, config_2_file.name, config_3_file.name) except Exception as reason: raise InvalidModelProxEventHandlerError( event, model_file_inst, reason) # (bundle, job_id, state) = self.uploader_runner.upload(uploader_tempdir_name, transaction=Transaction(submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, proposal=transaction_inst.proposal), transaction_key_values=[TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)]) with open(os.path.join(uploader_tempdir_name, 'upload-stdout.log'), mode='w') as uploader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'upload-stderr.log'), mode='w') as uploader_stderr_file: with contextlib.redirect_stdout(uploader_stdout_file): with contextlib.redirect_stderr( uploader_stderr_file): (bundle, job_id, state) = self.uploader_runner.upload( uploader_tempdir_name, transaction=Transaction( submitter=transaction_inst.submitter, instrument=transaction_inst. instrument, proposal=transaction_inst.proposal), transaction_key_values=[ TransactionKeyValue( key='Transactions._id', value=transaction_inst._id) ]) pass pass pass