def test_local_downloader_runner(self): with tempfile.TemporaryDirectory() as basedir_name: os.makedirs(os.path.join(basedir_name, 'filepath')) f_data = 'Hello, world!' with open(os.path.join(basedir_name, 'filepath', 'filename.ext'), mode='w') as f: f.write(f_data) with open(os.path.join(basedir_name, 'filepath', 'filename.ext'), mode='r') as f: self.assertEqual(f_data, f.read()) downloader_runner = LocalDownloaderRunner(basedir_name) with tempfile.TemporaryDirectory() as downloader_tempdir_name: openers = downloader_runner.download( downloader_tempdir_name, files=[File(name='filename.ext', subdir='filepath')]) self.assertEqual(1, len(openers)) with openers[0]() as f: self.assertEqual(f_data, f.read()) return
def test_file_path_error(self): inst = File() with self.assertRaises(AttributeError): path = inst.path return
def test_file_from_cloudevents_model_ok(self): inst_list = File.from_cloudevents_model(self._event_ok) self.assertEqual(1, len(inst_list)) for name in ['_id', 'name', 'subdir']: self.assertEqual(self._file_data.get(name, None), getattr(inst_list[0], name, None)) return
def test_file_from_cloudevents_model_error_invalid_source(self): with self.assertRaises(InvalidSourceValueError): inst_list = File.from_cloudevents_model(self._event_error_invalid_source) return
def test_file_path_ok_name_subdir(self): inst = File(name=self._file_data.get('name', None), subdir=self._file_data.get('subdir', None)) self.assertEqual(os.path.join(self._file_data.get('subdir', None), self._file_data.get('name', None)), inst.path) return
def test_file_path_ok_name(self): inst = File(name=self._file_data.get('name', None)) self.assertEqual(self._file_data.get('name', None), inst.path) return
def handle(self, event: Event) -> None: transaction_inst = Transaction.from_cloudevents_model(event) transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model( event) file_insts = File.from_cloudevents_model(event) config_by_config_id = _to_proxymod_config_by_config_id( transaction_key_values=transaction_key_value_insts) for config_id in ['config_1', 'config_2', 'config_3']: if config_id not in config_by_config_id: raise ConfigNotFoundProxEventHandlerError(event, config_id) config = config_by_config_id[config_id] if not _is_valid_proxymod_config(config): raise InvalidConfigProxEventHandlerError( event, config_id, config) input_file_insts = [] model_file_insts = [] _in_dir = config_by_config_id.get('config_1', {}).get('INPUTS', {}).get('in_dir', None) _in_file_one = config_by_config_id.get('config_1', {}).get( 'INPUTS', {}).get('in_file_one', None) _in_file_two = config_by_config_id.get('config_1', {}).get( 'INPUTS', {}).get('in_file_two', None) for file_inst in file_insts: if ('text/csv' == file_inst.mimetype) and ( file_inst.subdir is not None) and (_in_dir == file_inst.subdir) and ( file_inst.name is not None) and ((_in_file_one == file_inst.name) or (_in_file_two == file_inst.name)): input_file_insts.append(file_inst) elif ('text/x-python' == file_inst.mimetype) and ('models/' == file_inst.subdir): model_file_insts.append(file_inst) else: # NOTE Ignore other files. pass with tempfile.TemporaryDirectory() as downloader_tempdir_name: with tempfile.TemporaryDirectory() as uploader_tempdir_name: # model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts) with open(os.path.join(uploader_tempdir_name, 'download-stdout.log'), mode='w') as downloader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'download-stderr.log'), mode='w') as downloader_stderr_file: with contextlib.redirect_stdout( downloader_stdout_file): with contextlib.redirect_stderr( downloader_stderr_file): model_file_openers = self.downloader_runner.download( downloader_tempdir_name, model_file_insts) model_file_funcs = [] for model_file_inst, model_file_opener in zip( model_file_insts, model_file_openers): with model_file_opener() as file: try: name = os.path.splitext(model_file_inst.name)[0] spec = importlib.util.spec_from_file_location( name, file.name) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # NOTE Deliberately raise `AttributeError` if `name` does not exist. func = getattr(module, name) if callable(func): model_file_funcs.append(func) else: # NOTE Deliberately raise `TypeError` by calling an uncallable. func() except Exception as reason: raise InvalidModelProxEventHandlerError( event, model_file_inst, reason) # input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts) with open(os.path.join(uploader_tempdir_name, 'download-stdout.log'), mode='a') as downloader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'download-stderr.log'), mode='a') as downloader_stderr_file: with contextlib.redirect_stdout( downloader_stdout_file): with contextlib.redirect_stderr( downloader_stderr_file): input_file_openers = self.downloader_runner.download( downloader_tempdir_name, input_file_insts) abspath_config_by_config_id = copy.deepcopy( config_by_config_id) for config_id, config in abspath_config_by_config_id.items(): if 'INPUTS' in config: if 'in_dir' in config['INPUTS']: for input_file_inst, opener in zip( input_file_insts, input_file_openers): with opener() as file: config['INPUTS'][ 'in_dir'] = os.path.abspath( os.path.dirname(file.name)) break if 'OUTPUTS' in config: if 'out_dir' in config['OUTPUTS']: config['OUTPUTS']['out_dir'] = os.path.abspath( os.path.join(uploader_tempdir_name, config['OUTPUTS']['out_dir'])) for config_id, config in config_by_config_id.items(): with open(os.path.join(uploader_tempdir_name, '{0}.ini'.format(config_id)), mode='w') as config_file: config_file.write(_format_proxymod_config(config)) with tempfile.NamedTemporaryFile( suffix='.ini') as config_1_file: config_1_file.write( bytes( _format_proxymod_config( abspath_config_by_config_id['config_1']), 'utf-8')) config_1_file.seek(0) with tempfile.NamedTemporaryFile( suffix='.ini') as config_2_file: config_2_file.write( bytes( _format_proxymod_config( abspath_config_by_config_id['config_2']), 'utf-8')) config_2_file.seek(0) with tempfile.NamedTemporaryFile( suffix='.ini') as config_3_file: config_3_file.write( bytes( _format_proxymod_config( abspath_config_by_config_id['config_3'] ), 'utf-8')) config_3_file.seek(0) # for model_file_inst, model_file_func in zip(model_file_insts, model_file_funcs): # try: # model_file_func(config_1_file.name, config_2_file.name, config_3_file.name) # except Exception as reason: # raise InvalidModelProxEventHandlerError(event, model_file_inst, reason) with open(os.path.join(uploader_tempdir_name, 'stdout.log'), mode='w') as stdout_file: with open(os.path.join(uploader_tempdir_name, 'stderr.log'), mode='w') as stderr_file: with contextlib.redirect_stdout( stdout_file): with contextlib.redirect_stderr( stderr_file): for model_file_inst, model_file_func in zip( model_file_insts, model_file_funcs): try: model_file_func( config_1_file.name, config_2_file.name, config_3_file.name) except Exception as reason: raise InvalidModelProxEventHandlerError( event, model_file_inst, reason) # (bundle, job_id, state) = self.uploader_runner.upload(uploader_tempdir_name, transaction=Transaction(submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, proposal=transaction_inst.proposal), transaction_key_values=[TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)]) with open(os.path.join(uploader_tempdir_name, 'upload-stdout.log'), mode='w') as uploader_stdout_file: with open(os.path.join(uploader_tempdir_name, 'upload-stderr.log'), mode='w') as uploader_stderr_file: with contextlib.redirect_stdout(uploader_stdout_file): with contextlib.redirect_stderr( uploader_stderr_file): (bundle, job_id, state) = self.uploader_runner.upload( uploader_tempdir_name, transaction=Transaction( submitter=transaction_inst.submitter, instrument=transaction_inst. instrument, proposal=transaction_inst.proposal), transaction_key_values=[ TransactionKeyValue( key='Transactions._id', value=transaction_inst._id) ]) pass pass pass
def test_local_runners(self): transaction = Transaction(submitter=1, instrument=1, proposal=1) transaction_key_values = [ TransactionKeyValue(key='Transactions._id', value=1), ] files = [File(name='filename.ext', subdir='filepath')] file_strs = ['Hello, world!'] self.assertEqual(len(files), len(file_strs)) with tempfile.TemporaryDirectory() as basedir_name: for file, file_str in zip(files, file_strs): os.makedirs( os.path.join(basedir_name, os.path.dirname(file.path))) with open(os.path.join(basedir_name, file.path), mode='w') as f: f.write(file_str) with open(os.path.join(basedir_name, file.path), mode='r') as f: self.assertEqual(file_str, f.read()) downloader_runner = LocalDownloaderRunner(basedir_name) with tempfile.TemporaryDirectory() as downloader_tempdir_name: openers = downloader_runner.download(downloader_tempdir_name, files=files) self.assertEqual(len(files), len(openers)) with tempfile.TemporaryDirectory() as uploader_tempdir_name: uploader_runner = LocalUploaderRunner() for file, opener in zip(files, openers): os.makedirs( os.path.join(uploader_tempdir_name, os.path.dirname(file.path))) with opener() as orig_f: with open(os.path.join(uploader_tempdir_name, file.path), mode='w') as new_f: new_f.write(orig_f.read().upper()) with opener() as orig_f: with open(os.path.join(uploader_tempdir_name, file.path), mode='r') as new_f: self.assertEqual(orig_f.read().upper(), new_f.read()) (bundle, job_id, state) = uploader_runner.upload( uploader_tempdir_name, transaction=transaction, transaction_key_values=transaction_key_values) self.assertTrue(bundle.md_obj.is_valid()) self.assertEqual(len(files), len(bundle.file_data)) for file, file_data in zip(files, bundle.file_data): self.assertEqual(os.path.join('data', file.path), file_data.get('name', None)) self.assertEqual(None, job_id) self.assertEqual({}, state) return