def test_remote_downloader_runner(self): """Test the downloader runner class.""" # Hash sum the README.md at root of repo (it's what we use to system test). hash_obj = hashlib.sha1() with open(os.path.join('..', 'README.md'), 'r') as readme_fd: hash_obj.update(bytes(readme_fd.read(), 'utf8')) hashsum = hash_obj.hexdigest() with tempfile.TemporaryDirectory() as basedir_name: downloader_runner = RemoteDownloaderRunner(Downloader()) openers = downloader_runner.download(basedir_name, files=[ File(_id=103, name='foo.txt', subdir='a/b', hashtype='sha1', hashsum=hashsum), File(_id=104, name='bar\u00e9.txt', subdir='a/b/\u00e9', hashtype='sha1', hashsum=hashsum) ]) self.assertEqual(2, len(openers)) with openers[0]() as readme_fd: self.assertTrue('Pacifica Dispatcher' in readme_fd.read())
def handle(self, event: Event) -> None: """ Example handle event. This handler downloads all files in the event. Converts the files to uppercase and uploads them back to Pacifica. """ transaction_inst = Transaction.from_cloudevents_model(event) transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model( event) file_insts = File.from_cloudevents_model(event) with tempfile.TemporaryDirectory() as downloader_tempdir_name: with tempfile.TemporaryDirectory() as uploader_tempdir_name: for file_opener in self.downloader_runner.download( downloader_tempdir_name, file_insts): with file_opener() as file_fd: with open( os.path.join(uploader_tempdir_name, file_fd.name), 'w') as wfile_fd: wfile_fd.write(file_fd.read().upper()) (_bundle, _job_id, _state) = self.uploader_runner.upload( uploader_tempdir_name, transaction=Transaction( submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, project=transaction_inst.project), transaction_key_values=[ TransactionKeyValue(key='uppercase_text', value='True'), TransactionKeyValue(key='Transactions._id', value=transaction_inst._id) ])
def _handle_download(self, event: Event) -> None: """Handle the download of the data to the download directory.""" output_path = os.path.join(script_config.data_dir, event.event_id, script_config.output_dirs[0].directory) down_path = os.path.join(script_config.data_dir, event.event_id, 'download') if os.path.isdir(down_path): # pragma: no cover just sanity condition should never happen rmtree(down_path) os.makedirs(output_path) os.makedirs(down_path) file_insts = File.from_cloudevents_model(event) # just make sure we have everything and the file objs are closed with _redirect_stdout_stderr(output_path, 'download-'): self.downloader_runner.download(down_path, file_insts, timeout=script_config.download_timeout)
def test_exceptions(self): """Test the exceptions classes.""" exception = InvalidModelProxEventHandlerError( Event(self.event_data), File(name='some_file_name.txt', path='some_file_path.txt'), AssertionError('fake error')) self.assertEqual( 'proxymod model for file \'some_file_name.txt\' is invalid: fake error', str(exception)) exception = ConfigNotFoundProxEventHandlerError( Event(self.event_data), 'config_1') self.assertEqual('proxymod configuration \'config_1\' not found', str(exception)) exception = InvalidConfigProxEventHandlerError(Event(self.event_data), 'config_1', {}) self.assertEqual('proxymod configuration \'config_1\' is invalid', str(exception))
def test_local_downloader_runner(self): """Generate a local temporary directory and download to it.""" with tempfile.TemporaryDirectory() as basedir_name: os.makedirs(os.path.join(basedir_name, 'filepath')) f_data = 'Hello, world!' with open(os.path.join(basedir_name, 'filepath', 'filename.ext'), mode='w') as test_file: test_file.write(f_data) with open(os.path.join(basedir_name, 'filepath', 'filename.ext'), mode='r') as test_file: self.assertEqual(f_data, test_file.read()) downloader_runner = LocalDownloaderRunner(basedir_name) with tempfile.TemporaryDirectory() as downloader_tempdir_name: openers = downloader_runner.download( downloader_tempdir_name, files=[File(name='filename.ext', subdir='filepath')]) self.assertEqual(1, len(openers)) with openers[0]() as test_file: self.assertEqual(f_data, test_file.read())
def test_local_runners(self): """Test local runners.""" transaction = Transaction(submitter=1, instrument=1, project=1) transaction_key_values = [ TransactionKeyValue(key='Transactions._id', value=1), ] files = [File(name='filename.ext', subdir='filepath')] file_strs = ['Hello, world!'] self.assertEqual(len(files), len(file_strs)) with tempfile.TemporaryDirectory() as basedir_name: for file, file_str in zip(files, file_strs): os.makedirs( os.path.join(basedir_name, os.path.dirname(file.path))) with open(os.path.join(basedir_name, file.path), mode='w') as file_data: file_data.write(file_str) with open(os.path.join(basedir_name, file.path), mode='r') as file_data: self.assertEqual(file_str, file_data.read()) downloader_runner = LocalDownloaderRunner(basedir_name) with tempfile.TemporaryDirectory() as downloader_tempdir_name: openers = downloader_runner.download(downloader_tempdir_name, files=files) self.assertEqual(len(files), len(openers)) with tempfile.TemporaryDirectory() as uploader_tempdir_name: uploader_runner = LocalUploaderRunner() for file, opener in zip(files, openers): os.makedirs( os.path.join(uploader_tempdir_name, os.path.dirname(file.path))) with opener() as orig_f: with open(os.path.join(uploader_tempdir_name, file.path), mode='w') as new_f: new_f.write(orig_f.read().upper()) with opener() as orig_f: with open(os.path.join(uploader_tempdir_name, file.path), mode='r') as new_f: self.assertEqual(orig_f.read().upper(), new_f.read()) (bundle, job_id, state) = uploader_runner.upload( uploader_tempdir_name, transaction=transaction, transaction_key_values=transaction_key_values) self.assertTrue(bundle.md_obj.is_valid()) self.assertEqual(len(files), len(bundle.file_data)) for file, file_data in zip(files, bundle.file_data): self.assertEqual( 'data/{}'.format( file.path.replace(os.path.sep, '/')), file_data.get('name', None)) self.assertEqual(None, job_id) self.assertEqual({}, state)
def handle(self, event: Event) -> None: """Handle a CloudEvents notification. Args: event (cloudevents.model.Event): The CloudEvents notification to handle. """ # Extract the metadata descriptions for the Pacifica transaction, # transaction key-values and files from the payload of the CloudEvents # notification. # transaction_inst = Transaction.from_cloudevents_model( event) # type: pacifica.dispatcher.models.Transaction transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model( event ) # type: typing.List[pacifica.dispatcher.models.TransactionKeyValue] file_insts = File.from_cloudevents_model( event) # type: typing.List[pacifica.dispatcher.models.File] # Create a temporary directory for the files that will be downloaded by # the downloader runner. # with tempfile.TemporaryDirectory() as downloader_tempdir_name: # Create a temporary directory for the files that will be uploaded # by the uploader runner. # with tempfile.TemporaryDirectory() as uploader_tempdir_name: # Download the files to the temporary directory using the # downloader runner. # # The return value for this function call is a list of callables, # where each callable delegates to the ``open`` built-in # function and returns an IO object. # # The ordering of the return value is the same as that of the # list of metadata descriptions for Pacifica files. # file_openers = self.downloader_runner.download( downloader_tempdir_name, file_insts ) # type: typing.List[typing.Callable[[], typing.TextIO]] # Use the ``zip`` built-in function to associate the metadata # descriptions for Pacifica files with the callables. # for file_inst, file_opener in zip(file_insts, file_openers): # Open the original file. # with file_opener() as file: # Open a new file in write-only mode that is to be # uploaded by the uploader runner. # # In this example, the relative path to the new file is # the same relative path to the original file. # with open(os.path.join(uploader_tempdir_name, file_inst.path), mode='w') as new_file: # Read the content of the original file and then # write a copy of the content with all the cased # characters converted to uppercase. new_file.write(file.read().upper()) # Construct the metadata description for the new Pacifica # transaction. # # In this example, the "submitter", "instrument" and "project" # attributes from the original metadata description are reused. # new_transaction_inst = Transaction( submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, project=transaction_inst.project ) # type: pacifica.dispatcher.models.Transaction # Construct the metadata descriptions for the new Pacifica # key-values. # new_transaction_key_value_insts = [ # In this example, the relationship between the original and new # Pacifica transactions is asserted via the "Transactions._id" # key and its value, the ID for the original Pacifica # transaction. # # This is an example of the assertion of retrospective # provenance information. # TransactionKeyValue(key='Transactions._id', value=transaction_inst._id), # In this example, a second Pacifica transaction key-value # is asserted via the "example-key" and its value. # TransactionKeyValue(key='example-key', value='example-value'), ] # type: typing.List[pacifica.dispatcher.models.TransactionKeyValue] # Upload the files in the temporary directory using the uploader # runner. # # The return value for this function call is a tuple of the # uploader's bundle, the job ID for the upload, and the state of # the upload. # (_bundle, _job_id, _state) = self.uploader_runner.upload( uploader_tempdir_name, transaction=new_transaction_inst, transaction_key_values=new_transaction_key_value_insts ) # type: typing.Tuple[pacifica.uploader.bundler.Bundler, int, typing.Dict[str, typing.Any]]
def handle(self, event: Event) -> None: """Handle the proxymod event.""" transaction_inst = Transaction.from_cloudevents_model(event) transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(event) file_insts = File.from_cloudevents_model(event) config_by_config_id = _assert_valid_proxevent(transaction_key_value_insts, event) input_file_insts = _assert_valid_proxinputs(config_by_config_id, file_insts) model_file_insts = _assert_valid_proxmodels(file_insts) with tempfile.TemporaryDirectory() as downloader_tempdir_name: with tempfile.TemporaryDirectory() as uploader_tempdir_name: # model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts) with _redirect_stdout_stderr(uploader_tempdir_name, 'download-'): model_file_openers = self.downloader_runner.download( downloader_tempdir_name, model_file_insts) model_file_funcs = [] for model_file_inst, model_file_opener in zip(model_file_insts, model_file_openers): with model_file_opener() as file: try: name = os.path.splitext(model_file_inst.name)[0] spec = importlib.util.spec_from_file_location(name, file.name) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # NOTE Deliberately raise `AttributeError` if `name` does not exist. func = getattr(module, name) if callable(func): model_file_funcs.append(func) else: # pragma: no cover should do this later. # NOTE Deliberately raise `TypeError` by calling an uncallable. func() except Exception as reason: # pragma: no cover trying happy path first raise InvalidModelProxEventHandlerError(event, model_file_inst, reason) # input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts) with _redirect_stdout_stderr(uploader_tempdir_name, 'download-', 'a'): input_file_openers = self.downloader_runner.download( downloader_tempdir_name, input_file_insts) abspath_config_by_config_id = copy.deepcopy(config_by_config_id) for config_id, config in abspath_config_by_config_id.items(): if 'INPUTS' in config: if 'in_dir' in config['INPUTS']: for opener in input_file_openers: with opener() as file: config['INPUTS']['in_dir'] = os.path.abspath(os.path.dirname(file.name)) break if 'OUTPUTS' in config: if 'out_dir' in config['OUTPUTS']: config['OUTPUTS']['out_dir'] = os.path.abspath( os.path.join(uploader_tempdir_name, config['OUTPUTS']['out_dir'])) for config_id, config in config_by_config_id.items(): with open(os.path.join(uploader_tempdir_name, '{0}.ini'.format(config_id)), 'w') as config_file: config_file.write(_format_proxymod_config(config)) config_1_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False) config_1_file.write(bytes(_format_proxymod_config( abspath_config_by_config_id['config_1']), 'utf-8')) config_1_file.close() config_2_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False) config_2_file.write(bytes(_format_proxymod_config( abspath_config_by_config_id['config_2']), 'utf-8')) config_2_file.close() config_3_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False) config_3_file.write(bytes(_format_proxymod_config( abspath_config_by_config_id['config_3']), 'utf-8')) config_3_file.close() with _redirect_stdout_stderr(uploader_tempdir_name): inst_func_zip = zip(model_file_insts, model_file_funcs) for model_file_inst, model_file_func in inst_func_zip: try: model_file_func(config_1_file.name, config_2_file.name, config_3_file.name) except Exception as reason: # pragma: no cover happy path testing raise InvalidModelProxEventHandlerError( event, model_file_inst, reason) os.unlink(config_1_file.name) os.unlink(config_2_file.name) os.unlink(config_3_file.name) with _redirect_stdout_stderr(uploader_tempdir_name, 'upload-'): # pylint: disable=protected-access (_bundle, _job_id, _state) = self.uploader_runner.upload( uploader_tempdir_name, transaction=Transaction( submitter=transaction_inst.submitter, instrument=transaction_inst.instrument, project=transaction_inst.project ), transaction_key_values=[ TransactionKeyValue(key='Transactions._id', value=transaction_inst._id) ] )