def test_remote_downloader_runner(self):
"""Test the downloader runner class."""
# Hash sum the README.md at root of repo (it's what we use to system test).
hash_obj = hashlib.sha1()
with open(os.path.join('..', 'README.md'), 'r') as readme_fd:
hash_obj.update(bytes(readme_fd.read(), 'utf8'))
hashsum = hash_obj.hexdigest()
with tempfile.TemporaryDirectory() as basedir_name:
downloader_runner = RemoteDownloaderRunner(Downloader())
openers = downloader_runner.download(basedir_name,
files=[
File(_id=103,
name='foo.txt',
subdir='a/b',
hashtype='sha1',
hashsum=hashsum),
File(_id=104,
name='bar\u00e9.txt',
subdir='a/b/\u00e9',
hashtype='sha1',
hashsum=hashsum)
])
self.assertEqual(2, len(openers))
with openers[0]() as readme_fd:
self.assertTrue('Pacifica Dispatcher' in readme_fd.read())
def handle(self, event: Event) -> None:
"""
Example handle event.
This handler downloads all files in the event.
Converts the files to uppercase and uploads them back to Pacifica.
"""
transaction_inst = Transaction.from_cloudevents_model(event)
transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(
event)
file_insts = File.from_cloudevents_model(event)
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
for file_opener in self.downloader_runner.download(
downloader_tempdir_name, file_insts):
with file_opener() as file_fd:
with open(
os.path.join(uploader_tempdir_name,
file_fd.name), 'w') as wfile_fd:
wfile_fd.write(file_fd.read().upper())
(_bundle, _job_id, _state) = self.uploader_runner.upload(
uploader_tempdir_name,
transaction=Transaction(
submitter=transaction_inst.submitter,
instrument=transaction_inst.instrument,
project=transaction_inst.project),
transaction_key_values=[
TransactionKeyValue(key='uppercase_text',
value='True'),
TransactionKeyValue(key='Transactions._id',
value=transaction_inst._id)
])
def _handle_download(self, event: Event) -> None:
"""Handle the download of the data to the download directory."""
output_path = os.path.join(script_config.data_dir, event.event_id, script_config.output_dirs[0].directory)
down_path = os.path.join(script_config.data_dir, event.event_id, 'download')
if os.path.isdir(down_path): # pragma: no cover just sanity condition should never happen
rmtree(down_path)
os.makedirs(output_path)
os.makedirs(down_path)
file_insts = File.from_cloudevents_model(event)
# just make sure we have everything and the file objs are closed
with _redirect_stdout_stderr(output_path, 'download-'):
self.downloader_runner.download(down_path, file_insts, timeout=script_config.download_timeout)
def test_exceptions(self):
"""Test the exceptions classes."""
exception = InvalidModelProxEventHandlerError(
Event(self.event_data),
File(name='some_file_name.txt', path='some_file_path.txt'),
AssertionError('fake error'))
self.assertEqual(
'proxymod model for file \'some_file_name.txt\' is invalid: fake error',
str(exception))
exception = ConfigNotFoundProxEventHandlerError(
Event(self.event_data), 'config_1')
self.assertEqual('proxymod configuration \'config_1\' not found',
str(exception))
exception = InvalidConfigProxEventHandlerError(Event(self.event_data),
'config_1', {})
self.assertEqual('proxymod configuration \'config_1\' is invalid',
str(exception))
def test_local_downloader_runner(self):
"""Generate a local temporary directory and download to it."""
with tempfile.TemporaryDirectory() as basedir_name:
os.makedirs(os.path.join(basedir_name, 'filepath'))
f_data = 'Hello, world!'
with open(os.path.join(basedir_name, 'filepath', 'filename.ext'),
mode='w') as test_file:
test_file.write(f_data)
with open(os.path.join(basedir_name, 'filepath', 'filename.ext'),
mode='r') as test_file:
self.assertEqual(f_data, test_file.read())
downloader_runner = LocalDownloaderRunner(basedir_name)
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
openers = downloader_runner.download(
downloader_tempdir_name,
files=[File(name='filename.ext', subdir='filepath')])
self.assertEqual(1, len(openers))
with openers[0]() as test_file:
self.assertEqual(f_data, test_file.read())
def test_local_runners(self):
"""Test local runners."""
transaction = Transaction(submitter=1, instrument=1, project=1)
transaction_key_values = [
TransactionKeyValue(key='Transactions._id', value=1),
]
files = [File(name='filename.ext', subdir='filepath')]
file_strs = ['Hello, world!']
self.assertEqual(len(files), len(file_strs))
with tempfile.TemporaryDirectory() as basedir_name:
for file, file_str in zip(files, file_strs):
os.makedirs(
os.path.join(basedir_name, os.path.dirname(file.path)))
with open(os.path.join(basedir_name, file.path),
mode='w') as file_data:
file_data.write(file_str)
with open(os.path.join(basedir_name, file.path),
mode='r') as file_data:
self.assertEqual(file_str, file_data.read())
downloader_runner = LocalDownloaderRunner(basedir_name)
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
openers = downloader_runner.download(downloader_tempdir_name,
files=files)
self.assertEqual(len(files), len(openers))
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
uploader_runner = LocalUploaderRunner()
for file, opener in zip(files, openers):
os.makedirs(
os.path.join(uploader_tempdir_name,
os.path.dirname(file.path)))
with opener() as orig_f:
with open(os.path.join(uploader_tempdir_name,
file.path),
mode='w') as new_f:
new_f.write(orig_f.read().upper())
with opener() as orig_f:
with open(os.path.join(uploader_tempdir_name,
file.path),
mode='r') as new_f:
self.assertEqual(orig_f.read().upper(),
new_f.read())
(bundle, job_id, state) = uploader_runner.upload(
uploader_tempdir_name,
transaction=transaction,
transaction_key_values=transaction_key_values)
self.assertTrue(bundle.md_obj.is_valid())
self.assertEqual(len(files), len(bundle.file_data))
for file, file_data in zip(files, bundle.file_data):
self.assertEqual(
'data/{}'.format(
file.path.replace(os.path.sep, '/')),
file_data.get('name', None))
self.assertEqual(None, job_id)
self.assertEqual({}, state)
def handle(self, event: Event) -> None:
"""Handle a CloudEvents notification.
Args:
event (cloudevents.model.Event): The CloudEvents notification to
handle.
"""
# Extract the metadata descriptions for the Pacifica transaction,
# transaction key-values and files from the payload of the CloudEvents
# notification.
#
transaction_inst = Transaction.from_cloudevents_model(
event) # type: pacifica.dispatcher.models.Transaction
transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(
event
) # type: typing.List[pacifica.dispatcher.models.TransactionKeyValue]
file_insts = File.from_cloudevents_model(
event) # type: typing.List[pacifica.dispatcher.models.File]
# Create a temporary directory for the files that will be downloaded by
# the downloader runner.
#
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
# Create a temporary directory for the files that will be uploaded
# by the uploader runner.
#
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
# Download the files to the temporary directory using the
# downloader runner.
#
# The return value for this function call is a list of callables,
# where each callable delegates to the ``open`` built-in
# function and returns an IO object.
#
# The ordering of the return value is the same as that of the
# list of metadata descriptions for Pacifica files.
#
file_openers = self.downloader_runner.download(
downloader_tempdir_name, file_insts
) # type: typing.List[typing.Callable[[], typing.TextIO]]
# Use the ``zip`` built-in function to associate the metadata
# descriptions for Pacifica files with the callables.
#
for file_inst, file_opener in zip(file_insts, file_openers):
# Open the original file.
#
with file_opener() as file:
# Open a new file in write-only mode that is to be
# uploaded by the uploader runner.
#
# In this example, the relative path to the new file is
# the same relative path to the original file.
#
with open(os.path.join(uploader_tempdir_name,
file_inst.path),
mode='w') as new_file:
# Read the content of the original file and then
# write a copy of the content with all the cased
# characters converted to uppercase.
new_file.write(file.read().upper())
# Construct the metadata description for the new Pacifica
# transaction.
#
# In this example, the "submitter", "instrument" and "project"
# attributes from the original metadata description are reused.
#
new_transaction_inst = Transaction(
submitter=transaction_inst.submitter,
instrument=transaction_inst.instrument,
project=transaction_inst.project
) # type: pacifica.dispatcher.models.Transaction
# Construct the metadata descriptions for the new Pacifica
# key-values.
#
new_transaction_key_value_insts = [
# In this example, the relationship between the original and new
# Pacifica transactions is asserted via the "Transactions._id"
# key and its value, the ID for the original Pacifica
# transaction.
#
# This is an example of the assertion of retrospective
# provenance information.
#
TransactionKeyValue(key='Transactions._id',
value=transaction_inst._id),
# In this example, a second Pacifica transaction key-value
# is asserted via the "example-key" and its value.
#
TransactionKeyValue(key='example-key',
value='example-value'),
] # type: typing.List[pacifica.dispatcher.models.TransactionKeyValue]
# Upload the files in the temporary directory using the uploader
# runner.
#
# The return value for this function call is a tuple of the
# uploader's bundle, the job ID for the upload, and the state of
# the upload.
#
(_bundle, _job_id, _state) = self.uploader_runner.upload(
uploader_tempdir_name,
transaction=new_transaction_inst,
transaction_key_values=new_transaction_key_value_insts
) # type: typing.Tuple[pacifica.uploader.bundler.Bundler, int, typing.Dict[str, typing.Any]]
def handle(self, event: Event) -> None:
"""Handle the proxymod event."""
transaction_inst = Transaction.from_cloudevents_model(event)
transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(event)
file_insts = File.from_cloudevents_model(event)
config_by_config_id = _assert_valid_proxevent(transaction_key_value_insts, event)
input_file_insts = _assert_valid_proxinputs(config_by_config_id, file_insts)
model_file_insts = _assert_valid_proxmodels(file_insts)
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
# model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts)
with _redirect_stdout_stderr(uploader_tempdir_name, 'download-'):
model_file_openers = self.downloader_runner.download(
downloader_tempdir_name, model_file_insts)
model_file_funcs = []
for model_file_inst, model_file_opener in zip(model_file_insts, model_file_openers):
with model_file_opener() as file:
try:
name = os.path.splitext(model_file_inst.name)[0]
spec = importlib.util.spec_from_file_location(name, file.name)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# NOTE Deliberately raise `AttributeError` if `name` does not exist.
func = getattr(module, name)
if callable(func):
model_file_funcs.append(func)
else: # pragma: no cover should do this later.
# NOTE Deliberately raise `TypeError` by calling an uncallable.
func()
except Exception as reason: # pragma: no cover trying happy path first
raise InvalidModelProxEventHandlerError(event, model_file_inst, reason)
# input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts)
with _redirect_stdout_stderr(uploader_tempdir_name, 'download-', 'a'):
input_file_openers = self.downloader_runner.download(
downloader_tempdir_name, input_file_insts)
abspath_config_by_config_id = copy.deepcopy(config_by_config_id)
for config_id, config in abspath_config_by_config_id.items():
if 'INPUTS' in config:
if 'in_dir' in config['INPUTS']:
for opener in input_file_openers:
with opener() as file:
config['INPUTS']['in_dir'] = os.path.abspath(os.path.dirname(file.name))
break
if 'OUTPUTS' in config:
if 'out_dir' in config['OUTPUTS']:
config['OUTPUTS']['out_dir'] = os.path.abspath(
os.path.join(uploader_tempdir_name, config['OUTPUTS']['out_dir']))
for config_id, config in config_by_config_id.items():
with open(os.path.join(uploader_tempdir_name, '{0}.ini'.format(config_id)), 'w') as config_file:
config_file.write(_format_proxymod_config(config))
config_1_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False)
config_1_file.write(bytes(_format_proxymod_config(
abspath_config_by_config_id['config_1']), 'utf-8'))
config_1_file.close()
config_2_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False)
config_2_file.write(bytes(_format_proxymod_config(
abspath_config_by_config_id['config_2']), 'utf-8'))
config_2_file.close()
config_3_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False)
config_3_file.write(bytes(_format_proxymod_config(
abspath_config_by_config_id['config_3']), 'utf-8'))
config_3_file.close()
with _redirect_stdout_stderr(uploader_tempdir_name):
inst_func_zip = zip(model_file_insts, model_file_funcs)
for model_file_inst, model_file_func in inst_func_zip:
try:
model_file_func(config_1_file.name,
config_2_file.name, config_3_file.name)
except Exception as reason: # pragma: no cover happy path testing
raise InvalidModelProxEventHandlerError(
event, model_file_inst, reason)
os.unlink(config_1_file.name)
os.unlink(config_2_file.name)
os.unlink(config_3_file.name)
with _redirect_stdout_stderr(uploader_tempdir_name, 'upload-'):
# pylint: disable=protected-access
(_bundle, _job_id, _state) = self.uploader_runner.upload(
uploader_tempdir_name, transaction=Transaction(
submitter=transaction_inst.submitter,
instrument=transaction_inst.instrument,
project=transaction_inst.project
), transaction_key_values=[
TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)
]
)