def handle(self, event: Event) -> None:
"""
Example handle event.
This handler downloads all files in the event.
Converts the files to uppercase and uploads them back to Pacifica.
"""
transaction_inst = Transaction.from_cloudevents_model(event)
transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(
event)
file_insts = File.from_cloudevents_model(event)
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
for file_opener in self.downloader_runner.download(
downloader_tempdir_name, file_insts):
with file_opener() as file_fd:
with open(
os.path.join(uploader_tempdir_name,
file_fd.name), 'w') as wfile_fd:
wfile_fd.write(file_fd.read().upper())
(_bundle, _job_id, _state) = self.uploader_runner.upload(
uploader_tempdir_name,
transaction=Transaction(
submitter=transaction_inst.submitter,
instrument=transaction_inst.instrument,
project=transaction_inst.project),
transaction_key_values=[
TransactionKeyValue(key='uppercase_text',
value='True'),
TransactionKeyValue(key='Transactions._id',
value=transaction_inst._id)
])
def _handle_download(self, event: Event) -> None:
"""Handle the download of the data to the download directory."""
output_path = os.path.join(script_config.data_dir, event.event_id, script_config.output_dirs[0].directory)
down_path = os.path.join(script_config.data_dir, event.event_id, 'download')
if os.path.isdir(down_path): # pragma: no cover just sanity condition should never happen
rmtree(down_path)
os.makedirs(output_path)
os.makedirs(down_path)
file_insts = File.from_cloudevents_model(event)
# just make sure we have everything and the file objs are closed
with _redirect_stdout_stderr(output_path, 'download-'):
self.downloader_runner.download(down_path, file_insts, timeout=script_config.download_timeout)
def handle(self, event: Event) -> None:
"""Handle a CloudEvents notification.
Args:
event (cloudevents.model.Event): The CloudEvents notification to
handle.
"""
# Extract the metadata descriptions for the Pacifica transaction,
# transaction key-values and files from the payload of the CloudEvents
# notification.
#
transaction_inst = Transaction.from_cloudevents_model(
event) # type: pacifica.dispatcher.models.Transaction
transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(
event
) # type: typing.List[pacifica.dispatcher.models.TransactionKeyValue]
file_insts = File.from_cloudevents_model(
event) # type: typing.List[pacifica.dispatcher.models.File]
# Create a temporary directory for the files that will be downloaded by
# the downloader runner.
#
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
# Create a temporary directory for the files that will be uploaded
# by the uploader runner.
#
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
# Download the files to the temporary directory using the
# downloader runner.
#
# The return value for this function call is a list of callables,
# where each callable delegates to the ``open`` built-in
# function and returns an IO object.
#
# The ordering of the return value is the same as that of the
# list of metadata descriptions for Pacifica files.
#
file_openers = self.downloader_runner.download(
downloader_tempdir_name, file_insts
) # type: typing.List[typing.Callable[[], typing.TextIO]]
# Use the ``zip`` built-in function to associate the metadata
# descriptions for Pacifica files with the callables.
#
for file_inst, file_opener in zip(file_insts, file_openers):
# Open the original file.
#
with file_opener() as file:
# Open a new file in write-only mode that is to be
# uploaded by the uploader runner.
#
# In this example, the relative path to the new file is
# the same relative path to the original file.
#
with open(os.path.join(uploader_tempdir_name,
file_inst.path),
mode='w') as new_file:
# Read the content of the original file and then
# write a copy of the content with all the cased
# characters converted to uppercase.
new_file.write(file.read().upper())
# Construct the metadata description for the new Pacifica
# transaction.
#
# In this example, the "submitter", "instrument" and "project"
# attributes from the original metadata description are reused.
#
new_transaction_inst = Transaction(
submitter=transaction_inst.submitter,
instrument=transaction_inst.instrument,
project=transaction_inst.project
) # type: pacifica.dispatcher.models.Transaction
# Construct the metadata descriptions for the new Pacifica
# key-values.
#
new_transaction_key_value_insts = [
# In this example, the relationship between the original and new
# Pacifica transactions is asserted via the "Transactions._id"
# key and its value, the ID for the original Pacifica
# transaction.
#
# This is an example of the assertion of retrospective
# provenance information.
#
TransactionKeyValue(key='Transactions._id',
value=transaction_inst._id),
# In this example, a second Pacifica transaction key-value
# is asserted via the "example-key" and its value.
#
TransactionKeyValue(key='example-key',
value='example-value'),
] # type: typing.List[pacifica.dispatcher.models.TransactionKeyValue]
# Upload the files in the temporary directory using the uploader
# runner.
#
# The return value for this function call is a tuple of the
# uploader's bundle, the job ID for the upload, and the state of
# the upload.
#
(_bundle, _job_id, _state) = self.uploader_runner.upload(
uploader_tempdir_name,
transaction=new_transaction_inst,
transaction_key_values=new_transaction_key_value_insts
) # type: typing.Tuple[pacifica.uploader.bundler.Bundler, int, typing.Dict[str, typing.Any]]
def handle(self, event: Event) -> None:
"""Handle the proxymod event."""
transaction_inst = Transaction.from_cloudevents_model(event)
transaction_key_value_insts = TransactionKeyValue.from_cloudevents_model(event)
file_insts = File.from_cloudevents_model(event)
config_by_config_id = _assert_valid_proxevent(transaction_key_value_insts, event)
input_file_insts = _assert_valid_proxinputs(config_by_config_id, file_insts)
model_file_insts = _assert_valid_proxmodels(file_insts)
with tempfile.TemporaryDirectory() as downloader_tempdir_name:
with tempfile.TemporaryDirectory() as uploader_tempdir_name:
# model_file_openers = self.downloader_runner.download(downloader_tempdir_name, model_file_insts)
with _redirect_stdout_stderr(uploader_tempdir_name, 'download-'):
model_file_openers = self.downloader_runner.download(
downloader_tempdir_name, model_file_insts)
model_file_funcs = []
for model_file_inst, model_file_opener in zip(model_file_insts, model_file_openers):
with model_file_opener() as file:
try:
name = os.path.splitext(model_file_inst.name)[0]
spec = importlib.util.spec_from_file_location(name, file.name)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
# NOTE Deliberately raise `AttributeError` if `name` does not exist.
func = getattr(module, name)
if callable(func):
model_file_funcs.append(func)
else: # pragma: no cover should do this later.
# NOTE Deliberately raise `TypeError` by calling an uncallable.
func()
except Exception as reason: # pragma: no cover trying happy path first
raise InvalidModelProxEventHandlerError(event, model_file_inst, reason)
# input_file_openers = self.downloader_runner.download(downloader_tempdir_name, input_file_insts)
with _redirect_stdout_stderr(uploader_tempdir_name, 'download-', 'a'):
input_file_openers = self.downloader_runner.download(
downloader_tempdir_name, input_file_insts)
abspath_config_by_config_id = copy.deepcopy(config_by_config_id)
for config_id, config in abspath_config_by_config_id.items():
if 'INPUTS' in config:
if 'in_dir' in config['INPUTS']:
for opener in input_file_openers:
with opener() as file:
config['INPUTS']['in_dir'] = os.path.abspath(os.path.dirname(file.name))
break
if 'OUTPUTS' in config:
if 'out_dir' in config['OUTPUTS']:
config['OUTPUTS']['out_dir'] = os.path.abspath(
os.path.join(uploader_tempdir_name, config['OUTPUTS']['out_dir']))
for config_id, config in config_by_config_id.items():
with open(os.path.join(uploader_tempdir_name, '{0}.ini'.format(config_id)), 'w') as config_file:
config_file.write(_format_proxymod_config(config))
config_1_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False)
config_1_file.write(bytes(_format_proxymod_config(
abspath_config_by_config_id['config_1']), 'utf-8'))
config_1_file.close()
config_2_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False)
config_2_file.write(bytes(_format_proxymod_config(
abspath_config_by_config_id['config_2']), 'utf-8'))
config_2_file.close()
config_3_file = tempfile.NamedTemporaryFile(suffix='.ini', delete=False)
config_3_file.write(bytes(_format_proxymod_config(
abspath_config_by_config_id['config_3']), 'utf-8'))
config_3_file.close()
with _redirect_stdout_stderr(uploader_tempdir_name):
inst_func_zip = zip(model_file_insts, model_file_funcs)
for model_file_inst, model_file_func in inst_func_zip:
try:
model_file_func(config_1_file.name,
config_2_file.name, config_3_file.name)
except Exception as reason: # pragma: no cover happy path testing
raise InvalidModelProxEventHandlerError(
event, model_file_inst, reason)
os.unlink(config_1_file.name)
os.unlink(config_2_file.name)
os.unlink(config_3_file.name)
with _redirect_stdout_stderr(uploader_tempdir_name, 'upload-'):
# pylint: disable=protected-access
(_bundle, _job_id, _state) = self.uploader_runner.upload(
uploader_tempdir_name, transaction=Transaction(
submitter=transaction_inst.submitter,
instrument=transaction_inst.instrument,
project=transaction_inst.project
), transaction_key_values=[
TransactionKeyValue(key='Transactions._id', value=transaction_inst._id)
]
)