def run_plugin_chain(input_data, plugin_list, processing_dir, mpi=False,
processes=["CPU0"], process=0):
"""Runs a chain of plugins
:param input_data: The input data to give to the chain
:type input_data: savu.data.structure.
:param plugin_list: Names of all the plugins to process in order.
:type plugin_list: list of str.
:param processing_dir: Location of the processing directory.
:type processing_dir: str.
:param mpi: Whether this is running in mpi, default is false.
:type mpi: bool.
"""
logging.debug("Starting run_plugin_chain")
in_data = input_data
output = None
count = 0
for plugin_name in plugin_list:
logging.debug("Loading plugin %s", plugin_name)
plugin = pu.load_plugin(plugin_name)
# generate somewhere for the data to go
file_name = os.path.join(processing_dir,
"%02i_%s.h5" % (count, plugin.name))
logging.debug("Creating output file : %s", file_name)
output = pu.create_output_data(plugin, in_data, file_name, plugin.name,
mpi)
plugin.set_parameters(None)
logging.debug("Starting processing plugin %s", plugin_name)
plugin.run_plugin(in_data, output, processes, process)
logging.debug("Completed processing plugin %s", plugin_name)
if in_data is not output:
in_data.complete()
in_data = output
if mpi:
logging.debug("MPI awaiting barrier")
MPI.COMM_WORLD.barrier()
count += 1
if output is not None:
output.complete()
def run_plugin_list(self, input_file, plugin_list, processing_dir, mpi=False,
processes=["CPU0"], process=0):
"""Runs a chain of plugins
:param input_data: The input data to give to the chain
:type input_data: savu.data.structure.
:param plugin_list: Plugin list.
:type plugin_list: savu.data.structure.PluginList.
:param processing_dir: Location of the processing directory.
:type processing_dir: str.
:param mpi: Whether this is running in mpi, default is false.
:type mpi: bool.
"""
input_data = pu.load_raw_data(input_file)
logging.debug("Running plugin list, just a check")
filename = os.path.basename(input_data.backing_file.filename)
filename = os.path.splitext(filename)[0]
output_filename = \
os.path.join(processing_dir,
"%s_processed_%s.nxs" % (filename,
time.strftime("%Y%m%d%H%M%S")))
if process == 0:
logging.debug("Running process List.save_list_to_file")
plugin_list.save_list_to_file(output_filename)
in_data = input_data
output = None
logging.debug("generating all output files")
files = []
count = 0
for plugin_dict in plugin_list.plugin_list:
logging.debug("Loading plugin %s", plugin_dict['id'])
plugin = pu.load_plugin(plugin_dict['id'])
# generate somewhere for the data to go
file_name = os.path.join(processing_dir,
"%s%02i_%s.h5" % (plugin_list.name, count,
plugin_dict['id']))
group_name = "%i-%s" % (count, plugin.name)
logging.debug("Creating output file %s", file_name)
output = pu.create_output_data(plugin, in_data, file_name, group_name,
mpi)
files.append(output)
in_data = output
count += 1
logging.debug("processing Plugins")
in_data = input_data
count = 0
for plugin_dict in plugin_list.plugin_list:
logging.debug("Loading plugin %s", plugin_dict['id'])
plugin = pu.load_plugin(plugin_dict['id'])
output = files[count]
plugin.set_parameters(plugin_dict['data'])
logging.debug("Starting processing plugin %s", plugin_dict['id'])
plugin.run_plugin(in_data, output, processes, process, self)
logging.debug("Completed processing plugin %s", plugin_dict['id'])
if in_data is not output:
in_data.complete()
in_data = output
if mpi:
logging.debug("Blocking till all processes complete")
MPI.COMM_WORLD.Barrier()
if plugin == 0:
cite_info = plugin.get_citation_information()
if cite_info is not None:
plugin_list.add_plugin_citation(output_filename, count,
cite_info)
group_name = "%i-%s" % (count, plugin.name)
plugin_list.add_intermediate_data_link(output_filename,
output, group_name)
count += 1
if output is not None:
output.complete()
