def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
self.exp = Experiment(self.options)
plugin_list = self.exp.meta_data.plugin_list.plugin_list
logging.info("run_plugin_list: 1")
self.exp._barrier()
self._run_plugin_list_check(plugin_list)
logging.info("run_plugin_list: 2")
self.exp._barrier()
expInfo = self.exp.meta_data
logging.debug("Running process List.save_list_to_file")
expInfo.plugin_list._save_plugin_list(
expInfo.get_meta_data("nxs_filename"), exp=self.exp)
logging.info("run_plugin_list: 3")
self.exp._barrier()
self._transport_run_plugin_list()
logging.info("run_plugin_list: 4")
self.exp._barrier()
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
self.exp.nxs_file.close()
return self.exp
def __init__(self, options):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
self._transport_control_setup(options)
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
self.exp = Experiment(options)
def __init__(self, options, name='PluginRunner'):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
super(PluginRunner, self).__init__()
# ********* transport function ***********
self._transport_initialise(options)
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
self.exp = Experiment(options)
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
self.exp = Experiment(self.options)
plugin_list = self.exp.meta_data.plugin_list.plugin_list
logging.info("run_plugin_list: 1")
self.exp._barrier()
self._run_plugin_list_check(plugin_list)
logging.info("run_plugin_list: 2")
self.exp._barrier()
expInfo = self.exp.meta_data
logging.debug("Running process List.save_list_to_file")
expInfo.plugin_list._save_plugin_list(
expInfo.get_meta_data("nxs_filename"), exp=self.exp)
logging.info("run_plugin_list: 3")
self.exp._barrier()
self._transport_run_plugin_list()
logging.info("run_plugin_list: 4")
self.exp._barrier()
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
self.exp.nxs_file.close()
return self.exp
def fake_plugin_runner(options):
# Stripped down version of the plugin runner
# Loads the loader plugin but stops before processing plugins
plugin_runner = PluginRunner(options)
plugin_runner.exp = Experiment(options)
plugin_list = plugin_runner.exp.meta_data.plugin_list.plugin_list
exp = plugin_runner.exp
pu.plugin_loader(exp, plugin_list[0])
exp._set_nxs_file()
return plugin_runner
def __init__(self, options, name='PluginRunner'):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
super(PluginRunner, self).__init__()
# ********* transport function ***********
self._transport_initialise(options)
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
self.exp = Experiment(options)
def create_chunking_instance(self, current_list, nnext_list, nProcs):
current = self.create_pattern('a', current_list)
nnext = self.create_pattern('b', nnext_list)
options = tu.set_experiment('tomoRaw')
options['processes'] = range(nProcs)
# set a dummy process list
options['process_file'] = \
tu.get_test_process_path('basic_tomo_process.nxs')
exp = Experiment(options)
test_dict = {'current': current, 'next': nnext}
chunking = Chunking(exp, test_dict)
return chunking
def plugin_runner_load_plugin(options):
plugin_runner = PluginRunner(options)
plugin_runner.exp = Experiment(options)
plugin_list = plugin_runner.exp.meta_data.plugin_list.plugin_list
exp = plugin_runner.exp
pu.plugin_loader(exp, plugin_list[0])
exp._set_nxs_file()
plugin_dict = plugin_list[1]
plugin = pu.get_plugin(plugin_dict['id'])
plugin.exp = exp
return plugin
def plugin_runner_real_plugin_run(options):
plugin_runner = PluginRunner(options)
plugin_runner.exp = Experiment(options)
plugin_list = plugin_runner.exp.meta_data.plugin_list.plugin_list
plugin_runner._run_plugin_list_check(plugin_list)
exp = plugin_runner.exp
pu.plugin_loader(exp, plugin_list[0])
start_in_data = copy.deepcopy(exp.index['in_data'])
in_data = exp.index["in_data"][exp.index["in_data"].keys()[0]]
out_data_objs, stop = in_data._load_data(1)
exp._clear_data_objects()
exp.index['in_data'] = copy.deepcopy(start_in_data)
for key in out_data_objs[0]:
exp.index["out_data"][key] = out_data_objs[0][key]
plugin = pu.plugin_loader(exp, plugin_list[1])
plugin._run_plugin(exp, plugin_runner)
class PluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
self._transport_control_setup(options)
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
self.exp = Experiment(options)
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
plugin_list = self.exp.meta_data.plugin_list
self.exp._barrier()
self._run_plugin_list_check(plugin_list)
self.exp._barrier()
expInfo = self.exp.meta_data
logging.debug("Running process List.save_list_to_file")
expInfo.plugin_list._save_plugin_list(
expInfo.get_meta_data("nxs_filename"), exp=self.exp)
self.exp._barrier()
self._transport_run_plugin_list()
self.exp._barrier()
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
self.exp.nxs_file.close()
return self.exp
def _run_plugin_list_check(self, plugin_list):
""" Run the plugin list through the framework without executing the
main processing.
"""
self.exp._barrier()
plugin_list._check_loaders_and_savers()
self.__check_gpu()
self.exp._barrier()
self.__fake_plugin_list_run(plugin_list.plugin_list, check=True)
self.exp._barrier()
self.exp._clear_data_objects()
self.exp._barrier()
cu.user_message("Plugin list check complete!")
def __fake_plugin_list_run(self, plugin_list, **kwargs):
""" Run through the plugin list without any processing (setup only)\
and fill in missing dataset names.
"""
n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()
for i in range(n_loaders):
pu.plugin_loader(self.exp, plugin_list[i])
self.exp._set_nxs_filename()
check = kwargs.get('check', False)
for i in range(n_loaders, len(plugin_list) - 1):
self.exp._barrier()
plugin = pu.plugin_loader(self.exp, plugin_list[i], check=check)
plugin_list[i]['cite'] = plugin.get_citation_information()
plugin._clean_up()
self.exp._merge_out_data_to_in()
def __check_gpu(self):
""" Check if the process list contains GPU processes and determine if
GPUs exists. Add GPU processes to the processes list if required."""
if not self.exp.meta_data.plugin_list._contains_gpu_processes():
return
try:
import pynvml as pv
except:
logging.debug("pyNVML module not found")
raise Exception("pyNVML module not found")
try:
pv.nvmlInit()
count = int(pv.nvmlDeviceGetCount())
logging.debug("%s GPUs have been found.", count)
for i in range(count):
handle = pv.nvmlDeviceGetHandleByIndex(i)
if pv.nvmlDeviceGetComputeRunningProcesses(handle):
raise Exception("Unfortunately, GPU %i is busy. Try \
resubmitting the job to the queue." % i)
except:
logging.debug("No GPUs have been found.")
raise Exception("The process list contains GPU plugins, but "
" no GPUs have been found.")
self.__set_gpu_processes(count)
def __set_gpu_processes(self, count):
processes = self.exp.meta_data.get_meta_data('processes')
if not [i for i in processes if 'GPU' in i]:
logging.debug("GPU processes missing. GPUs found so adding them.")
cpus = ['CPU' + str(i) for i in range(count)]
gpus = ['GPU' + str(i) for i in range(count)]
for i in range(min(count, len(processes))):
processes[processes.index(cpus[i])] = gpus[i]
self.exp.meta_data.set_meta_data('processes', processes)
def _get_checkpoint(self, cfile=True):
exp = Experiment(self.get_options(cfile=cfile))
checkpoint = Checkpointing(exp)
return checkpoint
class PluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options, name='PluginRunner'):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
super(PluginRunner, self).__init__()
# ********* transport function ***********
self._transport_initialise(options)
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
self.exp = Experiment(options)
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
self.exp._setup(self)
plugin_list = self.exp.meta_data.plugin_list
logging.info('Running the plugin list check')
self._run_plugin_list_setup(plugin_list)
exp_coll = self.exp._get_collection()
n_plugins = plugin_list._get_n_processing_plugins()
# ********* transport function ***********
logging.info('Running transport_pre_plugin_list_run()')
self._transport_pre_plugin_list_run()
cp = self.exp.checkpoint
checkpoint_plugin = cp.get_checkpoint_plugin()
for i in range(checkpoint_plugin, n_plugins):
self.exp._set_experiment_for_current_plugin(i)
memory_before = cu.get_memory_usage_linux()
plugin_name = self.__run_plugin(exp_coll['plugin_dict'][i])
self.exp._barrier(msg='PluginRunner: plugin complete.')
memory_after = cu.get_memory_usage_linux()
logging.debug(
"{} memory usage before: {} MB, after: {} MB, change: {} MB".
format(plugin_name, memory_before, memory_after,
memory_after - memory_before))
# ********* transport functions ***********
# end the plugin run if savu has been killed
if self._transport_kill_signal():
self._transport_cleanup(i + 1)
break
self.exp._barrier(msg='PluginRunner: No kill signal... continue.')
cp.output_plugin_checkpoint()
# ********* transport function ***********
logging.info('Running transport_post_plugin_list_run')
self._transport_post_plugin_list_run()
# terminate any remaining datasets
for data in list(self.exp.index['in_data'].values()):
self._transport_terminate_dataset(data)
self.__output_final_message()
if self.exp.meta_data.get('email'):
cu.send_email(self.exp.meta_data.get('email'))
return self.exp
def __output_final_message(self):
kill = True if 'killsignal' in \
self.exp.meta_data.get_dictionary().keys() else False
msg = "interrupted by killsignal" if kill else "Complete"
stars = 40 if kill else 23
cu.user_message("*" * stars)
cu.user_message("* Processing " + msg + " *")
cu.user_message("*" * stars)
def __run_plugin(self, plugin_dict):
plugin = self._transport_load_plugin(self.exp, plugin_dict)
# ********* transport function ***********
self._transport_pre_plugin()
cu.user_message("*Running the %s plugin*" % plugin.name)
# ******** transport 'process' function is called inside here ********
plugin._run_plugin(self.exp, self) # plugin driver
self.exp._barrier(msg="Plugin returned from driver in Plugin Runner")
cu._output_summary(self.exp.meta_data.get("mpi"), plugin)
plugin._clean_up()
finalise = self.exp._finalise_experiment_for_current_plugin()
# ********* transport function ***********
self._transport_post_plugin()
for data in finalise['remove'] + finalise['replace']:
# ********* transport function ***********
self._transport_terminate_dataset(data)
self.exp._reorganise_datasets(finalise)
return plugin.name
def _run_plugin_list_setup(self, plugin_list):
""" Run the plugin list through the framework without executing the
main processing.
"""
plugin_list._check_loaders()
self.__check_gpu()
n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()
n_plugins = plugin_list._get_n_processing_plugins()
plist = plugin_list.plugin_list
# set loaders
for i in range(n_loaders):
pu.plugin_loader(self.exp, plist[i])
self.exp._set_initial_datasets()
# run all plugin setup methods and store information in experiment
# collection
count = 0
for plugin_dict in plist[n_loaders:n_loaders + n_plugins]:
self.__plugin_setup(plugin_dict, count)
count += 1
plugin_list._add_missing_savers(self.exp)
# ********* transport function ***********
self._transport_update_plugin_list()
# check added savers
for plugin_dict in plist[n_loaders + count:]:
self.__plugin_setup(plugin_dict, count)
count += 1
self.exp._reset_datasets()
self.exp._finalise_setup(plugin_list)
cu.user_message("Plugin list check complete!")
def __plugin_setup(self, plugin_dict, count):
self.exp.meta_data.set("nPlugin", count)
plugin = pu.plugin_loader(self.exp, plugin_dict, check=True)
plugin._revert_preview(plugin.get_in_datasets())
plugin_dict['cite'] = plugin.get_citation_information()
plugin._clean_up()
self.exp._update(plugin_dict)
self.exp._merge_out_data_to_in()
def __check_gpu(self):
""" Check if the process list contains GPU processes and determine if
GPUs exists. Add GPU processes to the processes list if required."""
if not self.exp.meta_data.plugin_list._contains_gpu_processes():
return
try:
import pynvml as pv
except Exception:
logging.debug("pyNVML module not found")
raise Exception("pyNVML module not found")
try:
pv.nvmlInit()
count = int(pv.nvmlDeviceGetCount())
if count == 0:
raise Exception("No GPUs found")
logging.debug("%s GPUs have been found.", count)
if not self.exp.meta_data.get('test_state'):
for i in range(count):
handle = pv.nvmlDeviceGetHandleByIndex(i)
if pv.nvmlDeviceGetComputeRunningProcesses(handle):
raise Exception("Unfortunately, GPU %i is busy. Try \
resubmitting the job to the queue." % i)
except Exception as e:
raise Exception("Unable to run GPU plugins: %s", str(e))
self.__set_gpu_processes(count)
def __set_gpu_processes(self, count):
processes = self.exp.meta_data.get('processes')
if not [i for i in processes if 'GPU' in i]:
logging.debug("GPU processes missing. GPUs found so adding them.")
cpus = ['CPU' + str(i) for i in range(count)]
gpus = ['GPU' + str(i) for i in range(count)]
for i in range(min(count, len(processes))):
processes[processes.index(cpus[i])] = gpus[i]
self.exp.meta_data.set('processes', processes)
class PluginRunner(object):
"""
The PluginRunner class controls the plugins and performs the interaction
between the plugin and transport layers. It inherits from the chosen
transport mechanism.
"""
def __init__(self, options):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
self.transport_control_setup(options)
self.exp = None
# TODO : Do we need to have options passed here as it is passed to sp12778
def run_plugin_list(self, options):
self.exp = Experiment(options)
plugin_list = self.exp.meta_data.plugin_list.plugin_list
logging.info("run_plugin_list: 1")
self.exp.barrier()
self.run_plugin_list_check(plugin_list)
logging.info("run_plugin_list: 2")
self.exp.barrier()
expInfo = self.exp.meta_data
logging.debug("Running process List.save_list_to_file")
expInfo.plugin_list.save_plugin_list(
expInfo.get_meta_data("nxs_filename"), exp=self.exp)
logging.info("run_plugin_list: 3")
self.exp.barrier()
self.transport_run_plugin_list()
logging.info("run_plugin_list: 4")
self.exp.barrier()
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
self.exp.nxs_file.close()
return self.exp
def run_plugin_list_check(self, plugin_list):
self.exp.barrier()
self.check_loaders_and_savers(plugin_list)
self.exp.barrier()
pu.run_plugins(self.exp, plugin_list, check=True)
self.exp.barrier()
self.exp.clear_data_objects()
self.exp.barrier()
cu.user_message("Plugin list check complete!")
def check_loaders_and_savers(self, plugin_list):
first_plugin = plugin_list[0]
end_plugin = plugin_list[-1]
plugin = pu.load_plugin(first_plugin['id'])
# check the first plugin is a loader
if not isinstance(plugin, BaseLoader):
sys.exit("The first plugin in the process must "
"inherit from BaseLoader")
plugin = pu.load_plugin(end_plugin['id'])
# check the final plugin is a saver
if not isinstance(plugin, BaseSaver):
sys.exit("The final plugin in the process must "
"inherit from BaseSaver")
class PluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options, name='PluginRunner'):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
super(PluginRunner, self).__init__()
# ********* transport function ***********
self._transport_initialise(options)
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
self.exp = Experiment(options)
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
plugin_list = self.exp.meta_data.plugin_list
logging.info('Running the plugin list check')
self._run_plugin_list_check(plugin_list)
logging.info('Setting up the experiment')
self.exp._experiment_setup()
exp_coll = self.exp._get_experiment_collection()
n_plugins = plugin_list._get_n_processing_plugins()
# ********* transport function ***********
logging.info('Running transport_pre_plugin_list_run()')
self._transport_pre_plugin_list_run()
for i in range(n_plugins):
self.exp._set_experiment_for_current_plugin(i)
self.__run_plugin(exp_coll['plugin_dict'][i])
# ********* transport function ***********
logging.info('Running transport_post_plugin_list_run')
self._transport_post_plugin_list_run()
# terminate any remaining datasets
for data in self.exp.index['in_data'].values():
self._transport_terminate_dataset(data)
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
logging.info('Processing complete')
if self.exp.meta_data.get('email'):
cu.send_email(self.exp.meta_data.get('email'))
return self.exp
def __run_plugin(self, plugin_dict):
plugin = self._transport_load_plugin(self.exp, plugin_dict)
# ********* transport function ***********
self._transport_pre_plugin()
cu.user_message("*Running the %s plugin*" % plugin.name)
# ******** transport 'process' function is called inside here ********
plugin._run_plugin(self.exp, self) # plugin driver
self.exp._barrier()
cu._output_summary(self.exp.meta_data.get("mpi"), plugin)
plugin._clean_up()
finalise = self.exp._finalise_experiment_for_current_plugin()
# ********* transport function ***********
self._transport_post_plugin()
for data in finalise['remove'] + finalise['replace']:
# ********* transport function ***********
self._transport_terminate_dataset(data)
self.exp._reorganise_datasets(finalise)
def _run_plugin_list_check(self, plugin_list):
""" Run the plugin list through the framework without executing the
main processing.
"""
plugin_list._check_loaders()
n_loaders = plugin_list._get_n_loaders()
self.__check_gpu()
check_list = np.arange(len(plugin_list.plugin_list)) - n_loaders
self.__fake_plugin_list_run(plugin_list, check_list, setnxs=True)
savers_idx_before = plugin_list._get_savers_index()
plugin_list._add_missing_savers(self.exp.index['in_data'].keys())
# ********* transport function ***********
self._transport_update_plugin_list()
self.exp._clear_data_objects()
check_list = np.array(list(set(plugin_list._get_savers_index()).
difference(set(savers_idx_before)))) - n_loaders
self.__fake_plugin_list_run(plugin_list, check_list)
self.exp._clear_data_objects()
cu.user_message("Plugin list check complete!")
def __fake_plugin_list_run(self, plugin_list, check_list, setnxs=False):
""" Run through the plugin list without any processing (setup only)\
and fill in missing dataset names.
"""
#plugin_list._reset_datasets_list()
n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()
n_plugins = plugin_list._get_n_processing_plugins()
plist = plugin_list.plugin_list
for i in range(n_loaders):
pu.plugin_loader(self.exp, plugin_list.plugin_list[i])
if setnxs:
self.exp._set_nxs_filename()
check = [True if x in check_list else False for x in range(n_plugins)]
count = 0
for i in range(n_loaders, n_loaders+n_plugins):
self.exp._barrier()
plugin = pu.plugin_loader(self.exp, plist[i], check=check[count])
plist[i]['cite'] = plugin.get_citation_information()
plugin._clean_up()
self.exp._merge_out_data_to_in()
count += 1
def __check_gpu(self):
""" Check if the process list contains GPU processes and determine if
GPUs exists. Add GPU processes to the processes list if required."""
if not self.exp.meta_data.plugin_list._contains_gpu_processes():
return
try:
import pynvml as pv
except:
logging.debug("pyNVML module not found")
raise Exception("pyNVML module not found")
try:
pv.nvmlInit()
count = int(pv.nvmlDeviceGetCount())
logging.debug("%s GPUs have been found.", count)
if not self.exp.meta_data.get('test_state'):
for i in range(count):
handle = pv.nvmlDeviceGetHandleByIndex(i)
if pv.nvmlDeviceGetComputeRunningProcesses(handle):
raise Exception("Unfortunately, GPU %i is busy. Try \
resubmitting the job to the queue." % i)
except Exception as e:
raise Exception("Unable to run GPU plugins: %s", e.message)
self.__set_gpu_processes(count)
def __set_gpu_processes(self, count):
processes = self.exp.meta_data.get('processes')
if not [i for i in processes if 'GPU' in i]:
logging.debug("GPU processes missing. GPUs found so adding them.")
cpus = ['CPU'+str(i) for i in range(count)]
gpus = ['GPU'+str(i) for i in range(count)]
for i in range(min(count, len(processes))):
processes[processes.index(cpus[i])] = gpus[i]
self.exp.meta_data.set('processes', processes)
class BasicPluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options, name='PluginRunner'):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
super(BasicPluginRunner, self).__init__()
# ********* transport function ***********
self._transport_initialise(options)
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
self.exp = Experiment(options)
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
self.exp.checkpoint = None
plugin_list = self.exp.meta_data.plugin_list
plugin_list._check_loaders()
self.exp._set_nxs_filename()
if self.exp.meta_data.get('process') == 0:
fname = self.exp.meta_data.get('nxs_filename')
plugin_list._save_plugin_list(fname)
self.exp._set_loaders()
# ********* transport function ***********
self._transport_pre_plugin_list_run()
n_plugins = plugin_list._get_n_processing_plugins()
n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()
plist = plugin_list.plugin_list
count = 0
for plugin_dict in plist[n_loaders:n_loaders+n_plugins]:
self.exp.meta_data.set('nPlugin', count)
self.__run_plugin(plugin_dict)
count += 1
# terminate any remaining datasets
for data in self.exp.index['in_data'].values():
self._transport_terminate_dataset(data)
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
logging.info('Processing complete')
return self.exp
def __run_plugin(self, plugin_dict):
plugin = self._transport_load_plugin(self.exp, plugin_dict)
self.exp.plugin = plugin
plugin._main_setup(self.exp, plugin_dict['data'])
self._transport_pre_plugin()
plugin._run_plugin(self.exp, self) # plugin driver
self.exp._barrier()
cu._output_summary(self.exp.meta_data.get("mpi"), plugin)
plugin._clean_up()
finalise = self.exp._finalise_experiment_for_current_plugin()
# ********* transport function ***********
self._transport_post_plugin()
for data in finalise['remove'] + finalise['replace']:
# ********* transport function ***********
self._transport_terminate_dataset(data)
self.exp._reorganise_datasets(finalise)
class PluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
self._transport_control_setup(options)
self.exp = None
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
self.exp = Experiment(self.options)
plugin_list = self.exp.meta_data.plugin_list.plugin_list
logging.info("run_plugin_list: 1")
self.exp._barrier()
self._run_plugin_list_check(plugin_list)
logging.info("run_plugin_list: 2")
self.exp._barrier()
expInfo = self.exp.meta_data
logging.debug("Running process List.save_list_to_file")
expInfo.plugin_list._save_plugin_list(
expInfo.get_meta_data("nxs_filename"), exp=self.exp)
logging.info("run_plugin_list: 3")
self.exp._barrier()
self._transport_run_plugin_list()
logging.info("run_plugin_list: 4")
self.exp._barrier()
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
self.exp.nxs_file.close()
return self.exp
def _run_plugin_list_check(self, plugin_list):
""" Run the plugin list through the framework without executing the
main processing.
"""
self.exp._barrier()
self.__check_loaders_and_savers()
self.__check_gpu()
self.exp._barrier()
pu.run_plugins(self.exp, plugin_list, check=True)
self.exp._barrier()
self.exp._clear_data_objects()
self.exp._barrier()
cu.user_message("Plugin list check complete!")
def __check_loaders_and_savers(self):
""" Check plugin list starts with a loader and ends with a saver.
"""
plugin_obj = self.exp.meta_data.plugin_list
loaders, savers = plugin_obj._get_loaders_and_savers_index()
if loaders:
if loaders[0] is not 0 or loaders[-1] + 1 is not len(loaders):
raise Exception("All loader plugins must be at the beginning "
"of the plugin list")
else:
raise Exception("The first plugin in the plugin list must be a "
"loader.")
if not savers or savers[0] is not plugin_obj.n_plugins - 1:
raise Exception("The final plugin in the plugin list must be a "
"saver")
def __check_gpu(self):
""" Check if the process list contains GPU processes and determine if
GPUs exists. Add GPU processes to the processes list if required."""
if not self.exp.meta_data.plugin_list._contains_gpu_processes():
return
try:
import pynvml as pv
except:
logging.debug("pyNVML module not found")
raise Exception("pyNVML module not found")
try:
pv.nvmlInit()
count = int(pv.nvmlDeviceGetCount())
logging.debug("%s GPUs have been found.", count)
except:
logging.debug("No GPUs have been found.")
raise Exception("The process list contains GPU plugins, but "
" no GPUs have been found.")
processes = self.exp.meta_data.get_meta_data('processes')
if not [i for i in processes if 'GPU' in i]:
logging.debug("GPU processes missing. GPUs found so adding them.")
cpus = ['CPU' + str(i) for i in range(count)]
gpus = ['GPU' + str(i) for i in range(count)]
for i in range(min(count, len(processes))):
processes[processes.index(cpus[i])] = gpus[i]
self.exp.meta_data.set_meta_data('processes', processes)
class PluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options, name='PluginRunner'):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
super(PluginRunner, self).__init__()
# ********* transport function ***********
self._transport_initialise(options)
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
self.exp = Experiment(options)
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
plugin_list = self.exp.meta_data.plugin_list
logging.info('Running the plugin list check')
self._run_plugin_list_check(plugin_list)
logging.info('Setting up the experiment')
self.exp._experiment_setup(self)
exp_coll = self.exp._get_experiment_collection()
n_plugins = plugin_list._get_n_processing_plugins()
# ********* transport function ***********
logging.info('Running transport_pre_plugin_list_run()')
self._transport_pre_plugin_list_run()
cp = self.exp.checkpoint
for i in range(cp.get_checkpoint_plugin(), n_plugins):
self.exp._set_experiment_for_current_plugin(i)
self.__run_plugin(exp_coll['plugin_dict'][i])
# end the plugin run if savu has been killed
self.exp._barrier(msg='PluginRunner: plugin complete.')
# ********* transport functions ***********
if self._transport_kill_signal():
self._transport_cleanup(i+1)
break
self.exp._barrier(msg='PluginRunner: No kill signal... continue.')
cp.output_plugin_checkpoint()
# ********* transport function ***********
logging.info('Running transport_post_plugin_list_run')
self._transport_post_plugin_list_run()
# terminate any remaining datasets
for data in self.exp.index['in_data'].values():
self._transport_terminate_dataset(data)
self.__output_final_message()
if self.exp.meta_data.get('email'):
cu.send_email(self.exp.meta_data.get('email'))
return self.exp
def __output_final_message(self):
kill = True if 'killsignal' in \
self.exp.meta_data.get_dictionary().keys() else False
msg = "interrupted by killsignal" if kill else "Complete"
stars = 40 if kill else 23
cu.user_message("*"*stars)
cu.user_message("* Processing " + msg + " *")
cu.user_message("*"*stars)
def __run_plugin(self, plugin_dict):
plugin = self._transport_load_plugin(self.exp, plugin_dict)
# ********* transport function ***********
self._transport_pre_plugin()
cu.user_message("*Running the %s plugin*" % plugin.name)
# ******** transport 'process' function is called inside here ********
plugin._run_plugin(self.exp, self) # plugin driver
self.exp._barrier(msg="Plugin returned from driver in Plugin Runner")
cu._output_summary(self.exp.meta_data.get("mpi"), plugin)
plugin._clean_up()
finalise = self.exp._finalise_experiment_for_current_plugin()
# ********* transport function ***********
self._transport_post_plugin()
for data in finalise['remove'] + finalise['replace']:
# ********* transport function ***********
self._transport_terminate_dataset(data)
self.exp._reorganise_datasets(finalise)
def _run_plugin_list_check(self, plugin_list):
""" Run the plugin list through the framework without executing the
main processing.
"""
plugin_list._check_loaders()
n_loaders = plugin_list._get_n_loaders()
self.__check_gpu()
check_list = np.arange(len(plugin_list.plugin_list)) - n_loaders
self.__fake_plugin_list_run(plugin_list, check_list, setnxs=True)
savers_idx_before = plugin_list._get_savers_index()
plugin_list._add_missing_savers(self.exp.index['in_data'].keys())
# ********* transport function ***********
self._transport_update_plugin_list()
self.exp._clear_data_objects()
check_list = np.array(list(set(plugin_list._get_savers_index()).
difference(set(savers_idx_before)))) - n_loaders
self.__fake_plugin_list_run(plugin_list, check_list)
self.exp._clear_data_objects()
cu.user_message("Plugin list check complete!")
def __fake_plugin_list_run(self, plugin_list, check_list, setnxs=False):
""" Run through the plugin list without any processing (setup only)\
and fill in missing dataset names.
"""
# plugin_list._reset_datasets_list()
n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()
n_plugins = plugin_list._get_n_processing_plugins()
plist = plugin_list.plugin_list
for i in range(n_loaders):
plugin = pu.plugin_loader(self.exp, plugin_list.plugin_list[i])
if setnxs:
self.exp._set_nxs_filename()
check = [True if x in check_list else False for x in range(n_plugins)]
count = 0
for i in range(n_loaders, n_loaders+n_plugins):
self.exp._barrier()
plugin = pu.plugin_loader(self.exp, plist[i], check=check[count])
plugin._revert_preview(plugin.get_in_datasets())
plist[i]['cite'] = plugin.get_citation_information()
plugin._clean_up()
self.exp._merge_out_data_to_in()
count += 1
def __check_gpu(self):
""" Check if the process list contains GPU processes and determine if
GPUs exists. Add GPU processes to the processes list if required."""
if not self.exp.meta_data.plugin_list._contains_gpu_processes():
return
try:
import pynvml as pv
except:
logging.debug("pyNVML module not found")
raise Exception("pyNVML module not found")
try:
pv.nvmlInit()
count = int(pv.nvmlDeviceGetCount())
logging.debug("%s GPUs have been found.", count)
if not self.exp.meta_data.get('test_state'):
for i in range(count):
handle = pv.nvmlDeviceGetHandleByIndex(i)
if pv.nvmlDeviceGetComputeRunningProcesses(handle):
raise Exception("Unfortunately, GPU %i is busy. Try \
resubmitting the job to the queue." % i)
except Exception as e:
raise Exception("Unable to run GPU plugins: %s", e.message)
self.__set_gpu_processes(count)
def __set_gpu_processes(self, count):
processes = self.exp.meta_data.get('processes')
if not [i for i in processes if 'GPU' in i]:
logging.debug("GPU processes missing. GPUs found so adding them.")
cpus = ['CPU'+str(i) for i in range(count)]
gpus = ['GPU'+str(i) for i in range(count)]
for i in range(min(count, len(processes))):
processes[processes.index(cpus[i])] = gpus[i]
self.exp.meta_data.set('processes', processes)
class PluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
self._transport_control_setup(options)
self.exp = None
self.options = options
# add all relevent locations to the path
pu.get_plugins_paths()
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
self.exp = Experiment(self.options)
plugin_list = self.exp.meta_data.plugin_list.plugin_list
logging.info("run_plugin_list: 1")
self.exp._barrier()
self._run_plugin_list_check(plugin_list)
logging.info("run_plugin_list: 2")
self.exp._barrier()
expInfo = self.exp.meta_data
logging.debug("Running process List.save_list_to_file")
expInfo.plugin_list._save_plugin_list(
expInfo.get_meta_data("nxs_filename"), exp=self.exp)
logging.info("run_plugin_list: 3")
self.exp._barrier()
self._transport_run_plugin_list()
logging.info("run_plugin_list: 4")
self.exp._barrier()
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
self.exp.nxs_file.close()
return self.exp
def _run_plugin_list_check(self, plugin_list):
""" Run the plugin list through the framework without executing the
main processing.
"""
self.exp._barrier()
self.__check_loaders_and_savers()
self.__check_gpu()
self.exp._barrier()
pu.run_plugins(self.exp, plugin_list, check=True)
self.exp._barrier()
self.exp._clear_data_objects()
self.exp._barrier()
cu.user_message("Plugin list check complete!")
def __check_loaders_and_savers(self):
""" Check plugin list starts with a loader and ends with a saver.
"""
plugin_obj = self.exp.meta_data.plugin_list
loaders, savers = plugin_obj._get_loaders_and_savers_index()
if loaders:
if loaders[0] is not 0 or loaders[-1]+1 is not len(loaders):
raise Exception("All loader plugins must be at the beginning "
"of the plugin list")
else:
raise Exception("The first plugin in the plugin list must be a "
"loader.")
if not savers or savers[0] is not plugin_obj.n_plugins-1:
raise Exception("The final plugin in the plugin list must be a "
"saver")
def __check_gpu(self):
""" Check if the process list contains GPU processes and determine if
GPUs exists. Add GPU processes to the processes list if required."""
if not self.exp.meta_data.plugin_list._contains_gpu_processes():
return
try:
import pynvml as pv
except:
logging.debug("pyNVML module not found")
raise Exception("pyNVML module not found")
try:
pv.nvmlInit()
count = int(pv.nvmlDeviceGetCount())
logging.debug("%s GPUs have been found.", count)
except:
logging.debug("No GPUs have been found.")
raise Exception("The process list contains GPU plugins, but "
" no GPUs have been found.")
processes = self.exp.meta_data.get_meta_data('processes')
if not [i for i in processes if 'GPU' in i]:
logging.debug("GPU processes missing. GPUs found so adding them.")
cpus = ['CPU'+str(i) for i in range(count)]
gpus = ['GPU'+str(i) for i in range(count)]
for i in range(min(count, len(processes))):
processes[processes.index(cpus[i])] = gpus[i]
self.exp.meta_data.set_meta_data('processes', processes)
class PluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
self._transport_control_setup(options)
self.exp = None
self.options = options
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
self.exp = Experiment(self.options)
plugin_list = self.exp.meta_data.plugin_list.plugin_list
logging.info("run_plugin_list: 1")
self.exp._barrier()
self._run_plugin_list_check(plugin_list)
logging.info("run_plugin_list: 2")
self.exp._barrier()
expInfo = self.exp.meta_data
logging.debug("Running process List.save_list_to_file")
expInfo.plugin_list._save_plugin_list(
expInfo.get_meta_data("nxs_filename"), exp=self.exp)
logging.info("run_plugin_list: 3")
self.exp._barrier()
self._transport_run_plugin_list()
logging.info("run_plugin_list: 4")
self.exp._barrier()
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
self.exp.nxs_file.close()
return self.exp
def _run_plugin_list_check(self, plugin_list):
""" Run the plugin list through the framework without executing the
main processing.
"""
self.exp._barrier()
self.__check_loaders_and_savers()
self.exp._barrier()
pu.run_plugins(self.exp, plugin_list, check=True)
self.exp._barrier()
self.exp._clear_data_objects()
self.exp._barrier()
cu.user_message("Plugin list check complete!")
def __check_loaders_and_savers(self):
""" Check plugin list starts with a loader and ends with a saver.
"""
plugin_obj = self.exp.meta_data.plugin_list
loaders, savers = plugin_obj._get_loaders_and_savers_index()
if loaders:
if loaders[0] is not 0 or loaders[-1] + 1 is not len(loaders):
raise Exception("All loader plugins must be at the beginning "
"of the plugin list")
else:
raise Exception("The first plugin in the plugin list must be a "
"loader.")
if not savers or savers[0] is not plugin_obj.n_plugins - 1:
raise Exception("The final plugin in the plugin list must be a "
"saver")
class PluginRunner(object):
""" Plugin list runner, which passes control to the transport layer.
"""
def __init__(self, options):
class_name = "savu.core.transports." + options["transport"] \
+ "_transport"
cu.add_base(self, cu.import_class(class_name))
self._transport_control_setup(options)
self.exp = None
self.options = options
def _run_plugin_list(self):
""" Create an experiment and run the plugin list.
"""
self.exp = Experiment(self.options)
plugin_list = self.exp.meta_data.plugin_list.plugin_list
logging.info("run_plugin_list: 1")
self.exp._barrier()
self._run_plugin_list_check(plugin_list)
logging.info("run_plugin_list: 2")
self.exp._barrier()
expInfo = self.exp.meta_data
logging.debug("Running process List.save_list_to_file")
expInfo.plugin_list._save_plugin_list(
expInfo.get_meta_data("nxs_filename"), exp=self.exp)
logging.info("run_plugin_list: 3")
self.exp._barrier()
self._transport_run_plugin_list()
logging.info("run_plugin_list: 4")
self.exp._barrier()
cu.user_message("***********************")
cu.user_message("* Processing Complete *")
cu.user_message("***********************")
self.exp.nxs_file.close()
return self.exp
def _run_plugin_list_check(self, plugin_list):
""" Run the plugin list through the framework without executing the
main processing.
"""
self.exp._barrier()
self.__check_loaders_and_savers()
self.exp._barrier()
pu.run_plugins(self.exp, plugin_list, check=True)
self.exp._barrier()
self.exp._clear_data_objects()
self.exp._barrier()
cu.user_message("Plugin list check complete!")
def __check_loaders_and_savers(self):
""" Check plugin list starts with a loader and ends with a saver.
"""
plugin_obj = self.exp.meta_data.plugin_list
loaders, savers = plugin_obj._get_loaders_and_savers_index()
if loaders:
if loaders[0] is not 0 or loaders[-1]+1 is not len(loaders):
raise Exception("All loader plugins must be at the beginning "
"of the plugin list")
else:
raise Exception("The first plugin in the plugin list must be a "
"loader.")
if not savers or savers[0] is not plugin_obj.n_plugins-1:
raise Exception("The final plugin in the plugin list must be a "
"saver")