Пример #1
0
    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        self.exp = Experiment(self.options)
        plugin_list = self.exp.meta_data.plugin_list.plugin_list

        logging.info("run_plugin_list: 1")
        self.exp._barrier()
        self._run_plugin_list_check(plugin_list)

        logging.info("run_plugin_list: 2")
        self.exp._barrier()
        expInfo = self.exp.meta_data
        logging.debug("Running process List.save_list_to_file")
        expInfo.plugin_list._save_plugin_list(
            expInfo.get_meta_data("nxs_filename"), exp=self.exp)

        logging.info("run_plugin_list: 3")
        self.exp._barrier()
        self._transport_run_plugin_list()

        logging.info("run_plugin_list: 4")
        self.exp._barrier()

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        self.exp.nxs_file.close()
        return self.exp
Пример #2
0
 def __init__(self, options):
     class_name = "savu.core.transports." + options["transport"] \
                  + "_transport"
     cu.add_base(self, cu.import_class(class_name))
     self._transport_control_setup(options)
     self.options = options
     # add all relevent locations to the path
     pu.get_plugins_paths()
     self.exp = Experiment(options)
Пример #3
0
    def __init__(self, options, name='PluginRunner'):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        super(PluginRunner, self).__init__()

        #  ********* transport function ***********
        self._transport_initialise(options)
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()
        self.exp = Experiment(options)
Пример #4
0
    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        self.exp = Experiment(self.options)
        plugin_list = self.exp.meta_data.plugin_list.plugin_list

        logging.info("run_plugin_list: 1")
        self.exp._barrier()
        self._run_plugin_list_check(plugin_list)

        logging.info("run_plugin_list: 2")
        self.exp._barrier()
        expInfo = self.exp.meta_data
        logging.debug("Running process List.save_list_to_file")
        expInfo.plugin_list._save_plugin_list(
            expInfo.get_meta_data("nxs_filename"), exp=self.exp)

        logging.info("run_plugin_list: 3")
        self.exp._barrier()
        self._transport_run_plugin_list()

        logging.info("run_plugin_list: 4")
        self.exp._barrier()

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        self.exp.nxs_file.close()
        return self.exp
Пример #5
0
def fake_plugin_runner(options):
    # Stripped down version of the plugin runner
    # Loads the loader plugin but stops before processing plugins
    plugin_runner = PluginRunner(options)
    plugin_runner.exp = Experiment(options)
    plugin_list = plugin_runner.exp.meta_data.plugin_list.plugin_list

    exp = plugin_runner.exp
    pu.plugin_loader(exp, plugin_list[0])
    exp._set_nxs_file()
    return plugin_runner
Пример #6
0
    def __init__(self, options, name='PluginRunner'):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        super(PluginRunner, self).__init__()

        #  ********* transport function ***********
        self._transport_initialise(options)
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()
        self.exp = Experiment(options)
Пример #7
0
 def create_chunking_instance(self, current_list, nnext_list, nProcs):
     current = self.create_pattern('a', current_list)
     nnext = self.create_pattern('b', nnext_list)
     options = tu.set_experiment('tomoRaw')
     options['processes'] = range(nProcs)
     # set a dummy process list
     options['process_file'] = \
         tu.get_test_process_path('basic_tomo_process.nxs')
     exp = Experiment(options)
     test_dict = {'current': current, 'next': nnext}
     chunking = Chunking(exp, test_dict)
     return chunking
Пример #8
0
def plugin_runner_load_plugin(options):
    plugin_runner = PluginRunner(options)
    plugin_runner.exp = Experiment(options)
    plugin_list = plugin_runner.exp.meta_data.plugin_list.plugin_list

    exp = plugin_runner.exp
    pu.plugin_loader(exp, plugin_list[0])
    exp._set_nxs_file()

    plugin_dict = plugin_list[1]
    plugin = pu.get_plugin(plugin_dict['id'])
    plugin.exp = exp

    return plugin
Пример #9
0
def plugin_runner_real_plugin_run(options):
    plugin_runner = PluginRunner(options)
    plugin_runner.exp = Experiment(options)
    plugin_list = plugin_runner.exp.meta_data.plugin_list.plugin_list
    plugin_runner._run_plugin_list_check(plugin_list)

    exp = plugin_runner.exp

    pu.plugin_loader(exp, plugin_list[0])

    start_in_data = copy.deepcopy(exp.index['in_data'])
    in_data = exp.index["in_data"][exp.index["in_data"].keys()[0]]
    out_data_objs, stop = in_data._load_data(1)
    exp._clear_data_objects()
    exp.index['in_data'] = copy.deepcopy(start_in_data)

    for key in out_data_objs[0]:
        exp.index["out_data"][key] = out_data_objs[0][key]

    plugin = pu.plugin_loader(exp, plugin_list[1])
    plugin._run_plugin(exp, plugin_runner)
Пример #10
0
class PluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """
    def __init__(self, options):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        self._transport_control_setup(options)
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()
        self.exp = Experiment(options)

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        plugin_list = self.exp.meta_data.plugin_list

        self.exp._barrier()
        self._run_plugin_list_check(plugin_list)

        self.exp._barrier()
        expInfo = self.exp.meta_data
        logging.debug("Running process List.save_list_to_file")
        expInfo.plugin_list._save_plugin_list(
            expInfo.get_meta_data("nxs_filename"), exp=self.exp)

        self.exp._barrier()
        self._transport_run_plugin_list()

        self.exp._barrier()

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        self.exp.nxs_file.close()
        return self.exp

    def _run_plugin_list_check(self, plugin_list):
        """ Run the plugin list through the framework without executing the
        main processing.
        """
        self.exp._barrier()
        plugin_list._check_loaders_and_savers()
        self.__check_gpu()

        self.exp._barrier()
        self.__fake_plugin_list_run(plugin_list.plugin_list, check=True)

        self.exp._barrier()
        self.exp._clear_data_objects()

        self.exp._barrier()
        cu.user_message("Plugin list check complete!")

    def __fake_plugin_list_run(self, plugin_list, **kwargs):
        """ Run through the plugin list without any processing (setup only)\
        and fill in missing dataset names.
        """
        n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()

        for i in range(n_loaders):
            pu.plugin_loader(self.exp, plugin_list[i])

        self.exp._set_nxs_filename()

        check = kwargs.get('check', False)
        for i in range(n_loaders, len(plugin_list) - 1):
            self.exp._barrier()
            plugin = pu.plugin_loader(self.exp, plugin_list[i], check=check)
            plugin_list[i]['cite'] = plugin.get_citation_information()
            plugin._clean_up()
            self.exp._merge_out_data_to_in()

    def __check_gpu(self):
        """ Check if the process list contains GPU processes and determine if
        GPUs exists. Add GPU processes to the processes list if required."""
        if not self.exp.meta_data.plugin_list._contains_gpu_processes():
            return
        try:
            import pynvml as pv
        except:
            logging.debug("pyNVML module not found")
            raise Exception("pyNVML module not found")
        try:
            pv.nvmlInit()
            count = int(pv.nvmlDeviceGetCount())
            logging.debug("%s GPUs have been found.", count)
            for i in range(count):
                handle = pv.nvmlDeviceGetHandleByIndex(i)
                if pv.nvmlDeviceGetComputeRunningProcesses(handle):
                    raise Exception("Unfortunately, GPU %i is busy. Try \
                        resubmitting the job to the queue." % i)
        except:
            logging.debug("No GPUs have been found.")
            raise Exception("The process list contains GPU plugins, but "
                            " no GPUs have been found.")
        self.__set_gpu_processes(count)

    def __set_gpu_processes(self, count):
        processes = self.exp.meta_data.get_meta_data('processes')
        if not [i for i in processes if 'GPU' in i]:
            logging.debug("GPU processes missing. GPUs found so adding them.")
            cpus = ['CPU' + str(i) for i in range(count)]
            gpus = ['GPU' + str(i) for i in range(count)]
            for i in range(min(count, len(processes))):
                processes[processes.index(cpus[i])] = gpus[i]
            self.exp.meta_data.set_meta_data('processes', processes)
Пример #11
0
 def _get_checkpoint(self, cfile=True):
     exp = Experiment(self.get_options(cfile=cfile))
     checkpoint = Checkpointing(exp)
     return checkpoint
Пример #12
0
class PluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """
    def __init__(self, options, name='PluginRunner'):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        super(PluginRunner, self).__init__()

        #  ********* transport function ***********
        self._transport_initialise(options)
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()
        self.exp = Experiment(options)

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        self.exp._setup(self)

        plugin_list = self.exp.meta_data.plugin_list
        logging.info('Running the plugin list check')
        self._run_plugin_list_setup(plugin_list)

        exp_coll = self.exp._get_collection()
        n_plugins = plugin_list._get_n_processing_plugins()

        #  ********* transport function ***********
        logging.info('Running transport_pre_plugin_list_run()')
        self._transport_pre_plugin_list_run()

        cp = self.exp.checkpoint
        checkpoint_plugin = cp.get_checkpoint_plugin()
        for i in range(checkpoint_plugin, n_plugins):
            self.exp._set_experiment_for_current_plugin(i)
            memory_before = cu.get_memory_usage_linux()

            plugin_name = self.__run_plugin(exp_coll['plugin_dict'][i])

            self.exp._barrier(msg='PluginRunner: plugin complete.')

            memory_after = cu.get_memory_usage_linux()
            logging.debug(
                "{} memory usage before: {} MB, after: {} MB, change: {} MB".
                format(plugin_name, memory_before, memory_after,
                       memory_after - memory_before))

            #  ********* transport functions ***********
            # end the plugin run if savu has been killed
            if self._transport_kill_signal():
                self._transport_cleanup(i + 1)
                break
            self.exp._barrier(msg='PluginRunner: No kill signal... continue.')
            cp.output_plugin_checkpoint()

        #  ********* transport function ***********
        logging.info('Running transport_post_plugin_list_run')
        self._transport_post_plugin_list_run()

        # terminate any remaining datasets
        for data in list(self.exp.index['in_data'].values()):
            self._transport_terminate_dataset(data)

        self.__output_final_message()

        if self.exp.meta_data.get('email'):
            cu.send_email(self.exp.meta_data.get('email'))

        return self.exp

    def __output_final_message(self):
        kill = True if 'killsignal' in \
                       self.exp.meta_data.get_dictionary().keys() else False
        msg = "interrupted by killsignal" if kill else "Complete"
        stars = 40 if kill else 23
        cu.user_message("*" * stars)
        cu.user_message("* Processing " + msg + " *")
        cu.user_message("*" * stars)

    def __run_plugin(self, plugin_dict):
        plugin = self._transport_load_plugin(self.exp, plugin_dict)

        #  ********* transport function ***********
        self._transport_pre_plugin()
        cu.user_message("*Running the %s plugin*" % plugin.name)

        #  ******** transport 'process' function is called inside here ********
        plugin._run_plugin(self.exp, self)  # plugin driver

        self.exp._barrier(msg="Plugin returned from driver in Plugin Runner")
        cu._output_summary(self.exp.meta_data.get("mpi"), plugin)
        plugin._clean_up()
        finalise = self.exp._finalise_experiment_for_current_plugin()

        #  ********* transport function ***********
        self._transport_post_plugin()

        for data in finalise['remove'] + finalise['replace']:
            #  ********* transport function ***********
            self._transport_terminate_dataset(data)

        self.exp._reorganise_datasets(finalise)
        return plugin.name

    def _run_plugin_list_setup(self, plugin_list):
        """ Run the plugin list through the framework without executing the
        main processing.
        """
        plugin_list._check_loaders()
        self.__check_gpu()

        n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()
        n_plugins = plugin_list._get_n_processing_plugins()
        plist = plugin_list.plugin_list

        # set loaders
        for i in range(n_loaders):
            pu.plugin_loader(self.exp, plist[i])
            self.exp._set_initial_datasets()

        # run all plugin setup methods and store information in experiment
        # collection
        count = 0
        for plugin_dict in plist[n_loaders:n_loaders + n_plugins]:
            self.__plugin_setup(plugin_dict, count)
            count += 1

        plugin_list._add_missing_savers(self.exp)

        #  ********* transport function ***********
        self._transport_update_plugin_list()

        # check added savers
        for plugin_dict in plist[n_loaders + count:]:
            self.__plugin_setup(plugin_dict, count)
            count += 1

        self.exp._reset_datasets()
        self.exp._finalise_setup(plugin_list)
        cu.user_message("Plugin list check complete!")

    def __plugin_setup(self, plugin_dict, count):
        self.exp.meta_data.set("nPlugin", count)
        plugin = pu.plugin_loader(self.exp, plugin_dict, check=True)
        plugin._revert_preview(plugin.get_in_datasets())
        plugin_dict['cite'] = plugin.get_citation_information()
        plugin._clean_up()
        self.exp._update(plugin_dict)
        self.exp._merge_out_data_to_in()

    def __check_gpu(self):
        """ Check if the process list contains GPU processes and determine if
        GPUs exists. Add GPU processes to the processes list if required."""
        if not self.exp.meta_data.plugin_list._contains_gpu_processes():
            return

        try:
            import pynvml as pv
        except Exception:
            logging.debug("pyNVML module not found")
            raise Exception("pyNVML module not found")

        try:
            pv.nvmlInit()
            count = int(pv.nvmlDeviceGetCount())
            if count == 0:
                raise Exception("No GPUs found")
            logging.debug("%s GPUs have been found.", count)

            if not self.exp.meta_data.get('test_state'):
                for i in range(count):
                    handle = pv.nvmlDeviceGetHandleByIndex(i)
                    if pv.nvmlDeviceGetComputeRunningProcesses(handle):
                        raise Exception("Unfortunately, GPU %i is busy. Try \
                            resubmitting the job to the queue." % i)
        except Exception as e:
            raise Exception("Unable to run GPU plugins: %s", str(e))
        self.__set_gpu_processes(count)

    def __set_gpu_processes(self, count):
        processes = self.exp.meta_data.get('processes')
        if not [i for i in processes if 'GPU' in i]:
            logging.debug("GPU processes missing. GPUs found so adding them.")
            cpus = ['CPU' + str(i) for i in range(count)]
            gpus = ['GPU' + str(i) for i in range(count)]
            for i in range(min(count, len(processes))):
                processes[processes.index(cpus[i])] = gpus[i]
            self.exp.meta_data.set('processes', processes)
Пример #13
0
class PluginRunner(object):
    """
    The PluginRunner class controls the plugins and performs the interaction
    between the plugin and transport layers.  It inherits from the chosen
    transport mechanism.
    """

    def __init__(self, options):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        self.transport_control_setup(options)
        self.exp = None

    # TODO : Do we need to have options passed here as it is passed to sp12778
    def run_plugin_list(self, options):
        self.exp = Experiment(options)
        plugin_list = self.exp.meta_data.plugin_list.plugin_list

        logging.info("run_plugin_list: 1")
        self.exp.barrier()
        self.run_plugin_list_check(plugin_list)

        logging.info("run_plugin_list: 2")
        self.exp.barrier()
        expInfo = self.exp.meta_data
        logging.debug("Running process List.save_list_to_file")
        expInfo.plugin_list.save_plugin_list(
            expInfo.get_meta_data("nxs_filename"), exp=self.exp)

        logging.info("run_plugin_list: 3")
        self.exp.barrier()
        self.transport_run_plugin_list()

        logging.info("run_plugin_list: 4")
        self.exp.barrier()

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        self.exp.nxs_file.close()

        return self.exp

    def run_plugin_list_check(self, plugin_list):
        self.exp.barrier()
        self.check_loaders_and_savers(plugin_list)

        self.exp.barrier()
        pu.run_plugins(self.exp, plugin_list, check=True)

        self.exp.barrier()
        self.exp.clear_data_objects()

        self.exp.barrier()
        cu.user_message("Plugin list check complete!")

    def check_loaders_and_savers(self, plugin_list):
        first_plugin = plugin_list[0]
        end_plugin = plugin_list[-1]

        plugin = pu.load_plugin(first_plugin['id'])
        # check the first plugin is a loader
        if not isinstance(plugin, BaseLoader):
            sys.exit("The first plugin in the process must "
                     "inherit from BaseLoader")

        plugin = pu.load_plugin(end_plugin['id'])
        # check the final plugin is a saver
        if not isinstance(plugin, BaseSaver):
            sys.exit("The final plugin in the process must "
                     "inherit from BaseSaver")
Пример #14
0
class PluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """

    def __init__(self, options, name='PluginRunner'):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        super(PluginRunner, self).__init__()

        #  ********* transport function ***********
        self._transport_initialise(options)
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()
        self.exp = Experiment(options)

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        plugin_list = self.exp.meta_data.plugin_list
        logging.info('Running the plugin list check')
        self._run_plugin_list_check(plugin_list)

        logging.info('Setting up the experiment')
        self.exp._experiment_setup()
        exp_coll = self.exp._get_experiment_collection()
        n_plugins = plugin_list._get_n_processing_plugins()

        #  ********* transport function ***********
        logging.info('Running transport_pre_plugin_list_run()')
        self._transport_pre_plugin_list_run()

        for i in range(n_plugins):
            self.exp._set_experiment_for_current_plugin(i)
            self.__run_plugin(exp_coll['plugin_dict'][i])

        #  ********* transport function ***********
        logging.info('Running transport_post_plugin_list_run')
        self._transport_post_plugin_list_run()

        # terminate any remaining datasets
        for data in self.exp.index['in_data'].values():
            self._transport_terminate_dataset(data)

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        logging.info('Processing complete')
        if self.exp.meta_data.get('email'):
            cu.send_email(self.exp.meta_data.get('email'))
        return self.exp

    def __run_plugin(self, plugin_dict):
        plugin = self._transport_load_plugin(self.exp, plugin_dict)

        #  ********* transport function ***********
        self._transport_pre_plugin()

        cu.user_message("*Running the %s plugin*" % plugin.name)

        #  ******** transport 'process' function is called inside here ********
        plugin._run_plugin(self.exp, self)  # plugin driver
        self.exp._barrier()

        cu._output_summary(self.exp.meta_data.get("mpi"), plugin)

        plugin._clean_up()

        finalise = self.exp._finalise_experiment_for_current_plugin()

        #  ********* transport function ***********
        self._transport_post_plugin()

        for data in finalise['remove'] + finalise['replace']:
            #  ********* transport function ***********
            self._transport_terminate_dataset(data)

        self.exp._reorganise_datasets(finalise)

    def _run_plugin_list_check(self, plugin_list):
        """ Run the plugin list through the framework without executing the
        main processing.
        """
        plugin_list._check_loaders()
        n_loaders = plugin_list._get_n_loaders()

        self.__check_gpu()

        check_list = np.arange(len(plugin_list.plugin_list)) - n_loaders
        self.__fake_plugin_list_run(plugin_list, check_list, setnxs=True)

        savers_idx_before = plugin_list._get_savers_index()
        plugin_list._add_missing_savers(self.exp.index['in_data'].keys())

        #  ********* transport function ***********
        self._transport_update_plugin_list()

        self.exp._clear_data_objects()

        check_list = np.array(list(set(plugin_list._get_savers_index()).
                              difference(set(savers_idx_before)))) - n_loaders
        self.__fake_plugin_list_run(plugin_list, check_list)

        self.exp._clear_data_objects()
        cu.user_message("Plugin list check complete!")

    def __fake_plugin_list_run(self, plugin_list, check_list, setnxs=False):
        """ Run through the plugin list without any processing (setup only)\
        and fill in missing dataset names.
        """
        #plugin_list._reset_datasets_list()
        n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()
        n_plugins = plugin_list._get_n_processing_plugins()

        plist = plugin_list.plugin_list
        for i in range(n_loaders):
            pu.plugin_loader(self.exp, plugin_list.plugin_list[i])

        if setnxs:
            self.exp._set_nxs_filename()

        check = [True if x in check_list else False for x in range(n_plugins)]

        count = 0
        for i in range(n_loaders, n_loaders+n_plugins):
            self.exp._barrier()
            plugin = pu.plugin_loader(self.exp, plist[i], check=check[count])
            plist[i]['cite'] = plugin.get_citation_information()
            plugin._clean_up()
            self.exp._merge_out_data_to_in()
            count += 1

    def __check_gpu(self):
        """ Check if the process list contains GPU processes and determine if
        GPUs exists. Add GPU processes to the processes list if required."""
        if not self.exp.meta_data.plugin_list._contains_gpu_processes():
            return

        try:
            import pynvml as pv
        except:
            logging.debug("pyNVML module not found")
            raise Exception("pyNVML module not found")

        try:
            pv.nvmlInit()
            count = int(pv.nvmlDeviceGetCount())
            logging.debug("%s GPUs have been found.", count)

            if not self.exp.meta_data.get('test_state'):
                for i in range(count):
                    handle = pv.nvmlDeviceGetHandleByIndex(i)
                    if pv.nvmlDeviceGetComputeRunningProcesses(handle):
                        raise Exception("Unfortunately, GPU %i is busy. Try \
                            resubmitting the job to the queue." % i)
        except Exception as e:
            raise Exception("Unable to run GPU plugins: %s", e.message)
        self.__set_gpu_processes(count)

    def __set_gpu_processes(self, count):
        processes = self.exp.meta_data.get('processes')
        if not [i for i in processes if 'GPU' in i]:
            logging.debug("GPU processes missing. GPUs found so adding them.")
            cpus = ['CPU'+str(i) for i in range(count)]
            gpus = ['GPU'+str(i) for i in range(count)]
            for i in range(min(count, len(processes))):
                processes[processes.index(cpus[i])] = gpus[i]
            self.exp.meta_data.set('processes', processes)
Пример #15
0
class BasicPluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """

    def __init__(self, options, name='PluginRunner'):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        super(BasicPluginRunner, self).__init__()

        #  ********* transport function ***********
        self._transport_initialise(options)
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()
        self.exp = Experiment(options)

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        self.exp.checkpoint = None
        plugin_list = self.exp.meta_data.plugin_list
        plugin_list._check_loaders()

        self.exp._set_nxs_filename()
        if self.exp.meta_data.get('process') == 0:
            fname = self.exp.meta_data.get('nxs_filename')
            plugin_list._save_plugin_list(fname)

        self.exp._set_loaders()

        #  ********* transport function ***********
        self._transport_pre_plugin_list_run()

        n_plugins = plugin_list._get_n_processing_plugins()
        n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()

        plist = plugin_list.plugin_list
        count = 0
        for plugin_dict in plist[n_loaders:n_loaders+n_plugins]:
            self.exp.meta_data.set('nPlugin', count)
            self.__run_plugin(plugin_dict)
            count += 1

        # terminate any remaining datasets
        for data in self.exp.index['in_data'].values():
            self._transport_terminate_dataset(data)

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        logging.info('Processing complete')
        return self.exp

    def __run_plugin(self, plugin_dict):
        plugin = self._transport_load_plugin(self.exp, plugin_dict)
        self.exp.plugin = plugin
        plugin._main_setup(self.exp, plugin_dict['data'])

        self._transport_pre_plugin()

        plugin._run_plugin(self.exp, self)  # plugin driver
        self.exp._barrier()
        cu._output_summary(self.exp.meta_data.get("mpi"), plugin)
        plugin._clean_up()

        finalise = self.exp._finalise_experiment_for_current_plugin()

        #  ********* transport function ***********
        self._transport_post_plugin()

        for data in finalise['remove'] + finalise['replace']:
            #  ********* transport function ***********
            self._transport_terminate_dataset(data)

        self.exp._reorganise_datasets(finalise)
Пример #16
0
class PluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """
    def __init__(self, options):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        self._transport_control_setup(options)
        self.exp = None
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        self.exp = Experiment(self.options)
        plugin_list = self.exp.meta_data.plugin_list.plugin_list

        logging.info("run_plugin_list: 1")
        self.exp._barrier()
        self._run_plugin_list_check(plugin_list)

        logging.info("run_plugin_list: 2")
        self.exp._barrier()
        expInfo = self.exp.meta_data
        logging.debug("Running process List.save_list_to_file")
        expInfo.plugin_list._save_plugin_list(
            expInfo.get_meta_data("nxs_filename"), exp=self.exp)

        logging.info("run_plugin_list: 3")
        self.exp._barrier()
        self._transport_run_plugin_list()

        logging.info("run_plugin_list: 4")
        self.exp._barrier()

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        self.exp.nxs_file.close()
        return self.exp

    def _run_plugin_list_check(self, plugin_list):
        """ Run the plugin list through the framework without executing the
        main processing.
        """
        self.exp._barrier()
        self.__check_loaders_and_savers()
        self.__check_gpu()

        self.exp._barrier()
        pu.run_plugins(self.exp, plugin_list, check=True)

        self.exp._barrier()
        self.exp._clear_data_objects()

        self.exp._barrier()
        cu.user_message("Plugin list check complete!")

    def __check_loaders_and_savers(self):
        """ Check plugin list starts with a loader and ends with a saver.
        """
        plugin_obj = self.exp.meta_data.plugin_list
        loaders, savers = plugin_obj._get_loaders_and_savers_index()

        if loaders:
            if loaders[0] is not 0 or loaders[-1] + 1 is not len(loaders):
                raise Exception("All loader plugins must be at the beginning "
                                "of the plugin list")
        else:
            raise Exception("The first plugin in the plugin list must be a "
                            "loader.")

        if not savers or savers[0] is not plugin_obj.n_plugins - 1:
            raise Exception("The final plugin in the plugin list must be a "
                            "saver")

    def __check_gpu(self):
        """ Check if the process list contains GPU processes and determine if
        GPUs exists. Add GPU processes to the processes list if required."""
        if not self.exp.meta_data.plugin_list._contains_gpu_processes():
            return
        try:
            import pynvml as pv
        except:
            logging.debug("pyNVML module not found")
            raise Exception("pyNVML module not found")
        try:
            pv.nvmlInit()
            count = int(pv.nvmlDeviceGetCount())
            logging.debug("%s GPUs have been found.", count)
        except:
            logging.debug("No GPUs have been found.")
            raise Exception("The process list contains GPU plugins, but "
                            " no GPUs have been found.")

        processes = self.exp.meta_data.get_meta_data('processes')
        if not [i for i in processes if 'GPU' in i]:
            logging.debug("GPU processes missing. GPUs found so adding them.")
            cpus = ['CPU' + str(i) for i in range(count)]
            gpus = ['GPU' + str(i) for i in range(count)]
            for i in range(min(count, len(processes))):
                processes[processes.index(cpus[i])] = gpus[i]
            self.exp.meta_data.set_meta_data('processes', processes)
Пример #17
0
class PluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """

    def __init__(self, options, name='PluginRunner'):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        super(PluginRunner, self).__init__()

        #  ********* transport function ***********
        self._transport_initialise(options)
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()
        self.exp = Experiment(options)

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        plugin_list = self.exp.meta_data.plugin_list
        logging.info('Running the plugin list check')
        self._run_plugin_list_check(plugin_list)

        logging.info('Setting up the experiment')
        self.exp._experiment_setup(self)

        exp_coll = self.exp._get_experiment_collection()
        n_plugins = plugin_list._get_n_processing_plugins()

        #  ********* transport function ***********
        logging.info('Running transport_pre_plugin_list_run()')
        self._transport_pre_plugin_list_run()

        cp = self.exp.checkpoint
        for i in range(cp.get_checkpoint_plugin(), n_plugins):
            self.exp._set_experiment_for_current_plugin(i)
            self.__run_plugin(exp_coll['plugin_dict'][i])
            # end the plugin run if savu has been killed
            self.exp._barrier(msg='PluginRunner: plugin complete.')

            #  ********* transport functions ***********
            if self._transport_kill_signal():
                self._transport_cleanup(i+1)
                break
            self.exp._barrier(msg='PluginRunner: No kill signal... continue.')
            cp.output_plugin_checkpoint()

        #  ********* transport function ***********
        logging.info('Running transport_post_plugin_list_run')
        self._transport_post_plugin_list_run()

        # terminate any remaining datasets
        for data in self.exp.index['in_data'].values():
            self._transport_terminate_dataset(data)

        self.__output_final_message()

        if self.exp.meta_data.get('email'):
            cu.send_email(self.exp.meta_data.get('email'))

        return self.exp

    def __output_final_message(self):
        kill = True if 'killsignal' in \
            self.exp.meta_data.get_dictionary().keys() else False
        msg = "interrupted by killsignal" if kill else "Complete"
        stars = 40 if kill else 23
        cu.user_message("*"*stars)
        cu.user_message("* Processing " + msg + " *")
        cu.user_message("*"*stars)

    def __run_plugin(self, plugin_dict):
        plugin = self._transport_load_plugin(self.exp, plugin_dict)

        #  ********* transport function ***********
        self._transport_pre_plugin()
        cu.user_message("*Running the %s plugin*" % plugin.name)

        #  ******** transport 'process' function is called inside here ********
        plugin._run_plugin(self.exp, self)  # plugin driver

        self.exp._barrier(msg="Plugin returned from driver in Plugin Runner")
        cu._output_summary(self.exp.meta_data.get("mpi"), plugin)
        plugin._clean_up()
        finalise = self.exp._finalise_experiment_for_current_plugin()

        #  ********* transport function ***********
        self._transport_post_plugin()

        for data in finalise['remove'] + finalise['replace']:
            #  ********* transport function ***********
            self._transport_terminate_dataset(data)

        self.exp._reorganise_datasets(finalise)

    def _run_plugin_list_check(self, plugin_list):
        """ Run the plugin list through the framework without executing the
        main processing.
        """
        plugin_list._check_loaders()
        n_loaders = plugin_list._get_n_loaders()
        self.__check_gpu()
        check_list = np.arange(len(plugin_list.plugin_list)) - n_loaders
        self.__fake_plugin_list_run(plugin_list, check_list, setnxs=True)
        savers_idx_before = plugin_list._get_savers_index()
        plugin_list._add_missing_savers(self.exp.index['in_data'].keys())

        #  ********* transport function ***********
        self._transport_update_plugin_list()

        self.exp._clear_data_objects()

        check_list = np.array(list(set(plugin_list._get_savers_index()).
                              difference(set(savers_idx_before)))) - n_loaders
        self.__fake_plugin_list_run(plugin_list, check_list)

        self.exp._clear_data_objects()
        cu.user_message("Plugin list check complete!")

    def __fake_plugin_list_run(self, plugin_list, check_list, setnxs=False):
        """ Run through the plugin list without any processing (setup only)\
        and fill in missing dataset names.
        """
        # plugin_list._reset_datasets_list()
        n_loaders = self.exp.meta_data.plugin_list._get_n_loaders()
        n_plugins = plugin_list._get_n_processing_plugins()

        plist = plugin_list.plugin_list
        for i in range(n_loaders):
            plugin = pu.plugin_loader(self.exp, plugin_list.plugin_list[i])

        if setnxs:
            self.exp._set_nxs_filename()
        
        check = [True if x in check_list else False for x in range(n_plugins)]

        count = 0
        for i in range(n_loaders, n_loaders+n_plugins):
            self.exp._barrier()
            plugin = pu.plugin_loader(self.exp, plist[i], check=check[count])
            plugin._revert_preview(plugin.get_in_datasets())
            plist[i]['cite'] = plugin.get_citation_information()
            plugin._clean_up()
            self.exp._merge_out_data_to_in()
            count += 1

    def __check_gpu(self):
        """ Check if the process list contains GPU processes and determine if
        GPUs exists. Add GPU processes to the processes list if required."""
        if not self.exp.meta_data.plugin_list._contains_gpu_processes():
            return

        try:
            import pynvml as pv
        except:
            logging.debug("pyNVML module not found")
            raise Exception("pyNVML module not found")

        try:
            pv.nvmlInit()
            count = int(pv.nvmlDeviceGetCount())
            logging.debug("%s GPUs have been found.", count)

            if not self.exp.meta_data.get('test_state'):
                for i in range(count):
                    handle = pv.nvmlDeviceGetHandleByIndex(i)
                    if pv.nvmlDeviceGetComputeRunningProcesses(handle):
                        raise Exception("Unfortunately, GPU %i is busy. Try \
                            resubmitting the job to the queue." % i)
        except Exception as e:
            raise Exception("Unable to run GPU plugins: %s", e.message)
        self.__set_gpu_processes(count)

    def __set_gpu_processes(self, count):
        processes = self.exp.meta_data.get('processes')
        if not [i for i in processes if 'GPU' in i]:
            logging.debug("GPU processes missing. GPUs found so adding them.")
            cpus = ['CPU'+str(i) for i in range(count)]
            gpus = ['GPU'+str(i) for i in range(count)]
            for i in range(min(count, len(processes))):
                processes[processes.index(cpus[i])] = gpus[i]
            self.exp.meta_data.set('processes', processes)
Пример #18
0
class PluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """

    def __init__(self, options):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        self._transport_control_setup(options)
        self.exp = None
        self.options = options
        # add all relevent locations to the path
        pu.get_plugins_paths()

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        self.exp = Experiment(self.options)
        plugin_list = self.exp.meta_data.plugin_list.plugin_list

        logging.info("run_plugin_list: 1")
        self.exp._barrier()
        self._run_plugin_list_check(plugin_list)

        logging.info("run_plugin_list: 2")
        self.exp._barrier()
        expInfo = self.exp.meta_data
        logging.debug("Running process List.save_list_to_file")
        expInfo.plugin_list._save_plugin_list(
            expInfo.get_meta_data("nxs_filename"), exp=self.exp)

        logging.info("run_plugin_list: 3")
        self.exp._barrier()
        self._transport_run_plugin_list()

        logging.info("run_plugin_list: 4")
        self.exp._barrier()

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        self.exp.nxs_file.close()
        return self.exp

    def _run_plugin_list_check(self, plugin_list):
        """ Run the plugin list through the framework without executing the
        main processing.
        """
        self.exp._barrier()
        self.__check_loaders_and_savers()
        self.__check_gpu()

        self.exp._barrier()
        pu.run_plugins(self.exp, plugin_list, check=True)

        self.exp._barrier()
        self.exp._clear_data_objects()

        self.exp._barrier()
        cu.user_message("Plugin list check complete!")

    def __check_loaders_and_savers(self):
        """ Check plugin list starts with a loader and ends with a saver.
        """
        plugin_obj = self.exp.meta_data.plugin_list
        loaders, savers = plugin_obj._get_loaders_and_savers_index()

        if loaders:
            if loaders[0] is not 0 or loaders[-1]+1 is not len(loaders):
                raise Exception("All loader plugins must be at the beginning "
                                "of the plugin list")
        else:
            raise Exception("The first plugin in the plugin list must be a "
                            "loader.")

        if not savers or savers[0] is not plugin_obj.n_plugins-1:
            raise Exception("The final plugin in the plugin list must be a "
                            "saver")

    def __check_gpu(self):
        """ Check if the process list contains GPU processes and determine if
        GPUs exists. Add GPU processes to the processes list if required."""
        if not self.exp.meta_data.plugin_list._contains_gpu_processes():
            return
        try:
            import pynvml as pv
        except:
            logging.debug("pyNVML module not found")
            raise Exception("pyNVML module not found")
        try:
            pv.nvmlInit()
            count = int(pv.nvmlDeviceGetCount())
            logging.debug("%s GPUs have been found.", count)
        except:
            logging.debug("No GPUs have been found.")
            raise Exception("The process list contains GPU plugins, but "
                            " no GPUs have been found.")

        processes = self.exp.meta_data.get_meta_data('processes')
        if not [i for i in processes if 'GPU' in i]:
            logging.debug("GPU processes missing. GPUs found so adding them.")
            cpus = ['CPU'+str(i) for i in range(count)]
            gpus = ['GPU'+str(i) for i in range(count)]
            for i in range(min(count, len(processes))):
                processes[processes.index(cpus[i])] = gpus[i]
            self.exp.meta_data.set_meta_data('processes', processes)
Пример #19
0
class PluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """
    def __init__(self, options):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        self._transport_control_setup(options)
        self.exp = None
        self.options = options

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        self.exp = Experiment(self.options)
        plugin_list = self.exp.meta_data.plugin_list.plugin_list

        logging.info("run_plugin_list: 1")
        self.exp._barrier()
        self._run_plugin_list_check(plugin_list)

        logging.info("run_plugin_list: 2")
        self.exp._barrier()
        expInfo = self.exp.meta_data
        logging.debug("Running process List.save_list_to_file")
        expInfo.plugin_list._save_plugin_list(
            expInfo.get_meta_data("nxs_filename"), exp=self.exp)

        logging.info("run_plugin_list: 3")
        self.exp._barrier()
        self._transport_run_plugin_list()

        logging.info("run_plugin_list: 4")
        self.exp._barrier()

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        self.exp.nxs_file.close()
        return self.exp

    def _run_plugin_list_check(self, plugin_list):
        """ Run the plugin list through the framework without executing the
        main processing.
        """
        self.exp._barrier()
        self.__check_loaders_and_savers()

        self.exp._barrier()
        pu.run_plugins(self.exp, plugin_list, check=True)

        self.exp._barrier()
        self.exp._clear_data_objects()

        self.exp._barrier()
        cu.user_message("Plugin list check complete!")

    def __check_loaders_and_savers(self):
        """ Check plugin list starts with a loader and ends with a saver.
        """
        plugin_obj = self.exp.meta_data.plugin_list
        loaders, savers = plugin_obj._get_loaders_and_savers_index()

        if loaders:
            if loaders[0] is not 0 or loaders[-1] + 1 is not len(loaders):
                raise Exception("All loader plugins must be at the beginning "
                                "of the plugin list")
        else:
            raise Exception("The first plugin in the plugin list must be a "
                            "loader.")

        if not savers or savers[0] is not plugin_obj.n_plugins - 1:
            raise Exception("The final plugin in the plugin list must be a "
                            "saver")
Пример #20
0
class PluginRunner(object):
    """ Plugin list runner, which passes control to the transport layer.
    """

    def __init__(self, options):
        class_name = "savu.core.transports." + options["transport"] \
                     + "_transport"
        cu.add_base(self, cu.import_class(class_name))
        self._transport_control_setup(options)
        self.exp = None
        self.options = options

    def _run_plugin_list(self):
        """ Create an experiment and run the plugin list.
        """
        self.exp = Experiment(self.options)
        plugin_list = self.exp.meta_data.plugin_list.plugin_list

        logging.info("run_plugin_list: 1")
        self.exp._barrier()
        self._run_plugin_list_check(plugin_list)

        logging.info("run_plugin_list: 2")
        self.exp._barrier()
        expInfo = self.exp.meta_data
        logging.debug("Running process List.save_list_to_file")
        expInfo.plugin_list._save_plugin_list(
            expInfo.get_meta_data("nxs_filename"), exp=self.exp)

        logging.info("run_plugin_list: 3")
        self.exp._barrier()
        self._transport_run_plugin_list()

        logging.info("run_plugin_list: 4")
        self.exp._barrier()

        cu.user_message("***********************")
        cu.user_message("* Processing Complete *")
        cu.user_message("***********************")

        self.exp.nxs_file.close()
        return self.exp

    def _run_plugin_list_check(self, plugin_list):
        """ Run the plugin list through the framework without executing the
        main processing.
        """
        self.exp._barrier()
        self.__check_loaders_and_savers()

        self.exp._barrier()
        pu.run_plugins(self.exp, plugin_list, check=True)

        self.exp._barrier()
        self.exp._clear_data_objects()

        self.exp._barrier()
        cu.user_message("Plugin list check complete!")

    def __check_loaders_and_savers(self):
        """ Check plugin list starts with a loader and ends with a saver.
        """
        plugin_obj = self.exp.meta_data.plugin_list
        loaders, savers = plugin_obj._get_loaders_and_savers_index()

        if loaders:
            if loaders[0] is not 0 or loaders[-1]+1 is not len(loaders):
                raise Exception("All loader plugins must be at the beginning "
                                "of the plugin list")
        else:
            raise Exception("The first plugin in the plugin list must be a "
                            "loader.")

        if not savers or savers[0] is not plugin_obj.n_plugins-1:
            raise Exception("The final plugin in the plugin list must be a "
                            "saver")