def pre_process(self): in_pData = self.get_plugin_in_datasets()[0] in_meta_data = self.get_in_meta_data()[0] data = self.get_in_datasets()[0] data_shape = data.get_shape() width = data_shape[-1] use_mask = self.parameters['masking'] if use_mask is True: ratio = self.parameters['ratio'] if ratio is None: try: cor = np.min(in_meta_data.get('centre_of_rotation')) ratio = (min(cor, abs(width - cor))) / (width * 0.5) except KeyError: ratio = 1.0 self.mask = self.circle_mask(width, ratio) else: self.mask = np.ones((width, width), dtype=np.float32) self.method = self.parameters['method'] if not (self.method == 'percentile' or self.method == 'extrema'): msg = "\n***********************************************\n"\ "!!! ERROR !!! -> Wrong method. Please use only one of "\ "the provided options \n"\ "***********************************************\n" logging.warn(msg) cu.user_message(msg) raise ValueError(msg) self.p_min, self.p_max = np.sort( np.clip(np.asarray(self.parameters['p_range'], dtype=np.float32), 0.0, 100.0))
def _transport_process(self, plugin): """ Organise required data and execute the main plugin processing. :param plugin plugin: The current plugin instance. """ pDict, result, nTrans = self._initialise(plugin) cp, sProc, sTrans = self.__get_checkpoint_params(plugin) count = 0 # temporary solution prange = range(sProc, pDict['nProc']) kill = False for count in range(sTrans, nTrans): end = True if count == nTrans - 1 else False self._log_completion_status(count, nTrans, plugin.name) # get the transfer data transfer_data = self._transfer_all_data(count) # loop over the process data result, kill = self._process_loop(plugin, prange, transfer_data, count, pDict, result, cp) self._return_all_data(count, result, end) if kill: return 1 if not kill: cu.user_message("%s - 100%% complete" % (plugin.name))
def pre_process(self): self.drop = np.int16(self.parameters['row_drop']) self.smin, self.smax = np.int16(self.parameters['search_area']) self.search_radius = np.float32(self.parameters['search_radius']) self.search_step = np.float32(self.parameters['step']) self.ratio = np.float32(self.parameters['ratio']) self.est_cor = self.parameters['start_pixel'] self.broadcast_method = str(self.parameters['broadcast_method']) self.error_msg_1 = "" self.error_msg_2 = "" self.error_msg_3 = "" if not ((self.broadcast_method == 'mean') or (self.broadcast_method == 'median') or (self.broadcast_method == 'linear_fit') or (self.broadcast_method == 'nearest')): self.error_msg_3 = "!!! WARNING !!! Selected broadcasting "\ "method is out of the list. Use the default option: 'median'" logging.warn(self.error_msg_3) cu.user_message(self.error_msg_3) self.broadcast_method = 'median' in_pData = self.get_plugin_in_datasets()[0] data = self.get_in_datasets()[0] starts, stops, steps = data.get_preview().get_starts_stops_steps()[0:3] start_ind = starts[1] stop_ind = stops[1] step_ind = steps[1] name = data.get_name() pre_start = self.exp.meta_data.get(name + '_preview_starts')[1] pre_stop = self.exp.meta_data.get(name + '_preview_stops')[1] pre_step = self.exp.meta_data.get(name + '_preview_steps')[1] self.origin_prev = np.arange(pre_start, pre_stop, pre_step) self.plugin_prev = self.origin_prev[start_ind:stop_ind:step_ind]
def _transport_process(self, plugin): """ Organise required data and execute the main plugin processing. :param plugin plugin: The current plugin instance. """ self.process_setup(plugin) pDict = self.pDict result = [ np.empty(d._get_plugin_data().get_shape_transfer(), dtype=np.float32) for d in pDict['out_data'] ] # loop over the transfer data nTrans = pDict['nTrans'] for count in range(nTrans): end = True if count == nTrans - 1 else False percent_complete = count / (nTrans * 0.01) cu.user_message("%s - %3i%% complete" % (plugin.name, percent_complete)) # get the transfer data transfer_data = self._transfer_all_data(count) # loop over the process data for i in range(pDict['nProc']): data = self._get_input_data(plugin, transfer_data, i, count) res = self._get_output_data(plugin.plugin_process_frames(data), i) for j in pDict['nOut']: out_sl = pDict['out_sl']['process'][i][j] result[j][out_sl] = res[j] self._return_all_data(count, result, end) cu.user_message("%s - 100%% complete" % (plugin.name)) plugin._revert_preview(pDict['in_data'])
def _transport_process(self, plugin): """ Organise required data and execute the main plugin processing. :param plugin plugin: The current plugin instance. """ pDict, result, nTrans = self._initialise(plugin) cp, sProc, sTrans = self.__get_checkpoint_params(plugin) count = 0 # temporary solution prange = range(sProc, pDict['nProc']) kill = False for count in range(sTrans, nTrans): end = True if count == nTrans-1 else False self._log_completion_status(count, nTrans, plugin.name) # get the transfer data transfer_data = self._transfer_all_data(count) # loop over the process data result, kill = self._process_loop( plugin, prange, transfer_data, count, pDict, result, cp) self._return_all_data(count, result, end) if kill: return 1 if not kill: cu.user_message("%s - 100%% complete" % (plugin.name))
def _process(self, plugin): """ Organise required data and execute the main plugin processing. :param plugin plugin: The current plugin instance. """ self.process_checks() in_data, out_data = plugin.get_datasets() expInfo = plugin.exp.meta_data in_slice_list = self.__get_all_slice_lists(in_data, expInfo) out_slice_list = self.__get_all_slice_lists(out_data, expInfo) squeeze_dict = self.__set_functions(in_data, 'squeeze') expand_dict = self.__set_functions(out_data, 'expand') number_of_slices_to_process = len(in_slice_list[0]) for count in range(number_of_slices_to_process): percent_complete = count / (number_of_slices_to_process * 0.01) cu.user_message("%s - %3i%% complete" % (plugin.name, percent_complete)) section, slice_list = \ self.__get_all_padded_data(in_data, in_slice_list, count, squeeze_dict) result = plugin.process_frames(section, slice_list) self.__set_out_data(out_data, out_slice_list, result, count, expand_dict) cu.user_message("%s - 100%% complete" % (plugin.name)) plugin._revert_preview(in_data)
def __real_plugin_run(self, plugin_list, out_data_objs, start, stop): """ Execute the plugin. """ exp = self.exp for i in range(start, stop): link_type = "final_result" if i is len(plugin_list)-2 else \ "intermediate" exp._barrier() for key in out_data_objs[i - start]: exp.index["out_data"][key] = out_data_objs[i - start][key] exp._barrier() plugin = pu.plugin_loader(exp, plugin_list[i]) exp._barrier() cu.user_message("*Running the %s plugin*" % (plugin_list[i]['id'])) plugin._run_plugin(exp, self) exp._barrier() if self.mpi: cu.user_messages_from_all(plugin.name, plugin.executive_summary()) else: for message in plugin.executive_summary(): cu.user_message("%s - %s" % (plugin.name, message)) exp._barrier() out_datasets = plugin.parameters["out_datasets"] exp._reorganise_datasets(out_datasets, link_type)
def _run_plugin_list_setup(self, plugin_list): """ Run the plugin list through the framework without executing the main processing. """ plugin_list._check_loaders() self.__check_gpu() n_loaders = self.exp.meta_data.plugin_list._get_n_loaders() n_plugins = plugin_list._get_n_processing_plugins() plist = plugin_list.plugin_list self.exp._setup(self, plugin_list) # set loaders for i in range(n_loaders): pu.plugin_loader(self.exp, plist[i]) self.exp._set_initial_datasets() # run all plugin setup methods and store information in experiment # collection count = 0 for plugin_dict in plist[n_loaders:n_loaders + n_plugins]: plugin = pu.plugin_loader(self.exp, plugin_dict, check=True) plugin._revert_preview(plugin.get_in_datasets()) plugin_dict['cite'] = plugin.get_citation_information() plugin._clean_up() self.exp._update(plugin_dict) self.exp._merge_out_data_to_in() count += 1 self.exp._reset_datasets() plugin_list._add_missing_savers(self.exp) cu.user_message("Plugin list check complete!") # ********* transport function *********** self._transport_update_plugin_list()
def _run_plugin_list_check(self, plugin_list): """ Run the plugin list through the framework without executing the main processing. """ plugin_list._check_loaders() n_loaders = plugin_list._get_n_loaders() self.__check_gpu() check_list = np.arange(len(plugin_list.plugin_list)) - n_loaders self.__fake_plugin_list_run(plugin_list, check_list, setnxs=True) savers_idx_before = plugin_list._get_savers_index() plugin_list._add_missing_savers(self.exp.index['in_data'].keys()) # ********* transport function *********** self._transport_update_plugin_list() self.exp._clear_data_objects() check_list = np.array(list(set(plugin_list._get_savers_index()). difference(set(savers_idx_before)))) - n_loaders self.__fake_plugin_list_run(plugin_list, check_list) self.exp._clear_data_objects() cu.user_message("Plugin list check complete!")
def real_plugin_run(self, plugin_list, out_data_objs, start, stop): exp = self.exp for i in range(start, stop): link_type = "final_result" if i is len(plugin_list)-2 else \ "intermediate" exp.barrier() for key in out_data_objs[i - start]: exp.index["out_data"][key] = out_data_objs[i - start][key] exp.barrier() plugin = pu.plugin_loader(exp, plugin_list[i]) exp.barrier() cu.user_message("*Running the %s plugin*" % (plugin_list[i]['id'])) plugin.run_plugin(exp, self) exp.barrier() if self.mpi: cu.user_message_from_all(plugin.name, plugin.executive_summary()) else: cu.user_message("%s - %s" % (plugin.name, plugin.executive_summary())) exp.barrier() out_datasets = plugin.parameters["out_datasets"] exp.reorganise_datasets(out_datasets, link_type)
def __run_plugin(self, plugin_dict): plugin = self._transport_load_plugin(self.exp, plugin_dict) # ********* transport function *********** self._transport_pre_plugin() cu.user_message("*Running the %s plugin*" % plugin.name) # ******** transport 'process' function is called inside here ******** plugin._run_plugin(self.exp, self) # plugin driver self.exp._barrier() cu._output_summary(self.exp.meta_data.get("mpi"), plugin) plugin._clean_up() finalise = self.exp._finalise_experiment_for_current_plugin() # ********* transport function *********** self._transport_post_plugin() for data in finalise['remove'] + finalise['replace']: # ********* transport function *********** self._transport_terminate_dataset(data) self.exp._reorganise_datasets(finalise)
def _process(self, plugin): """ Organise required data and execute the main plugin processing. :param plugin plugin: The current plugin instance. """ self.process_checks() in_data, out_data = plugin.get_datasets() expInfo = plugin.exp.meta_data in_slice_list = self.__get_all_slice_lists(in_data, expInfo) out_slice_list = self.__get_all_slice_lists(out_data, expInfo) squeeze_dict = self.__set_functions(in_data, 'squeeze') expand_dict = self.__set_functions(out_data, 'expand') number_of_slices_to_process = len(in_slice_list[0]) for count in range(number_of_slices_to_process): percent_complete = count/(number_of_slices_to_process * 0.01) cu.user_message("%s - %3i%% complete" % (plugin.name, percent_complete)) section, slice_list = \ self.__get_all_padded_data(in_data, in_slice_list, count, squeeze_dict) result = plugin.process_frames(section, slice_list) self.__set_out_data(out_data, out_slice_list, result, count, expand_dict) cu.user_message("%s - 100%% complete" % (plugin.name)) plugin._revert_preview(in_data)
def executive_summary(self): if ((self.error_msg_1 == "") and (self.error_msg_2 == "")): msg = "Centre of rotation is : %s" % (str( self.cor_for_executive_summary)) else: msg = "\n" + self.error_msg_1 + "\n" + self.error_msg_2 msg2 = "(Not well) estimated centre of rotation is : %s" % (str( self.cor_for_executive_summary)) cu.user_message(msg2) return [msg]
def run_plugin_list_check(self, plugin_list): self.exp.barrier() self.check_loaders_and_savers(plugin_list) self.exp.barrier() pu.run_plugins(self.exp, plugin_list, check=True) self.exp.barrier() self.exp.clear_data_objects() self.exp.barrier() cu.user_message("Plugin list check complete!")
def pre_process(self): super(ImageSaver, self).pre_process() self.file_format = self.parameters['format'] num_bit = self.parameters['num_bit'] self.pData = self.get_plugin_in_datasets()[0] if not (num_bit==8 or num_bit==16 or num_bit==32): self.num_bit = 32 msg = "\n***********************************************\n"\ "This option %s is not available. Reset to 32 \n"\ %str(num_bit) cu.user_message(msg) else: self.num_bit = num_bit self._data_range = self._get_min_and_max()
def _run_plugin_list_check(self, plugin_list): """ Run the plugin list through the framework without executing the main processing. """ self.exp._barrier() self.__check_loaders_and_savers() self.exp._barrier() pu.run_plugins(self.exp, plugin_list, check=True) self.exp._barrier() self.exp._clear_data_objects() self.exp._barrier() cu.user_message("Plugin list check complete!")
def check_file_path(self, file_path): file_ext = ".tif" if file_path is None: msg = "!!! Please provide a file path to the MTF !!!" logging.warn(msg) cu.user_message(msg) raise ValueError(msg) else: if not os.path.isfile(file_path): msg = "!!! No such file: %s !!!"\ " Please check the file path" %str(file_path) logging.warn(msg) cu.user_message(msg) raise ValueError(msg) else: _, file_ext = os.path.splitext(file_path) return file_ext
def __get_outer_pad(self): factor = math.sqrt(2)-1 # length of diagonal of square is side*sqrt(2) pad = self.parameters['outer_pad'] if 'outer_pad' in \ self.parameters.keys() else False if pad is not False and not self.padding_alg: msg = 'This reconstruction algorithm cannot be padded.' cu.user_message(msg) return 0 if isinstance(pad, bool): return factor if pad is True else 0 factor = float(pad) if factor > MAX_OUTER_PAD: factor = MAX_OUTER_PAD msg = 'Maximum outer_pad value is 2.1, using this instead' cu.user_message(msg) return float(pad)
def _coarse_search(self, sino, start_cor, stop_cor, ratio, drop): """ Coarse search for finding the rotation center. """ (nrow, ncol) = sino.shape start_cor, stop_cor = np.sort((start_cor, stop_cor)) start_cor = np.int16(np.clip(start_cor, 0, ncol - 1)) stop_cor = np.int16(np.clip(stop_cor, 0, ncol - 1)) cen_fliplr = (ncol - 1.0) / 2.0 # Flip left-right the [0:Pi ] sinogram to make a full [0;2Pi] sinogram flip_sino = np.fliplr(sino) # Below image is used for compensating the shift of the [Pi;2Pi] sinogram # It helps to avoid local minima. comp_sino = np.flipud(sino) list_cor = np.arange(start_cor, stop_cor + 1.0) list_metric = np.zeros(len(list_cor), dtype=np.float32) mask = self._create_mask(2 * nrow, ncol, 0.5 * ratio * ncol, drop) sino_sino = np.vstack((sino, flip_sino)) for i, cor in enumerate(list_cor): shift = np.int16(2.0 * (cor - cen_fliplr)) _sino = sino_sino[nrow:] _sino[...] = np.roll(flip_sino, shift, axis=1) if shift >= 0: _sino[:, :shift] = comp_sino[:, :shift] else: _sino[:, shift:] = comp_sino[:, shift:] list_metric[i] = np.mean( np.abs(np.fft.fftshift(fft.fft2(sino_sino))) * mask) minpos = np.argmin(list_metric) if minpos == 0: self.error_msg_1 = "!!! WARNING !!! Global minimum is out of "\ "the searching range. Please extend smin" logging.warn(self.error_msg_1) cu.user_message(self.error_msg_1) if minpos == len(list_metric) - 1: self.error_msg_2 = "!!! WARNING !!! Global minimum is out of "\ "the searching range. Please extend smax" logging.warn(self.error_msg_2) cu.user_message(self.error_msg_2) rot_centre = list_cor[minpos] return rot_centre
def __find_dark_and_flat(self, data_obj): ignore = self.parameters['ignore_flats'] if \ self.parameters['ignore_flats'] else None try: image_key = \ data_obj.backing_file[self.parameters['image_key_path']][...] from savu.data.data_structures.data_type import ImageKey data_obj.data = \ ImageKey(data_obj, image_key, 0, ignore=ignore) #data_obj.set_shape(data_obj.data.get_shape()) except KeyError: cu.user_message("An image key was not found.") try: from savu.data.data_structures.data_type import NoImageKey data_obj.data = NoImageKey(data_obj, None, 0) entry = 'entry1/tomo_entry/instrument/detector/' data_obj.data._set_flat_path(entry + 'flatfield') data_obj.data._set_dark_path(entry + 'darkfield') except KeyError: cu.user_message("Dark/flat data was not found in input file.")
def _run_plugin_list(self): """ Create an experiment and run the plugin list. """ self.exp = Experiment(self.options) plugin_list = self.exp.meta_data.plugin_list.plugin_list logging.info("run_plugin_list: 1") self.exp._barrier() self._run_plugin_list_check(plugin_list) logging.info("run_plugin_list: 2") self.exp._barrier() expInfo = self.exp.meta_data logging.debug("Running process List.save_list_to_file") expInfo.plugin_list._save_plugin_list( expInfo.get_meta_data("nxs_filename"), exp=self.exp) logging.info("run_plugin_list: 3") self.exp._barrier() self._transport_run_plugin_list() logging.info("run_plugin_list: 4") self.exp._barrier() cu.user_message("***********************") cu.user_message("* Processing Complete *") cu.user_message("***********************") self.exp.nxs_file.close() return self.exp
def _get_min_and_max(self): data = self.get_in_datasets()[0] pattern = self.parameters['pattern'] try: self.the_min = np.min(data.meta_data.get(['stats', 'min', pattern])) self.the_max = np.max(data.meta_data.get(['stats', 'max', pattern])) self._data_range = (self.the_min, self.the_max) except KeyError: self._data_range = 'image' if (self.file_format=="tiff") or (self.file_format =="tif"): self.the_min = None self.the_max = None msg = "\n***********************************************\n"\ "!!!Warning!!!-> No global maximum and global minimum found\n"\ "in the metadata. Please run the MaxAndMin plugin before\n" \ "ImageSaver or input manually. Otherwise, local minimum\n" \ "and local maximum will be used for rescaling. This may\n"\ "result the fluctuation of brightness between slices.\n"\ "***********************************************\n" if (self.num_bit == 8) or (self.num_bit==16): cu.user_message(msg) return self._data_range
def __find_dark_and_flat(self, data_obj, flat=None, dark=None): ignore = self.parameters['ignore_flats'] if \ self.parameters['ignore_flats'] else None try: image_key = data_obj.backing_file[ 'entry1/tomo_entry/instrument/detector/image_key'][...] data_obj.data = \ ImageKey(data_obj, image_key, 0, ignore=ignore) except KeyError: cu.user_message("An image key was not found.") try: data_obj.data = NoImageKey(data_obj, None, 0) entry = 'entry1/tomo_entry/instrument/detector/' data_obj.data._set_flat_path(entry + 'flatfield') data_obj.data._set_dark_path(entry + 'darkfield') except KeyError: cu.user_message("Dark/flat data was not found in input file.") data_obj.data._set_dark_and_flat() if dark: data_obj.data.update_dark(dark) if flat: data_obj.data.update_flat(flat)
def __output_final_message(self): kill = True if 'killsignal' in \ self.exp.meta_data.get_dictionary().keys() else False msg = "interrupted by killsignal" if kill else "Complete" stars = 40 if kill else 23 cu.user_message("*"*stars) cu.user_message("* Processing " + msg + " *") cu.user_message("*"*stars)
def __run_plugin(self, plugin_dict): plugin = self._transport_load_plugin(self.exp, plugin_dict) # ********* transport function *********** self._transport_pre_plugin() cu.user_message("*Running the %s plugin*" % plugin.name) # ******** transport 'process' function is called inside here ******** plugin._run_plugin(self.exp, self) # plugin driver self.exp._barrier(msg="Plugin returned from driver in Plugin Runner") cu._output_summary(self.exp.meta_data.get("mpi"), plugin) plugin._clean_up() finalise = self.exp._finalise_experiment_for_current_plugin() # ********* transport function *********** self._transport_post_plugin() for data in finalise['remove'] + finalise['replace']: # ********* transport function *********** self._transport_terminate_dataset(data) self.exp._reorganise_datasets(finalise)
def __hdf5_file_write_failed_check(self, nBytes, nProcs): _2GB = 2e9 if nBytes / np.float(nProcs) < _2GB: return msg = "The data is too big for the number of processes, please " if self.exp.meta_data.get('femail') == \ '*****@*****.**': n_procs_big = 160 # number of processes for BIG data savu_mpi_big = True if nProcs is n_procs_big else False if savu_mpi_big or (nBytes / np.float(n_procs_big) >= _2GB): if self.exp.meta_data.get('femail'): msg += ("contact %s" % self.exp.meta_data.get('femail')) else: msg += "increase the number of cores." else: msg += "use savu_mpi_big." else: msg += "increase the number of cores." cu.user_message(msg) raise Exception(msg)
def _process(self, plugin): """ Organise required data and execute the main plugin processing. :param plugin plugin: The current plugin instance. """ in_data, out_data = plugin.get_datasets() expInfo = plugin.exp.meta_data in_slice_list, in_global_frame_idx = \ self.__get_all_slice_lists(in_data, expInfo) out_slice_list, _ = self.__get_all_slice_lists(out_data, expInfo) plugin.set_global_frame_index(in_global_frame_idx) squeeze_dict = self.__set_functions(in_data, 'squeeze') expand_dict = self.__set_functions(out_data, 'expand') number_of_slices_to_process = len(in_slice_list[0]) output_counter = -1 for count in range(number_of_slices_to_process): percent_complete = count / (number_of_slices_to_process * 0.01) rounded_amount_through = percent_complete // 5 if (rounded_amount_through) != output_counter: cu.user_message("%s - %3i%% complete" % (plugin.name, percent_complete)) output_counter = rounded_amount_through section, slice_list = \ self.__get_all_padded_data(in_data, in_slice_list, count, squeeze_dict) plugin.set_current_slice_list(slice_list) result = plugin.process_frames(section) self.__set_out_data(out_data, out_slice_list, result, count, expand_dict) cu.user_message("%s - 100%% complete" % (plugin.name)) plugin._revert_preview(in_data)
def _stitch_data(self, dObj, path, data): stype, dim = self._get_stitching_info(data) remove = data['remove'] if 'remove' in list(data.keys()) else None group_name, data_name = os.path.split(path) # find all files with the given name group = dObj.backing_file.require_group(group_name) matches = fnmatch.filter(list(group.keys()), data_name) number = [] for m in matches: diff_number = '' for diff in difflib.ndiff(m, data_name): split = diff.split('- ') if len(split) > 1: diff_number += split[-1] number.append(int(diff_number)) matches = [matches[i] for i in np.argsort(number)] dObj.data_info.set('wildcard_values', sorted(number)) data_obj_list = [] for match in matches: match_path = os.path.join(group_name, match) sub_obj = self.exp.create_data_object('in_data', match) sub_obj.backing_file = dObj.backing_file data_obj_list.append(self._setup_data(sub_obj, match_path)) del self.exp.index['in_data'][match] if data_obj_list: dObj.data = StitchData(data_obj_list, stype, dim, remove=remove) dObj.set_original_shape(dObj.data.get_shape()) else: cu.user_message("The data set %s is empty." % data_name) return dObj
def _run_plugin_list_setup(self, plugin_list): """ Run the plugin list through the framework without executing the main processing. """ plugin_list._check_loaders() self.__check_gpu() n_loaders = self.exp.meta_data.plugin_list._get_n_loaders() n_plugins = plugin_list._get_n_processing_plugins() plist = plugin_list.plugin_list # set loaders for i in range(n_loaders): pu.plugin_loader(self.exp, plist[i]) self.exp._set_initial_datasets() # run all plugin setup methods and store information in experiment # collection count = 0 for plugin_dict in plist[n_loaders:n_loaders + n_plugins]: self.__plugin_setup(plugin_dict, count) count += 1 plugin_list._add_missing_savers(self.exp) # ********* transport function *********** self._transport_update_plugin_list() # check added savers for plugin_dict in plist[n_loaders + count:]: self.__plugin_setup(plugin_dict, count) count += 1 self.exp._reset_datasets() self.exp._finalise_setup(plugin_list) cu.user_message("Plugin list check complete!")
def process(self, plugin): self.process_checks() in_data, out_data = plugin.get_datasets() expInfo = plugin.exp.meta_data in_slice_list = self.get_all_slice_lists(in_data, expInfo) out_slice_list = self.get_all_slice_lists(out_data, expInfo) squeeze_dict = self.set_functions(in_data, 'squeeze') expand_dict = self.set_functions(out_data, 'expand') number_of_slices_to_process = len(in_slice_list[0]) for count in range(number_of_slices_to_process): percent_complete = count/(number_of_slices_to_process * 0.01) cu.user_message("%s - %3i%% complete" % (plugin.name, percent_complete)) section, slice_list = \ self.get_all_padded_data(in_data, in_slice_list, count, squeeze_dict) result = plugin.process_frames(section, slice_list) self.set_out_data(out_data, out_slice_list, result, count, expand_dict) cu.user_message("%s - 100%% complete" % (plugin.name)) plugin.revert_preview(in_data)
def pre_process(self): inData = self.get_in_datasets()[0] dark = inData.data.dark() flat = inData.data.flat() self.data_size = inData.get_shape() (self.depth, self.height, self.width) = flat.shape file_path = self.parameters["file_path"] file_ext = self.check_file_path(file_path) if file_ext == ".npy": try: self.mtf_array = 1.0 * np.load(file_path) except IOError: msg = "\n*****************************************\n"\ "!!! ERROR !!! -> Can't open this file: %s \n"\ "*****************************************\n\ " % str(file_path) logging.warn(msg) cu.user_message(msg) raise ValueError(msg) else: try: self.mtf_array = 1.0 * np.float32(self.load_image(file_path)) except IOError: msg = "\n*****************************************\n"\ "!!! ERROR !!! -> Can't open this file: %s \n"\ "*****************************************\n\ " % str(file_path) logging.warn(msg) cu.user_message(msg) raise ValueError(msg) self.mtf_array[self.mtf_array <= 0.0] = 1.0 self.mtf_array = self.mtf_array / np.max(self.mtf_array) (height_mtf, width_mtf) = self.mtf_array.shape if (self.height != height_mtf) or (self.width != width_mtf): msg = "\n*****************************************\n"\ "!!! ERROR !!!-> Projection shape: ({0},{1}) is not the same as "\ "the mtf shape: ({2},{3})".format( self.height, self.width, height_mtf, width_mtf) logging.warn(msg) cu.user_message(msg) raise ValueError(msg) self.pad_width = np.clip(int(self.parameters["pad_width"]), 0, None) if flat.size: flat_updated = np.ones_like(flat, dtype=np.float32) for i in np.arange(self.depth): flat_updated[i] = self.psf_correction(flat[i], self.mtf_array, self.pad_width) inData.data.update_flat(flat_updated)
def _run_plugin_list(self): """ Create an experiment and run the plugin list. """ self.exp.checkpoint = None plugin_list = self.exp.meta_data.plugin_list plugin_list._check_loaders() self.exp._set_nxs_filename() if self.exp.meta_data.get('process') == 0: fname = self.exp.meta_data.get('nxs_filename') plugin_list._save_plugin_list(fname) self.exp._set_loaders() # ********* transport function *********** self._transport_pre_plugin_list_run() n_plugins = plugin_list._get_n_processing_plugins() n_loaders = self.exp.meta_data.plugin_list._get_n_loaders() plist = plugin_list.plugin_list count = 0 for plugin_dict in plist[n_loaders:n_loaders+n_plugins]: self.exp.meta_data.set('nPlugin', count) self.__run_plugin(plugin_dict) count += 1 # terminate any remaining datasets for data in self.exp.index['in_data'].values(): self._transport_terminate_dataset(data) cu.user_message("***********************") cu.user_message("* Processing Complete *") cu.user_message("***********************") logging.info('Processing complete') return self.exp
def _run_plugin_list(self): """ Create an experiment and run the plugin list. """ plugin_list = self.exp.meta_data.plugin_list logging.info('Running the plugin list check') self._run_plugin_list_check(plugin_list) logging.info('Setting up the experiment') self.exp._experiment_setup() exp_coll = self.exp._get_experiment_collection() n_plugins = plugin_list._get_n_processing_plugins() # ********* transport function *********** logging.info('Running transport_pre_plugin_list_run()') self._transport_pre_plugin_list_run() for i in range(n_plugins): self.exp._set_experiment_for_current_plugin(i) self.__run_plugin(exp_coll['plugin_dict'][i]) # ********* transport function *********** logging.info('Running transport_post_plugin_list_run') self._transport_post_plugin_list_run() # terminate any remaining datasets for data in self.exp.index['in_data'].values(): self._transport_terminate_dataset(data) cu.user_message("***********************") cu.user_message("* Processing Complete *") cu.user_message("***********************") logging.info('Processing complete') if self.exp.meta_data.get('email'): cu.send_email(self.exp.meta_data.get('email')) return self.exp
def _log_completion_status(self, count, nTrans, name): percent_complete = count / (nTrans * 0.01) cu.user_message("%s - %3i%% complete" % (name, percent_complete))
def executive_summary(self): if self.msg != "": cu.user_message(self.msg) raise ValueError(self.msg) else: return ["Nothing to Report"]
def pre_process(self): in_pData = self.get_plugin_in_datasets()[0] data = self.get_in_datasets()[0] name = data.get_name() shift = self.exp.meta_data.get(name + '_preview_starts') step = self.exp.meta_data.get(name + '_preview_steps') x_dim = data.get_data_dimension_by_axis_label('detector_x') y_dim = data.get_data_dimension_by_axis_label('detector_y') step_check = \ True if max([step[i] for i in [x_dim, y_dim]]) > 1 else False if step_check: self.msg = "\n***********************************************\n"\ "!!! ERROR !!! -> Plugin doesn't work with the step in the "\ "preview larger than 1 \n"\ "***********************************************\n" logging.warn(self.msg) cu.user_message(self.msg) raise ValueError(self.msg) x_offset = shift[x_dim] y_offset = shift[y_dim] file_path = self.parameters["file_path"] self.msg = "" x_center = 0.0 y_center = 0.0 if file_path is None: x_center = np.asarray(self.parameters['center_from_left'], dtype=np.float32) - x_offset y_center = np.asarray(self.parameters['center_from_top'], dtype=np.float32) - y_offset list_fact = np.float32(self.parameters['polynomial_coeffs']) else: if not (os.path.isfile(file_path)): self.msg = "!!! No such file: %s !!!"\ " Please check the file path" %str(file_path) cu.user_message(self.msg) raise ValueError(self.msg) try: (x_center, y_center, list_fact) = self.load_metadata_txt(file_path) x_center = x_center - x_offset y_center = y_center - y_offset except IOError: self.msg = "\n*****************************************\n"\ "!!! ERROR !!! -> Can't open this file: %s \n"\ "*****************************************\n\ " % str(file_path) logging.warn(self.msg) cu.user_message(self.msg) raise ValueError(self.msg) data_shape = data.get_shape() self.height, self.width = data_shape[y_dim], data_shape[x_dim] xu_list = np.arange(self.width) - x_center yu_list = np.arange(self.height) - y_center xu_mat, yu_mat = np.meshgrid(xu_list, yu_list) ru_mat = np.sqrt(xu_mat**2 + yu_mat**2) fact_mat = np.sum(np.asarray( [factor * ru_mat**i for i, factor in enumerate(list_fact)]), axis=0) xd_mat = np.float32( np.clip(x_center + fact_mat * xu_mat, 0, self.width - 1)) yd_mat = np.float32( np.clip(y_center + fact_mat * yu_mat, 0, self.height - 1)) diff_y = np.max(yd_mat) - np.min(yd_mat) if (diff_y < 1): self.msg = "\n*****************************************\n\n"\ "!!! ERROR !!! -> You need to increase the preview size"\ " for this plugin to work \n\n"\ "*****************************************\n" logging.warn(self.msg) cu.user_message(self.msg) raise ValueError(self.msg) self.indices = np.reshape(yd_mat, (-1, 1)),\ np.reshape(xd_mat, (-1, 1))
def _log_completion_status(self, count, nTrans, name): percent_complete = count/(nTrans * 0.01) cu.user_message("%s - %3i%% complete" % (name, percent_complete))