def _add_row_to_table(self, ws_name, row, run_no=None, bank=None, plotted=False, bgsub=False, niter=50, xwindow=None, SG=True): words = ws_name.split("_") # find xwindow from ws xunit if not specified if not xwindow: ws = self.model.get_loaded_workspaces()[ws_name] if ws.getAxis(0).getUnit().unitID() == "TOF": xwindow = 600 else: xwindow = 0.02 if run_no is not None and bank is not None: self.view.add_table_row(run_no, bank, plotted, bgsub, niter, xwindow, SG) elif len(words) == 4 and words[2] == "bank": logger.notice( "No sample logs present, determining information from workspace name." ) self.view.add_table_row(words[1], words[3], plotted, bgsub, niter, xwindow, SG) else: logger.warning( "The workspace '{}' was not in the correct naming format. Files should be named in the following way: " "INSTRUMENT_RUNNUMBER_bank_BANK. Using workspace name as identifier." .format(ws_name)) self.view.add_table_row(ws_name, "N/A", plotted, bgsub, niter, xwindow, SG)
def do_fit_all(self, ws_list, do_sequential=True): fitprop_list = [] prev_fitprop = self.view.read_fitprop_from_browser() for ws in ws_list: logger.notice(f'Starting to fit workspace {ws}') fitprop = deepcopy(prev_fitprop) # update I/O workspace name fitprop['properties']['Output'] = ws fitprop['properties']['InputWorkspace'] = ws # do fit fit_output = Fit(**fitprop['properties']) # update results fitprop['status'] = fit_output.OutputStatus funcstr = str(fit_output.Function.fun) fitprop['properties']['Function'] = funcstr if "success" in fitprop['status'].lower() and do_sequential: # update function in prev fitprop to use for next workspace prev_fitprop['properties']['Function'] = funcstr # update last fit in fit browser and save setup self.view.update_browser(fit_output.OutputStatus, funcstr, ws) # append a deep copy to output list (will be initial parameters if not successful) fitprop_list.append(fitprop) logger.notice('Sequential fitting finished.') self.fit_all_done_notifier.notify_subscribers(fitprop_list)
def _save_output(self, instrument, sample_run, van_run, bank, sample_workspace, rb_num, unit="TOF"): """ Save a focused workspace to the file system. Saves separate copies to a User directory if an rb number has been set. :param instrument: The instrument the data is from. :param sample_run: The sample run number that was focussed :param bank: The name of the bank being saved. :param sample_workspace: The name of the workspace to be saved. :param rb_num: Usually an experiment id, defines the name of the user directory. """ self._save_focused_output_files_as_nexus(instrument, sample_run, van_run, bank, sample_workspace, rb_num, unit) self._save_focused_output_files_as_gss(instrument, sample_run, van_run, bank, sample_workspace, rb_num, unit) self._save_focused_output_files_as_topas_xye(instrument, sample_run, van_run, bank, sample_workspace, rb_num, unit) output_path = path.join(output_settings.get_output_path(), 'Focus') logger.notice(f"\n\nFocus files saved to: \"{output_path}\"\n\n") if rb_num: output_path = path.join(output_settings.get_output_path(), 'User', rb_num, 'Focus') logger.notice( f"\n\nFocus files also saved to: \"{output_path}\"\n\n")
def _check_region_grouping_ws_exists(grouping_ws_name: str, inst_ws) -> bool: """ Check that the required grouping workspace for this focus exists, and if not present for a North/South bank focus, retrieve them from the user directories or create them (expected if first focus with loaded calibration) :param grouping_ws_name: Name of the grouping workspace whose presence in the ADS is being checked :param inst_ws: Workspace containing the instrument data for use in making a bank grouping workspace :return: True if the required workspace exists (or has just been loaded/created), False if not """ if not Ads.doesExist(grouping_ws_name): if "North" in grouping_ws_name: logger.notice( "NorthBank grouping workspace not present in ADS, loading") EnggUtils.get_bank_grouping_workspace(1, inst_ws) return True elif "South" in grouping_ws_name: logger.notice( "SouthBank grouping workspace not present in ADS, loading") EnggUtils.get_bank_grouping_workspace(2, inst_ws) return True else: logger.warning( f"Cannot focus as the grouping workspace \"{grouping_ws_name}\" is not present." ) return False return True
def PyExec(self): # setup progress bar prog_reporter = Progress(self, start=0.0, end=1.0, nreports=3) # Get input ws_list = self.getProperty("PeakWorkspaces").value a = self.getProperty('a').value b = self.getProperty('b').value c = self.getProperty('c').value alpha = self.getProperty('alpha').value beta = self.getProperty('beta').value gamma = self.getProperty('gamma').value self.tol = self.getProperty('Tolerance').value # Find initial UB and use to index peaks in all runs prog_reporter.report(1, "Find initial UB for peak indexing") self.find_initial_indexing( a, b, c, alpha, beta, gamma, ws_list) # removes runs from ws_list if can't index # optimize the lattice parameters across runs (i.e. B matrix) prog_reporter.report(2, "Optimize B") def fobj(x): return self.calcResiduals(x, ws_list) alatt0 = [a, b, c, alpha, beta, gamma] try: alatt, cov, info, msg, ier = leastsq(fobj, x0=alatt0, full_output=True) # eval the fobj at optimal solution to set UB (leastsq iteration stops at a next sub-optimal solution) fobj(alatt) except ValueError: logger.error( "CalculateUMatrix failed - check initial lattice parameters and tolerance provided." ) return success = ier in [ 1, 2, 3, 4 ] and cov is not None # cov is None when matrix is singular if success: # calculate errors dof = sum( [self.child_IndexPeaks(ws, RoundHKLs=True) for ws in ws_list]) - len(alatt0) err = np.sqrt(abs(np.diag(cov)) * (info['fvec']**2).sum() / dof) for wsname in ws_list: ws = AnalysisDataService.retrieve(wsname) ws.sample().getOrientedLattice().setError(*err) logger.notice( f"Lattice parameters successfully refined for workspaces: {ws_list}\n" f"Lattice Parameters: {np.array2string(alatt, precision=6)}\n" f"Parameter Errors : {np.array2string(err, precision=6)}") else: logger.warning( f"Error in optimization of lattice parameters: {msg}") # complete progress prog_reporter.report(3, "Done")
def create_or_update_bgsub_ws(self, ws_name, bg_params): ws = self._loaded_workspaces[ws_name] ws_bg = self._bg_sub_workspaces[ws_name] if not ws_bg or self._bg_params[ws_name] == [] or bg_params[1:] != self._bg_params[ws_name][1:]: background = self.estimate_background(ws_name, *bg_params[1:]) self._bg_params[ws_name] = bg_params bgsub_ws_name = ws_name + "_bgsub" bgsub_ws = Minus(LHSWorkspace=ws, RHSWorkspace=background, OutputWorkspace=bgsub_ws_name) self._bg_sub_workspaces[ws_name] = bgsub_ws DeleteWorkspace(background) else: logger.notice("Background workspace already calculated")
def needs_loading(property_value, loading_reduction_type): """ Checks whether a given unary input needs loading or is already loaded in ADS. @param property_value: the string value of the corresponding FileProperty @param loading_reduction_type : the reduction_type of input to load """ loading = False ws_name = '' if property_value: ws_name = path.splitext(path.basename(property_value))[0] if mtd.doesExist(ws_name): logger.notice('Reusing {0} workspace: {1}'.format( loading_reduction_type, ws_name)) else: loading = True return [loading, ws_name]
def _add_row_to_table(self, ws_name, row, run_no=None, bank=None, checked=False): words = ws_name.split("_") if run_no is not None and bank is not None: self.view.add_table_row(run_no, bank, checked) self.row_numbers[ws_name] = row elif len(words) == 4 and words[2] == "bank": logger.notice( "No sample logs present, determining information from workspace name." ) self.view.add_table_row(words[1], words[3], checked) self.row_numbers[ws_name] = row else: logger.warning( "The workspace '{}' was not in the correct naming format. Files should be named in the following way: " "INSTRUMENT_RUNNUMBER_bank_BANK. Using workspace name as identifier." .format(ws_name)) self.view.add_table_row(ws_name, "N/A", checked) self.row_numbers[ws_name] = row
def needs_processing(property_value, process_reduction_type): """ Checks whether a given unary reduction needs processing or is already cached in ADS with expected name. @param property_value: the string value of the corresponding MultipleFile input property @param process_reduction_type: the reduction_type of process """ do_process = False ws_name = '' if property_value: run_number = get_run_number(property_value) ws_name = run_number + '_' + process_reduction_type if mtd.doesExist(ws_name): if isinstance(mtd[ws_name], WorkspaceGroup): run = mtd[ws_name][0].getRun() else: run = mtd[ws_name].getRun() if run.hasProperty('ProcessedAs'): process = run.getLogData('ProcessedAs').value if process == process_reduction_type: logger.notice('Reusing {0} workspace: {1}'.format( process_reduction_type, ws_name)) else: logger.warning('{0} workspace found, but processed ' 'differently: {1}'.format( process_reduction_type, ws_name)) do_process = True else: logger.warning('{0} workspace found, but missing the ' 'ProcessedAs flag: {1}'.format( process_reduction_type, ws_name)) do_process = True else: do_process = True return [do_process, ws_name]
def on_calibrate_clicked(self): plot_output = self.view.get_plot_output() if self.view.get_new_checked() and self._validate(): sample_file = self.view.get_sample_filename() if self.view.get_crop_checked(): self.start_cropped_calibration_worker(sample_file, plot_output, self.rb_num) else: self.start_calibration_worker(sample_file,plot_output, self.rb_num) elif self.view.get_load_checked(): if not self.validate_path(): logger.notice("Invalid calibration path") return filename = self.view.get_path_filename() try: instrument, sample_file, grp_ws_name, roi_text, banks = \ self.model.load_existing_calibration_files(filename) except: return self.pending_calibration.set_calibration(sample_file, instrument) self.pending_calibration.set_roi_info_load(banks, grp_ws_name, roi_text) self.set_current_calibration() set_setting(output_settings.INTERFACES_SETTINGS_GROUP, output_settings.ENGINEERING_PREFIX, "last_calibration_path", filename)
def process_json(json_filename): """This will read a json file, process the data and save the calibration. Only ``Calibrant`` and ``Groups`` are required. An example input showing every possible options is: .. code-block:: JSON { "Calibrant": "12345", "Groups": "/path/to/groups.xml", "Mask": "/path/to/mask.xml", "Instrument": "NOM", "Date" : "2019_09_04", "SampleEnvironment": "shifter", "PreviousCalibration": "/path/to/cal.h5", "CalDirectory": "/path/to/output_directory", "CrossCorrelate": {"Step": 0.001, "DReference: 1.5, "Xmin": 1.0, "Xmax": 3.0, "MaxDSpaceShift": 0.25}, "PDCalibration": {"PeakPositions": [1, 2, 3], "TofBinning": (300,0.001,16666), "PeakFunction": 'Gaussian', "PeakWindow": 0.1, "PeakWidthPercent": 0.001} } """ with open(json_filename) as json_file: args = json.load(json_file) calibrant_file = args.get('CalibrantFile', None) if calibrant_file is None: calibrant = args['Calibrant'] groups = args['Groups'] out_groups_by = args.get('OutputGroupsBy', 'Group') sample_env = args.get('SampleEnvironment', 'UnknownSampleEnvironment') mask = args.get('Mask') instrument = args.get('Instrument', 'NOM') cc_kwargs = args.get('CrossCorrelate', {}) pdcal_kwargs = args.get('PDCalibration', {}) previous_calibration = args.get('PreviousCalibration') date = str(args.get('Date', datetime.datetime.now().strftime('%Y_%m_%d'))) caldirectory = str(args.get('CalDirectory', os.path.abspath('.'))) if calibrant_file is not None: ws = Load(calibrant_file) calibrant = ws.getRun().getProperty('run_number').value else: filename = f'{instrument}_{calibrant}' ws = Load(filename) calfilename = f'{caldirectory}/{instrument}_{calibrant}_{date}_{sample_env}.h5' logger.notice(f'going to create calibration file: {calfilename}') groups = LoadDetectorsGroupingFile(groups, InputWorkspace=ws) if mask: mask = LoadMask(instrument, mask) MaskDetectors(ws, MaskedWorkspace=mask) if previous_calibration: previous_calibration = LoadDiffCal(previous_calibration, MakeGroupingWorkspace=False, MakeMaskWorkspace=False) diffcal = do_group_calibration(ws, groups, previous_calibration, cc_kwargs=cc_kwargs, pdcal_kwargs=pdcal_kwargs) mask = mtd['group_calibration_pd_diffcal_mask'] CreateGroupingWorkspace(InputWorkspace=ws, GroupDetectorsBy=out_groups_by, OutputWorkspace='out_groups') SaveDiffCal(CalibrationWorkspace=diffcal, MaskWorkspace=mask, GroupingWorkspace=mtd['out_groups'], Filename=calfilename)
def cc_calibrate_groups(data_ws, group_ws, output_basename="_tmp_group_cc_calibration", previous_calibration=None, Step=0.001, DReference=1.2615, Xmin=1.22, Xmax=1.30, MaxDSpaceShift=None, OffsetThreshold=1E-4, SkipCrossCorrelation=[], PeakFunction="Gaussian", SmoothNPoints=0): """This will perform the CrossCorrelate/GetDetectorOffsets on a group of detector pixel. It works by looping over the different groups in the group_ws, extracting all unmasked spectra of a group, then running CrossCorrelate and GetDetectorOffsets on just that group, and combinning the results at the end. When running a group, CrossCorrelate and GetDetectorOffsets could be cycled until converging of offsets is reached, given the user input offset threshold. If offset threshold is specified to be equal to or larger than 1.0, no cycling will be carried out. The first unmasked spectra of the group will be used for the ReferenceSpectra in CrossCorrelate. :param data_ws: Input calibration raw data (in TOF), assumed to already be correctly masked :param group_ws: grouping workspace, e.g. output from LoadDetectorsGroupingFile :param output_basename: Optional name to use for temporay and output workspace :param previous_calibration: Optional previous diffcal workspace :param Step: step size for binning of data and input for GetDetectorOffsets, default 0.001 :param DReference: Derefernce parameter for GetDetectorOffsets, default 1.2615 :param Xmin: Xmin parameter for CrossCorrelate, default 1.22 :param Xmax: Xmax parameter for CrossCorrelate, default 1.30 :param MaxDSpaceShift: MaxDSpaceShift paramter for CrossCorrelate, default None :param OffsetThreshold: Convergence threshold for cycling cross correlation, default 1E-4 :param SkipCrossCorrelation: Skip cross correlation for specified groups :param PeakFunction: Peak function to use for extracting the offset :param SmoothNPoints: Number of points for smoothing spectra, for cross correlation ONLY :return: Combined DiffCal workspace from all the different groups """ if previous_calibration: ApplyDiffCal(data_ws, CalibrationWorkspace=previous_calibration) data_d = ConvertUnits(data_ws, Target='dSpacing', OutputWorkspace='data_d') group_list = np.unique(group_ws.extractY()) _accum_cc = None to_skip = [] for group in group_list: # Figure out input parameters for CrossCorrelate and GetDetectorOffset, specifically # for those parameters for which both a single value and a list is accepted. If a # list is given, that means different parameter setup will be used for different groups. Xmin_group = Xmin[int(group) - 1] if type(Xmin) == list else Xmin Xmax_group = Xmax[int(group) - 1] if type(Xmax) == list else Xmax MDS_group = MaxDSpaceShift[int(group) - 1] if type(MaxDSpaceShift) == list else MaxDSpaceShift DRef_group = DReference[int(group) - 1] if type(DReference) == list else DReference OT_group = OffsetThreshold[int(group) - 1] if type(OffsetThreshold) == list else OffsetThreshold pf_group = PeakFunction[int(group) - 1] if type(PeakFunction) == list else PeakFunction snpts_group = SmoothNPoints[int(group) - 1] if type(SmoothNPoints) == list else SmoothNPoints cycling = OT_group < 1.0 indexes = np.where(group_ws.extractY().flatten() == group)[0] sn = np.array(group_ws.getSpectrumNumbers())[indexes] try: ws_indexes = [data_d.getIndexFromSpectrumNumber(int(i)) for i in sn] except RuntimeError: # data does not contain spectrum in group continue if group in SkipCrossCorrelation: to_skip.extend(ws_indexes) ExtractSpectra(data_d, WorkspaceIndexList=ws_indexes, OutputWorkspace='_tmp_group_cc') ExtractUnmaskedSpectra('_tmp_group_cc', OutputWorkspace='_tmp_group_cc') ExtractSpectra(data_ws, WorkspaceIndexList=ws_indexes, OutputWorkspace='_tmp_group_cc_raw') ExtractUnmaskedSpectra('_tmp_group_cc_raw', OutputWorkspace='_tmp_group_cc_raw') num_spectra = mtd['_tmp_group_cc'].getNumberHistograms() if num_spectra < 2: continue Rebin('_tmp_group_cc', Params=f'{Xmin_group},{Step},{Xmax_group}', OutputWorkspace='_tmp_group_cc') if snpts_group >= 3: SmoothData('_tmp_group_cc', NPoints=snpts_group, OutputWorkspace='_tmp_group_cc') # Figure out brightest spectra to be used as the reference for cross correlation. CloneWorkspace('_tmp_group_cc_raw', OutputWorkspace='_tmp_group_cc_raw_tmp') intg = Integration('_tmp_group_cc_raw_tmp', StartWorkspaceIndex=0, EndWorkspaceIndex=num_spectra-1, OutputWorkspace='_tmp_group_intg') brightest_spec_index = int(np.argmax(np.array([intg.readY(i)[0] for i in range(num_spectra)]))) # Cycling cross correlation. At each step, we will use the obtained offsets and DIFC's from # previous step to obtain new DIFC's. In this way, spectra in group will come closer and closer # to each other as the cycle goes. This will continue until converging criterion is reached. The # converging criterion is set in such a way that the median value of all the non-zero offsets # should be smaller than the threshold (user tuned parameter, default to 1E-4, meaning 0.04% # relative offset). num_cycle = 1 while True: CrossCorrelate('_tmp_group_cc', Xmin=Xmin_group, XMax=Xmax_group, MaxDSpaceShift=MDS_group, ReferenceSpectra=brightest_spec_index, WorkspaceIndexMin=0, WorkspaceIndexMax=num_spectra-1, OutputWorkspace='_tmp_group_cc') bin_range = (Xmax_group-Xmin_group)/Step GetDetectorOffsets(InputWorkspace='_tmp_group_cc', Step=Step, Xmin=-bin_range, XMax=bin_range, DReference=DRef_group, MaxOffset=1, PeakFunction=pf_group, OutputWorkspace='_tmp_group_cc') if group not in SkipCrossCorrelation: offsets_tmp = [] for item in ws_indexes: if abs(mtd['_tmp_group_cc'].readY(item)) != 0: offsets_tmp.append(abs(mtd['_tmp_group_cc'].readY(item))) offsets_tmp = np.array(offsets_tmp) logger.notice(f'Running group-{group}, cycle-{num_cycle}.') logger.notice(f'Median offset (no sign) = {np.median(offsets_tmp)}') logger.notice(f'Running group-{group}, cycle-{num_cycle}.') logger.notice(f'Median offset (no sign) = {np.median(offsets_tmp)}') converged = np.median(offsets_tmp) < OT_group else: for item in ws_indexes: mtd['_tmp_group_cc'].dataY(item)[0] = 0.0 logger.notice(f'Cross correlation skipped for group-{group}.') converged = True if not cycling or converged: if cycling and converged: if group not in SkipCrossCorrelation: logger.notice(f'Cross correlation for group-{group} converged, ') logger.notice(f'with offset threshold {OT_group}.') break else: previous_calibration = ConvertDiffCal('_tmp_group_cc', PreviousCalibration=previous_calibration, OutputWorkspace='_tmp_group_cc_diffcal') ApplyDiffCal('_tmp_group_cc_raw', CalibrationWorkspace='_tmp_group_cc_diffcal') ConvertUnits('_tmp_group_cc_raw', Target='dSpacing', OutputWorkspace='_tmp_group_cc') Rebin('_tmp_group_cc', Params=f'{Xmin_group},{Step},{Xmax_group}', OutputWorkspace='_tmp_group_cc') num_cycle += 1 if not _accum_cc: _accum_cc = RenameWorkspace('_tmp_group_cc') else: _accum_cc += mtd['_tmp_group_cc'] # DeleteWorkspace('_tmp_group_cc') previous_calibration = ConvertDiffCal('_accum_cc', PreviousCalibration=previous_calibration, OutputWorkspace=f'{output_basename}_cc_diffcal') DeleteWorkspace('_accum_cc') DeleteWorkspace('_tmp_group_cc') DeleteWorkspace('_tmp_group_cc_raw') if cycling and '_tmp_group_cc_diffcal' in mtd: DeleteWorkspace('_tmp_group_cc_diffcal') return mtd[f'{output_basename}_cc_diffcal'], to_skip
def _log_xunit_change(self, xunit): logger.notice( "Subsequent files will be loaded with the x-axis unit:\t{}".format( xunit))