class FittingDataModel(object): def __init__(self): self._log_workspaces = None self._log_values = dict() # {ws_name: {log_name: [avg, er]} } self._loaded_workspaces = {} # Map stores using {WorkspaceName: Workspace} self._background_workspaces = {} self._last_added = [] # List of workspace names loaded in the last load action. self._bg_params = dict() # {ws_name: [isSub, niter, xwindow, doSG]} def load_files(self, filenames_string, xunit): self._last_added = [] filenames = [name.strip() for name in filenames_string.split(",")] for filename in filenames: ws_name = self._generate_workspace_name(filename, xunit) if ws_name not in self._loaded_workspaces: try: if not ADS.doesExist(ws_name): ws = Load(filename, OutputWorkspace=ws_name) if xunit != "TOF": ConvertUnits(InputWorkspace=ws, OutputWorkspace=ws_name, Target=xunit) else: ws = ADS.retrieve(ws_name) if ws.getNumberHistograms() == 1: self._loaded_workspaces[ws_name] = ws if ws_name not in self._background_workspaces: self._background_workspaces[ws_name] = None self._last_added.append(ws_name) self.add_log_to_table(ws_name, ws) else: logger.warning( f"Invalid number of spectra in workspace {ws_name}. Skipping loading of file.") except RuntimeError as e: logger.error( f"Failed to load file: {filename}. Error: {e}. \n Continuing loading of other files.") else: logger.warning(f"File {ws_name} has already been loaded") def create_log_table(self): # run information table run_info = CreateEmptyTableWorkspace() run_info.addColumn(type="str", name="Instrument") run_info.addColumn(type="int", name="Run") run_info.addColumn(type="int", name="Bank") run_info.addColumn(type="float", name="uAmps") run_info.addColumn(type="str", name="Title") self._log_workspaces = GroupWorkspaces([run_info], OutputWorkspace='logs') # a table per logs logs = get_setting(path_handling.INTERFACES_SETTINGS_GROUP, path_handling.ENGINEERING_PREFIX, "logs") if logs: for log in logs.split(','): ws = CreateEmptyTableWorkspace(OutputWorkspace=log) ws.addColumn(type="float", name="avg") ws.addColumn(type="float", name="stdev") self._log_workspaces.add(log) def add_log_to_table(self, ws_name, ws): # both ws and name needed in event a ws is renamed and ws.name() is no longer correct if not self._log_workspaces: self.create_log_table() # make dict for run if doesn't exist if ws_name not in self._log_values: self._log_values[ws_name] = dict() # add run info run = ws.getRun() self._log_workspaces[0].addRow( [ws.getInstrument().getFullName(), ws.getRunNumber(), run.getProperty('bankid').value, run.getProtonCharge(), ws.getTitle()]) # add log data - loop over existing log workspaces not logs in settings as these might have changed for ilog in range(1, len(self._log_workspaces)): log_name = self._log_workspaces[ilog].name() if log_name in self._log_values[ws_name]: avg, stdev = self._log_values[ws_name][log_name] else: try: avg, stdev = AverageLogData(ws_name, LogName=log_name, FixZero=False) self._log_values[ws_name][log_name] = [avg, stdev] except RuntimeError: avg, stdev = full(2, nan) logger.error( f"File {ws.name()} does not contain log {self._log_workspaces[ilog].name()}") self._log_workspaces[ilog].addRow([avg, stdev]) self.update_log_group_name() def remove_log_rows(self, row_numbers): DeleteTableRows(TableWorkspace=self._log_workspaces, Rows=list(row_numbers)) self.update_log_group_name() def clear_logs(self): if self._log_workspaces: ws_name = self._log_workspaces.name() self._log_workspaces = None DeleteWorkspace(ws_name) def repopulate_logs(self): self.clear_logs() for ws_name, ws in self._loaded_workspaces.items(): self.add_log_to_table(ws_name, ws) def update_log_group_name(self): run_info = ADS.retrieve('run_info') if run_info.rowCount() > 0: runs = run_info.column('Run') name = f"{run_info.row(0)['Instrument']}_{min(runs)}-{max(runs)}_logs" if not name == self._log_workspaces.name(): RenameWorkspace(InputWorkspace=self._log_workspaces.name(), OutputWorkspace=name) else: self.clear_logs() def update_workspace_name(self, old_name, new_name): if new_name not in self._loaded_workspaces: self._loaded_workspaces[new_name] = self._loaded_workspaces.pop(old_name) if old_name in self._background_workspaces: self._background_workspaces[new_name] = self._background_workspaces.pop(old_name) if old_name in self._bg_params: self._bg_params[new_name] = self._bg_params.pop(old_name) if old_name in self._log_values: self._log_values[new_name] = self._log_values.pop(old_name) else: logger.warning(f"There already exists a workspace with name {new_name}.") def get_loaded_workspaces(self): return self._loaded_workspaces def get_log_workspaces_name(self): return [ws.name() for ws in self._log_workspaces] if self._log_workspaces else '' def get_background_workspaces(self): return self._background_workspaces def get_bg_params(self): return self._bg_params def do_background_subtraction(self, ws_name, bg_params): ws = self._loaded_workspaces[ws_name] ws_bg = self._background_workspaces[ws_name] bg_changed = False if ws_bg and bg_params[1:] != self._bg_params[ws_name][1:]: # add bg back on to data (but don't change bgsub status) self.undo_background_subtraction(ws_name, isBGsub=bg_params[0]) bg_changed = True if bg_changed or not ws_bg: # re-evaluate background (or evaluate for first time) self._bg_params[ws_name] = bg_params ws_bg = self.estimate_background(ws_name, *bg_params[1:]) # update bg sub status before Minus (updates plot which repopulates table) self._bg_params[ws_name][0] = bg_params[0] Minus(LHSWorkspace=ws, RHSWorkspace=ws_bg, OutputWorkspace=ws_name) def undo_background_subtraction(self, ws_name, isBGsub=False): self._bg_params[ws_name][0] = isBGsub # must do this before plotting which refreshes table Plus(LHSWorkspace=ws_name, RHSWorkspace=self.get_background_workspaces()[ws_name], OutputWorkspace=ws_name) def estimate_background(self, ws_name, niter, xwindow, doSGfilter): ws_bg = EnggEstimateFocussedBackground(InputWorkspace=ws_name, OutputWorkspace=ws_name + "_bg", NIterations=niter, XWindow=xwindow, ApplyFilterSG=doSGfilter) self._background_workspaces[ws_name] = ws_bg return ws_bg def plot_background_figure(self, ws_name): ws = self._loaded_workspaces[ws_name] ws_bg = self._background_workspaces[ws_name] if ws_bg: fig, ax = subplots(2, 1, sharex=True, gridspec_kw={'height_ratios': [2, 1]}, subplot_kw={'projection': 'mantid'}) tmp = Plus(LHSWorkspace=ws_name, RHSWorkspace=ws_bg, StoreInADS=False) ax[0].plot(tmp, 'x') ax[0].plot(ws_bg, '-r') ax[1].plot(ws, 'x') fig.show() def get_last_added(self): return self._last_added def get_sample_log_from_ws(self, ws_name, log_name): return self._loaded_workspaces[ws_name].getSampleDetails().getLogData(log_name).value @staticmethod def _generate_workspace_name(filepath, xunit): return path.splitext(path.split(filepath)[1])[0] + '_' + xunit
class FittingDataModel(object): def __init__(self): self._log_names = [] self._log_workspaces = None self._log_values = dict() # {ws_name: {log_name: [avg, er]} } self._loaded_workspaces = {} # Map stores using {WorkspaceName: Workspace} self._bg_sub_workspaces = {} # Map as above, this contains the workspaces with the background sub applied self._fit_results = {} # {WorkspaceName: fit_result_dict} self._fit_workspaces = None self._last_added = [] # List of workspace names loaded in the last load action. self._bg_params = dict() # {ws_name: [isSub, niter, xwindow, doSG]} def restore_files(self, ws_names): for ws_name in ws_names: try: ws = ADS.retrieve(ws_name) if ws.getNumberHistograms() == 1: self._loaded_workspaces[ws_name] = ws if self._bg_params[ws_name]: self._bg_sub_workspaces[ws_name] = ADS.retrieve(ws_name + "_bgsub") else: self._bg_sub_workspaces[ws_name] = None if ws_name not in self._bg_params: self._bg_params[ws_name] = [] self._last_added.append(ws_name) else: logger.warning( f"Invalid number of spectra in workspace {ws_name}. Skipping restoration of workspace.") except RuntimeError as e: logger.error( f"Failed to restore workspace: {ws_name}. Error: {e}. \n Continuing loading of other files.") self.update_log_workspace_group() def load_files(self, filenames_string): self._last_added = [] filenames = [name.strip() for name in filenames_string.split(",")] for filename in filenames: ws_name = self._generate_workspace_name(filename) if ws_name not in self._loaded_workspaces: try: if not ADS.doesExist(ws_name): ws = Load(filename, OutputWorkspace=ws_name) else: ws = ADS.retrieve(ws_name) if ws.getNumberHistograms() == 1: self._loaded_workspaces[ws_name] = ws if ws_name not in self._bg_sub_workspaces: self._bg_sub_workspaces[ws_name] = None if ws_name not in self._bg_params: self._bg_params[ws_name] = [] self._last_added.append(ws_name) else: logger.warning( f"Invalid number of spectra in workspace {ws_name}. Skipping loading of file.") except RuntimeError as e: logger.error( f"Failed to load file: {filename}. Error: {e}. \n Continuing loading of other files.") else: logger.warning(f"File {ws_name} has already been loaded") self.update_log_workspace_group() def create_log_workspace_group(self): # run information table run_info = self.make_runinfo_table() self._log_workspaces = GroupWorkspaces([run_info], OutputWorkspace='logs') # a table per logs logs = get_setting(output_settings.INTERFACES_SETTINGS_GROUP, output_settings.ENGINEERING_PREFIX, "logs") if logs: self._log_names = logs.split(',') for log in self._log_names: self.make_log_table(log) self._log_workspaces.add(log) def make_log_table(self, log): ws_log = CreateEmptyTableWorkspace(OutputWorkspace=log) ws_log.addColumn(type="float", name="avg") ws_log.addColumn(type="float", name="stdev") return ws_log def make_runinfo_table(self): run_info = CreateEmptyTableWorkspace() run_info.addColumn(type="str", name="Instrument") run_info.addColumn(type="int", name="Run") run_info.addColumn(type="int", name="Bank") run_info.addColumn(type="float", name="uAmps") run_info.addColumn(type="str", name="Title") return run_info def update_log_workspace_group(self): # both ws and name needed in event a ws is renamed and ws.name() is no longer correct if not self._log_workspaces: self.create_log_workspace_group() else: for log in self._log_names: if not ADS.doesExist(log): self.make_log_table(log) self._log_workspaces.add(log) if not ADS.doesExist("run_info"): self.make_runinfo_table() self._log_workspaces.add("run_info") # update log tables for irow, (ws_name, ws) in enumerate(self._loaded_workspaces.items()): self.add_log_to_table(ws_name, ws, irow) # rename write_log_row def add_log_to_table(self, ws_name, ws, irow): # both ws and name needed in event a ws is renamed and ws.name() is no longer correct # make dict for run if doesn't exist if ws_name not in self._log_values: self._log_values[ws_name] = dict() # add run info run = ws.getRun() row = [ws.getInstrument().getFullName(), ws.getRunNumber(), run.getProperty('bankid').value, run.getProtonCharge(), ws.getTitle()] self.write_table_row(ADS.retrieve("run_info"), row, irow) # add log data - loop over existing log workspaces not logs in settings as these might have changed currentRunLogs = [l.name for l in run.getLogData()] nullLogValue = full(2, nan) # default nan if can't read/average log data if run.getProtonCharge() > 0 and "proton_charge" in currentRunLogs: for log in self._log_names: if log in self._log_values[ws_name]: avg, stdev = self._log_values[ws_name][log] # already averaged elif log in currentRunLogs: avg, stdev = AverageLogData(ws_name, LogName=log, FixZero=False) else: avg, stdev = nullLogValue self._log_values[ws_name][log] = [avg, stdev] # update model dict (even if nan) else: self._log_values[ws_name] = {log: nullLogValue for log in self._log_names} logger.warning(f"{ws.name()} does not contain a proton charge log - log values cannot be averaged.") # write log values to table (nan if log could not be averaged) for log, avg_and_stdev in self._log_values[ws_name].items(): self.write_table_row(ADS.retrieve(log), avg_and_stdev, irow) self.update_log_group_name() def remove_log_rows(self, row_numbers): DeleteTableRows(TableWorkspace=self._log_workspaces, Rows=list(row_numbers)) self.update_log_group_name() def clear_logs(self): if self._log_workspaces: ws_name = self._log_workspaces.name() self._log_workspaces = None DeleteWorkspace(ws_name) def update_log_group_name(self): run_info = ADS.retrieve('run_info') if run_info.rowCount() > 0: runs = run_info.column('Run') name = f"{run_info.row(0)['Instrument']}_{min(runs)}-{max(runs)}_logs" if not name == self._log_workspaces.name(): RenameWorkspace(InputWorkspace=self._log_workspaces.name(), OutputWorkspace=name) else: self.clear_logs() def get_ws_list(self): return list(self._loaded_workspaces.keys()) def get_ws_sorted_by_primary_log(self): ws_list = self.get_ws_list() tof_ws_inds = [ind for ind, ws in enumerate(ws_list) if self._loaded_workspaces[ws].getAxis(0).getUnit().caption() == 'Time-of-flight'] primary_log = get_setting(output_settings.INTERFACES_SETTINGS_GROUP, output_settings.ENGINEERING_PREFIX, "primary_log") sort_ascending = get_setting(output_settings.INTERFACES_SETTINGS_GROUP, output_settings.ENGINEERING_PREFIX, "sort_ascending") if primary_log: log_table = ADS.retrieve(primary_log) isort = argsort(array(log_table.column('avg'))) ws_list_tof = [ws_list[iws] for iws in isort if iws in tof_ws_inds] else: ws_list_tof = ws_list if sort_ascending == 'false': # settings can only be saved as text ws_list_tof = ws_list_tof[::-1] return ws_list_tof def update_fit(self, fit_props): for fit_prop in fit_props: wsname = fit_prop['properties']['InputWorkspace'] self._fit_results[wsname] = {'model': fit_prop['properties']['Function'], 'status': fit_prop['status']} self._fit_results[wsname]['results'] = defaultdict(list) # {function_param: [[Y1, E1], [Y2,E2],...] } fnames = [x.split('=')[-1] for x in findall('name=[^,]*', fit_prop['properties']['Function'])] # get num params for each function (first elem empty as str begins with 'name=') # need to remove ties and constraints which are enclosed in () nparams = [s.count('=') for s in sub(r'=\([^)]*\)', '', fit_prop['properties']['Function']).split('name=')[1:]] params_dict = ADS.retrieve(fit_prop['properties']['Output'] + '_Parameters').toDict() # loop over rows in output workspace to get value and error for each parameter istart = 0 for ifunc, fname in enumerate(fnames): for iparam in range(0, nparams[ifunc]): irow = istart + iparam key = '_'.join([fname, params_dict['Name'][irow].split('.')[-1]]) # funcname_param self._fit_results[wsname]['results'][key].append([ params_dict['Value'][irow], params_dict['Error'][irow]]) if key in fit_prop['peak_centre_params']: # param corresponds to a peak centre in TOF which we also need in dspacing # add another entry into the results dictionary key_d = key + "_dSpacing" dcen = self._convert_TOF_to_d(params_dict['Value'][irow], wsname) dcen_er = self._convert_TOFerror_to_derror(params_dict['Error'][irow], dcen, wsname) self._fit_results[wsname]['results'][key_d].append([dcen, dcen_er]) istart += nparams[ifunc] # append the cost function value (in this case always chisq/DOF) as don't let user change cost func # always last row in parameters table self._fit_results[wsname]['costFunction'] = params_dict['Value'][-1] self.create_fit_tables() def create_fit_tables(self): wslist = [] # ws to be grouped # extract fit parameters and errors nruns = len(self._loaded_workspaces.keys()) # num of rows of output workspace # get unique set of function parameters across all workspaces func_params = set(chain(*[list(d['results'].keys()) for d in self._fit_results.values()])) for param in func_params: # get max number of repeated func in a model (num columns of output workspace) nfuncs = max([len(d['results'][param]) for d in self._fit_results.values() if param in d['results']]) # make output workspace ipeak = list(range(1, nfuncs + 1)) * nruns ws = CreateWorkspace(OutputWorkspace=param, DataX=ipeak, DataY=ipeak, NSpec=nruns) # axis for labels in workspace axis = TextAxis.create(nruns) for iws, wsname in enumerate(self._loaded_workspaces.keys()): wsname_bgsub = wsname + "_bgsub" if wsname_bgsub in self._fit_results: wsname = wsname_bgsub if wsname in self._fit_results and param in self._fit_results[wsname]['results']: fitvals = array(self._fit_results[wsname]['results'][param]) data = vstack((fitvals, full((nfuncs - fitvals.shape[0], 2), nan))) else: data = full((nfuncs, 2), nan) ws.setY(iws, data[:, 0]) ws.setE(iws, data[:, 1]) # label row axis.setLabel(iws, wsname) ws.replaceAxis(1, axis) wslist += [ws] # table for model summary/info model = CreateEmptyTableWorkspace(OutputWorkspace='model') model.addColumn(type="str", name="Workspace") model.addColumn(type="float", name="chisq/DOF") # always is for LM minimiser (users can't change) model.addColumn(type="str", name="status") model.addColumn(type="str", name="Model") for iws, wsname in enumerate(self._loaded_workspaces.keys()): if wsname in self._fit_results: row = [wsname, self._fit_results[wsname]['costFunction'], self._fit_results[wsname]['status'], self._fit_results[wsname]['model']] self.write_table_row(model, row, iws) else: self.write_table_row(model, ['', nan, ''], iws) wslist += [model] group_name = self._log_workspaces.name().split('_log')[0] + '_fits' self._fit_workspaces = GroupWorkspaces(wslist, OutputWorkspace=group_name) def update_workspace_name(self, old_name, new_name): if new_name not in self._loaded_workspaces: self._loaded_workspaces[new_name] = self._loaded_workspaces.pop(old_name) if old_name in self._bg_sub_workspaces: self._bg_sub_workspaces[new_name] = self._bg_sub_workspaces.pop(old_name) if old_name in self._bg_params: self._bg_params[new_name] = self._bg_params.pop(old_name) if old_name in self._log_values: self._log_values[new_name] = self._log_values.pop(old_name) else: logger.warning(f"There already exists a workspace with name {new_name}.") def get_loaded_workspaces(self): return self._loaded_workspaces def get_log_workspaces_name(self): return [ws.name() for ws in self._log_workspaces] if self._log_workspaces else '' def get_bgsub_workspaces(self): return self._bg_sub_workspaces def get_bg_params(self): return self._bg_params def get_fit_results(self): return self._fit_results def create_or_update_bgsub_ws(self, ws_name, bg_params): ws = self._loaded_workspaces[ws_name] ws_bg = self._bg_sub_workspaces[ws_name] if not ws_bg or self._bg_params[ws_name] == [] or bg_params[1:] != self._bg_params[ws_name][1:]: background = self.estimate_background(ws_name, *bg_params[1:]) self._bg_params[ws_name] = bg_params bgsub_ws_name = ws_name + "_bgsub" bgsub_ws = Minus(LHSWorkspace=ws, RHSWorkspace=background, OutputWorkspace=bgsub_ws_name) self._bg_sub_workspaces[ws_name] = bgsub_ws DeleteWorkspace(background) else: logger.notice("Background workspace already calculated") def update_bgsub_status(self, ws_name, status): if self._bg_params[ws_name]: self._bg_params[ws_name][0] = status def estimate_background(self, ws_name, niter, xwindow, doSGfilter): try: ws_bg = EnggEstimateFocussedBackground(InputWorkspace=ws_name, OutputWorkspace=ws_name + "_bg", NIterations=niter, XWindow=xwindow, ApplyFilterSG=doSGfilter) except (ValueError, RuntimeError) as e: # ValueError when Niter not positive integer, RuntimeError when Window too small logger.error("Error on arguments supplied to EnggEstimateFocusedBackground: " + str(e)) ws_bg = SetUncertainties(InputWorkspace=ws_name) # copy data and zero errors ws_bg = Minus(LHSWorkspace=ws_bg, RHSWorkspace=ws_bg) # workspace of zeros with same num spectra return ws_bg def plot_background_figure(self, ws_name): ws = self._loaded_workspaces[ws_name] ws_bgsub = self._bg_sub_workspaces[ws_name] if ws_bgsub: fig, ax = subplots(2, 1, sharex=True, gridspec_kw={'height_ratios': [2, 1]}, subplot_kw={'projection': 'mantid'}) bg = Minus(LHSWorkspace=ws_name, RHSWorkspace=ws_bgsub, StoreInADS=False) ax[0].plot(ws, 'x') ax[1].plot(ws_bgsub, 'x') ax[0].plot(bg, '-r') fig.show() def get_last_added(self): return self._last_added def get_sample_log_from_ws(self, ws_name, log_name): return self._loaded_workspaces[ws_name].getSampleDetails().getLogData(log_name).value def set_log_workspaces_none(self): # to be used in the event of Ads clear, as trying to reference the deleted grp ws results in an error self._log_workspaces = None def _convert_TOF_to_d(self, tof, ws_name): diff_consts = self._get_diff_constants(ws_name) return UnitConversion.run("TOF", "dSpacing", tof, 0, DeltaEModeType.Elastic, diff_consts) # L1=0 (ignored) def _convert_TOFerror_to_derror(self, tof_error, d, ws_name): diff_consts = self._get_diff_constants(ws_name) difc = diff_consts[UnitParams.difc] difa = diff_consts[UnitParams.difa] if UnitParams.difa in diff_consts else 0 return tof_error / (2 * difa * d + difc) def _get_diff_constants(self, ws_name): """ Get diffractometer constants from workspace TOF = difc*d + difa*(d^2) + tzero """ ws = ADS.retrieve(ws_name) si = ws.spectrumInfo() diff_consts = si.diffractometerConstants(0) # output is a UnitParametersMap return diff_consts @staticmethod def write_table_row(ws_table, row, irow): if irow > ws_table.rowCount() - 1: ws_table.setRowCount(irow + 1) [ws_table.setCell(irow, icol, row[icol]) for icol in range(0, len(row))] @staticmethod def _generate_workspace_name(filepath): wsname = path.splitext(path.split(filepath)[1])[0] return wsname