def merge_d1s(d1s, detectors, reft=-1, save_merged=False, debug=False): """ utility function to merge 1D data sets, using functions under slnxs d1s should contain data corresponding to detectors """ s0 = Data1d() s0.qgrid = d1s[0].qgrid d_tot = np.zeros(s0.qgrid.shape) d_max = np.zeros(s0.qgrid.shape) d_min = np.zeros(s0.qgrid.shape) + 1.e32 e_tot = np.zeros(s0.qgrid.shape) c_tot = np.zeros(s0.qgrid.shape) label = None comments = "" for d1 in d1s: # empty part of the data is nan idx = ~np.isnan(d1.data) d_tot[idx] += d1.data[idx] e_tot[idx] += d1.err[idx] c_tot[idx] += 1 idx1 = (np.ma.fix_invalid(d1.data, fill_value=-1) > d_max).data d_max[idx1] = d1.data[idx1] idx2 = (np.ma.fix_invalid(d1.data, fill_value=1e32) < d_min).data d_min[idx2] = d1.data[idx2] comments += d1.comments if label is None: label = d1.label else: label = common_name(label, d1.label) s0.data = d_tot s0.err = e_tot idx = (c_tot > 1) s0.overlaps.append({ 'q_overlap': s0.qgrid[idx], 'raw_data1': d_max[idx], 'raw_data2': d_min[idx] }) s0.data[idx] /= c_tot[idx] s0.err[idx] /= np.sqrt(c_tot[idx]) s0.set_trans(ref_trans=reft, debug=debug) s0.label = label s0.comments = comments # .replace("# ", "## ") if save_merged: s0.save(s0.label + ".dd", debug=debug) return s0
def merge_detectors(fns, detectors, qgrid, reft=-1, plot_data=False, save_ave=False, save_merged=False, ax=None, qmax=-1, qmin=-1, fix_scale=1, debug=False): """ fns: filename, without the _SAXS/_WAXS suffix fix_scale is now default to 1 implicitly assume that all detectors have the same qgrid """ ss = [] t0 = time.time() for fn in fns: s0 = Data1d() d_tot = np.zeros(detectors[0].qgrid.shape) d_max = np.zeros(detectors[0].qgrid.shape) d_min = np.zeros(detectors[0].qgrid.shape) + 1.e32 e_tot = np.zeros(detectors[0].qgrid.shape) c_tot = np.zeros(detectors[0].qgrid.shape) label = None comments = "" #t1 = time.time() for det in detectors: #t2 = time.time() # revised 2017mar10 # old conversion: fn+det.extension gives the complete filename, the extension looks like this: "_SAXS.cbf" # this is a problem when the detector collect multiple images per trigger # new comvention: fn is a template, e.g. '/GPFS/xf16id/exp_path/301525/301016/temp1_000002%s_00001.cbf', # and the extension looks like this: "_SAXS" if "%s" in fn: fn1 = fn % det.extension else: fn1 = fn + det.extension if debug == True: print(fn, det.extension, fn1) s0.load_from_2D(fn1, det.exp_para, qgrid, det.pre_process, save_ave=save_ave, debug=debug) if save_ave: s0.save(fn1 + ".ave", debug=debug) if det.fix_scale is not None: fix_scale = det.fix_scale s0.scale(1. / fix_scale) # empty part of the data is nan idx = ~np.isnan(s0.data) d_tot[idx] += s0.data[idx] e_tot[idx] += s0.err[idx] c_tot[idx] += 1 idx1 = (np.ma.fix_invalid(s0.data, fill_value=-1) > d_max).data d_max[idx1] = s0.data[idx1] idx2 = (np.ma.fix_invalid(s0.data, fill_value=1e32) < d_min).data d_min[idx2] = s0.data[idx2] comments += s0.comments if label is None: label = s0.label else: label = common_name(label, s0.label) s0.data = d_tot s0.err = e_tot idx = (c_tot > 1) s0.overlaps.append({ 'q_overlap': detectors[0].qgrid[idx], 'raw_data1': d_max[idx], 'raw_data2': d_min[idx] }) s0.data[idx] /= c_tot[idx] s0.err[idx] /= c_tot[idx] s0.set_trans(ref_trans=reft, debug=debug) s0.label = label s0.comments = comments # .replace("# ", "## ") if save_merged: s0.save(s0.label + ".dd", debug=debug) ss.append(s0) return ss
def avg(self, dsets, plot_data=False, ax=None, debug=False): """ dset is a collection of Data1d ax is the Axes to plot the data in TODO: should calculate something like the cross-correlation between sets to evaluate the consistency between them """ if debug != 'quiet': print("averaging data with %s: \n" % self.label, end=' ') n = 1 if plot_data: if ax is None: plt.figure() plt.subplots_adjust(bottom=0.15) ax = plt.gca() ax.set_xlabel("$q (\AA^{-1})$", fontsize='x-large') ax.set_ylabel("$I$", fontsize='x-large') ax.set_xscale('log') ax.set_yscale('log') idx = (self.data > 0) ax.errorbar(self.qgrid[idx], self.data[idx], self.err[idx], label=self.label) for ov in self.overlaps: ax.plot(ov['q_overlap'], ov['raw_data1'], "v") ax.plot(ov['q_overlap'], ov['raw_data2'], "^") d0 = copy.deepcopy(self) if len(dsets) == 0: return d0 for d1 in dsets: if debug == True: print("%s " % d1.label, end=' ') if not (d0.qgrid == d1.qgrid).all(): raise Exception("\n1D sets cannot be averaged: qgrid mismatch") d0.trans += d1.trans d0.data += d1.data d0.err += d1.err if trans_mode == TRANS_FROM_BEAM_CENTER: d0.roi += d1.roi d0.comments += "# averaged with \n%s" % d1.comments.replace( "# ", "## ") if plot_data: idx = (d1.data > 0) # Remove Zeros on plot ax.errorbar(d1.qgrid[idx], d1.data[idx] * VOFFSET**n, d1.err[idx] * VOFFSET**n, label=d1.label) for i in range(len(d1.overlaps)): if plot_data: ax.plot(d1.overlaps[i]['q_overlap'], d1.overlaps[i]['raw_data1'] * VOFFSET**n, "v") ax.plot(d1.overlaps[i]['q_overlap'], d1.overlaps[i]['raw_data2'] * VOFFSET**n, "^") d0.overlaps[i]['raw_data1'] += d1.overlaps[i]['raw_data1'] d0.overlaps[i]['raw_data2'] += d1.overlaps[i]['raw_data2'] n += 1 d0.label = common_name(d0.label, d1.label) d0.trans /= n d0.data /= n d0.err /= np.sqrt(n) if trans_mode == TRANS_FROM_BEAM_CENTER: d0.roi /= n for ov in d0.overlaps: ov['raw_data1'] /= n ov['raw_data2'] /= n if debug == True: print("\naveraged set re-named to %s." % d0.label) if plot_data: # plot the averaged data over each individual curve for i in range(n): if i == 0: idx = (d0.data > 0) # Remove Zeros on plot handles, labels = ax.get_legend_handles_labels() lbl = "averaged" if "averaged" not in labels else "" ax.plot(d0.qgrid[idx], d0.data[idx] * VOFFSET**i, color="gray", lw=2, ls="--", label=lbl) else: idx = (d0.data > 0) # Remove Zeros on plot ax.plot(d0.qgrid[idx], d0.data[idx] * VOFFSET**i, color="gray", lw=2, ls="--") leg = ax.legend(loc='upper right', frameon=False) for t in leg.get_texts(): t.set_fontsize('small') return d0
def merge(self, d1, qmax=-1, qmin=-1, fix_scale=-1, debug=False): """ combine the data in self and d1 scale d1 intensity to match self self and d1 should have the same qgrid if qmax or qmin <0 simply keep the WAXS data that is beyond qmax for the SAXS data this is useful for utilizing WAXS to normalize intensity but keep SAXS data only """ if debug == True: print("merging data: %s and %s ..." % (self.label, d1.label)) if not (d1.qgrid == self.qgrid).all(): print("merging data sets should have the same qgrid.") exit() # this gives the overlapping region idx = (self.data > 0) & (d1.data > 0) if len(self.qgrid[idx]) > 0: qmin0 = min(d1.qgrid[idx]) qmax0 = max(self.qgrid[idx]) # merge SAXS/WAXS based on intensity in the overlapping region if qmax0 < qmax: qmax = qmax0 if qmin0 > qmin: qmin = qmin0 idx = (self.qgrid > qmin) & (self.qgrid < qmax) # save the raw data in case needed, e.g. for ploting self.overlaps.append({ 'q_overlap': self.qgrid[idx], 'raw_data1': self.data[idx], 'raw_data2': d1.data[idx] }) else: # no overlap # simply stack WAXS data to the high q end of SAXS data qmin = qmax = max(self.qgrid[self.data > 0]) self.overlaps.append({ 'q_overlap': np.empty(0), 'raw_data1': np.empty(0), 'raw_data2': np.empty(0) }) # idx = np.asarray([],dtype=int) if len(self.qgrid[idx]) == 0: if debug != 'quiet': print("data sets are not overlapping in the given q range.") if fix_scale < 0: fix_scale = 1 if debug != 'quiet': print("forcing fix_scale=1.") elif len(self.qgrid[idx]) < 5 and debug != 'quiet': print("too few overlapping points: %d" % len(self.qgrid[idx])) if fix_scale > 0: # For a given experimental configuration, the intensity normlization # factor between the SAXS and WAXS should be well-defined. This factor # can be determined using scattering data with siginificant intensity # in the overlapping q-range and applied to all data collected in the # same configuration. sc = fix_scale else: sc = np.linalg.lstsq( np.asmatrix(self.data[idx]).T, np.asmatrix(d1.data[idx]).T)[0] sc = np.trace(sc) d1.data /= sc d1.err /= sc if len(self.qgrid[idx]) > 0: if debug == True: print("Scaled Overlaps by 1/%f" % sc) self.overlaps[-1]['raw_data2'] /= sc self.overlaps[-1]['sc'] = sc self.label = common_name(self.label, d1.label) if debug == True: print("set2 scaled by 1/%f" % sc) print("merged set re-named %s." % self.label) if len(self.qgrid[idx]) > 0: self.data[idx] = (self.data[idx] + d1.data[idx]) / 2 # this won't work well if the merging data are mis-matched before bkg subtraction # but match well after bkg subtraction self.err[idx] = (self.err[idx] + d1.err[idx]) / 2 self.data[self.qgrid >= qmax] = d1.data[self.qgrid >= qmax] self.err[self.qgrid >= qmax] = d1.err[self.qgrid >= qmax] self.comments += "# merged with the following set by matching intensity within (%.4f, %.4f)," % ( qmin, qmax) self.comments += " scaled by %f\n" % sc self.comments += d1.comments.replace("# ", "## ")
def avg(self, dsets, weighted=False, qmax_for_weight=0.3, plot_data=False, ax=None, debug=False, fontsize='large'): """ dsets is a collection of Data1d weighted: if False the dsets are simply averaged together errorbar is increased if there are discrepencies between individual values?? otherwise weight should contain a list of weight factors, corresponding to each dset each dset is first scaled by the a weighted average (smaller errorbar has higher weight) is then performed ax is the Axes to plot the data in TODO: should calculate something like the cross-correlation between sets to evaluate the consistency between them """ if debug != 'quiet': print("averaging data with %s: \n" % self.label, end=' ') i_fs = get_font_size(fontsize)[0] n = 1 if plot_data: if ax is None: plt.figure() plt.subplots_adjust(bottom=0.15) ax = plt.gca() ax.set_xlabel("$q (\AA^{-1})$", fontsize=get_font_size(i_fs)[1]) ax.set_ylabel("$I$", fontsize=get_font_size(i_fs)[1]) ax.set_xscale('log') ax.set_yscale('log') idx = (self.data > 0) ax.errorbar(self.qgrid[idx], self.data[idx], self.err[idx], label=self.label) for ov in self.overlaps: ax.plot(ov['q_overlap'], ov['raw_data1'], "v") ax.plot(ov['q_overlap'], ov['raw_data2'], "^") d0 = copy.deepcopy(self) if len(dsets) == 0: return d0 if weighted: wt = [] for d in dsets + [self]: w0 = np.sum(np.fabs(d.data[d.qgrid < qmax_for_weight])) if w0 <= 0: raise Exception(f"weight for averaging <0: {w0}") wt.append(w0) wt = np.asarray(wt) wt /= wt.max() er2 = (self.err / wt[-1])**2 d0.err = 1. / er2 d0.data = d0.data / er2 if debug == True: print("weight factors: ", wt) for i in range(len(dsets)): d1 = dsets[i] if debug == True: print("%s " % d1.label, end=' ') if not (d0.qgrid == d1.qgrid).all(): raise Exception("\n1D sets cannot be averaged: qgrid mismatch") d0.trans += d1.trans d0.trans_w += d1.trans_w d0.trans_e += d1.trans_e if weighted: er2 = (d1.err / wt[i])**2 d0.err += 1 / er2 d0.data += d1.data / er2 else: d0.data += d1.data d0.err += d1.err #if self.transMode == trans_mode.from_beam_center: # d0.roi += d1.roi d0.comments += "# averaged with \n%s" % d1.comments.replace( "# ", "## ") if plot_data: idx = (d1.data > 0) # Remove Zeros on plot ax.errorbar(d1.qgrid[idx], d1.data[idx] * VOFFSET**n, d1.err[idx] * VOFFSET**n, label=d1.label) for i in range(len(d1.overlaps)): if plot_data: ax.plot(d1.overlaps[i]['q_overlap'], d1.overlaps[i]['raw_data1'] * VOFFSET**n, "v") ax.plot(d1.overlaps[i]['q_overlap'], d1.overlaps[i]['raw_data2'] * VOFFSET**n, "^") d0.overlaps[i]['raw_data1'] += d1.overlaps[i]['raw_data1'] d0.overlaps[i]['raw_data2'] += d1.overlaps[i]['raw_data2'] n += 1 d0.label = common_name(d0.label, d1.label) d0.trans /= n d0.trans_w /= n d0.trans_e /= n if weighted: d0.data /= d0.err d0.err = 1. / np.sqrt(d0.err) else: d0.data /= n d0.err /= ( n * np.sqrt(n) ) # should not be just sqrt(n), that would increase err after averaging #if self.transMode == trans_mode.from_beam_center: # d0.roi /= n for ov in d0.overlaps: ov['raw_data1'] /= n ov['raw_data2'] /= n if debug == True: print("\naveraged set re-named to %s." % d0.label) if plot_data: # plot the averaged data over each individual curve for i in range(n): if i == 0: idx = (d0.data > 0) # Remove Zeros on plot handles, labels = ax.get_legend_handles_labels() lbl = "averaged" if "averaged" not in labels else "" ax.plot(d0.qgrid[idx], d0.data[idx] * VOFFSET**i, color="gray", lw=2, ls="--", label=lbl) else: idx = (d0.data > 0) # Remove Zeros on plot ax.plot(d0.qgrid[idx], d0.data[idx] * VOFFSET**i, color="gray", lw=2, ls="--") leg = ax.legend(loc='upper right', frameon=False) for t in leg.get_texts(): t.set_fontsize(get_font_size(i_fs - 2)[1]) return d0