def merge_d1s(d1s, detectors, reft=-1, save_merged=False, debug=False):
""" utility function to merge 1D data sets, using functions under slnxs
d1s should contain data corresponding to detectors
"""
s0 = Data1d()
s0.qgrid = d1s[0].qgrid
d_tot = np.zeros(s0.qgrid.shape)
d_max = np.zeros(s0.qgrid.shape)
d_min = np.zeros(s0.qgrid.shape) + 1.e32
e_tot = np.zeros(s0.qgrid.shape)
c_tot = np.zeros(s0.qgrid.shape)
label = None
comments = ""
for d1 in d1s:
# empty part of the data is nan
idx = ~np.isnan(d1.data)
d_tot[idx] += d1.data[idx]
e_tot[idx] += d1.err[idx]
c_tot[idx] += 1
idx1 = (np.ma.fix_invalid(d1.data, fill_value=-1) > d_max).data
d_max[idx1] = d1.data[idx1]
idx2 = (np.ma.fix_invalid(d1.data, fill_value=1e32) < d_min).data
d_min[idx2] = d1.data[idx2]
comments += d1.comments
if label is None:
label = d1.label
else:
label = common_name(label, d1.label)
s0.data = d_tot
s0.err = e_tot
idx = (c_tot > 1)
s0.overlaps.append({
'q_overlap': s0.qgrid[idx],
'raw_data1': d_max[idx],
'raw_data2': d_min[idx]
})
s0.data[idx] /= c_tot[idx]
s0.err[idx] /= np.sqrt(c_tot[idx])
s0.set_trans(ref_trans=reft, debug=debug)
s0.label = label
s0.comments = comments # .replace("# ", "## ")
if save_merged:
s0.save(s0.label + ".dd", debug=debug)
return s0
def merge_detectors(fns,
detectors,
qgrid,
reft=-1,
plot_data=False,
save_ave=False,
save_merged=False,
ax=None,
qmax=-1,
qmin=-1,
fix_scale=1,
debug=False):
"""
fns: filename, without the _SAXS/_WAXS suffix
fix_scale is now default to 1
implicitly assume that all detectors have the same qgrid
"""
ss = []
t0 = time.time()
for fn in fns:
s0 = Data1d()
d_tot = np.zeros(detectors[0].qgrid.shape)
d_max = np.zeros(detectors[0].qgrid.shape)
d_min = np.zeros(detectors[0].qgrid.shape) + 1.e32
e_tot = np.zeros(detectors[0].qgrid.shape)
c_tot = np.zeros(detectors[0].qgrid.shape)
label = None
comments = ""
#t1 = time.time()
for det in detectors:
#t2 = time.time()
# revised 2017mar10
# old conversion: fn+det.extension gives the complete filename, the extension looks like this: "_SAXS.cbf"
# this is a problem when the detector collect multiple images per trigger
# new comvention: fn is a template, e.g. '/GPFS/xf16id/exp_path/301525/301016/temp1_000002%s_00001.cbf',
# and the extension looks like this: "_SAXS"
if "%s" in fn:
fn1 = fn % det.extension
else:
fn1 = fn + det.extension
if debug == True:
print(fn, det.extension, fn1)
s0.load_from_2D(fn1,
det.exp_para,
qgrid,
det.pre_process,
save_ave=save_ave,
debug=debug)
if save_ave:
s0.save(fn1 + ".ave", debug=debug)
if det.fix_scale is not None:
fix_scale = det.fix_scale
s0.scale(1. / fix_scale)
# empty part of the data is nan
idx = ~np.isnan(s0.data)
d_tot[idx] += s0.data[idx]
e_tot[idx] += s0.err[idx]
c_tot[idx] += 1
idx1 = (np.ma.fix_invalid(s0.data, fill_value=-1) > d_max).data
d_max[idx1] = s0.data[idx1]
idx2 = (np.ma.fix_invalid(s0.data, fill_value=1e32) < d_min).data
d_min[idx2] = s0.data[idx2]
comments += s0.comments
if label is None:
label = s0.label
else:
label = common_name(label, s0.label)
s0.data = d_tot
s0.err = e_tot
idx = (c_tot > 1)
s0.overlaps.append({
'q_overlap': detectors[0].qgrid[idx],
'raw_data1': d_max[idx],
'raw_data2': d_min[idx]
})
s0.data[idx] /= c_tot[idx]
s0.err[idx] /= c_tot[idx]
s0.set_trans(ref_trans=reft, debug=debug)
s0.label = label
s0.comments = comments # .replace("# ", "## ")
if save_merged:
s0.save(s0.label + ".dd", debug=debug)
ss.append(s0)
return ss
def avg(self, dsets, plot_data=False, ax=None, debug=False):
"""
dset is a collection of Data1d
ax is the Axes to plot the data in
TODO: should calculate something like the cross-correlation between sets
to evaluate the consistency between them
"""
if debug != 'quiet':
print("averaging data with %s: \n" % self.label, end=' ')
n = 1
if plot_data:
if ax is None:
plt.figure()
plt.subplots_adjust(bottom=0.15)
ax = plt.gca()
ax.set_xlabel("$q (\AA^{-1})$", fontsize='x-large')
ax.set_ylabel("$I$", fontsize='x-large')
ax.set_xscale('log')
ax.set_yscale('log')
idx = (self.data > 0)
ax.errorbar(self.qgrid[idx],
self.data[idx],
self.err[idx],
label=self.label)
for ov in self.overlaps:
ax.plot(ov['q_overlap'], ov['raw_data1'], "v")
ax.plot(ov['q_overlap'], ov['raw_data2'], "^")
d0 = copy.deepcopy(self)
if len(dsets) == 0:
return d0
for d1 in dsets:
if debug == True:
print("%s " % d1.label, end=' ')
if not (d0.qgrid == d1.qgrid).all():
raise Exception("\n1D sets cannot be averaged: qgrid mismatch")
d0.trans += d1.trans
d0.data += d1.data
d0.err += d1.err
if trans_mode == TRANS_FROM_BEAM_CENTER:
d0.roi += d1.roi
d0.comments += "# averaged with \n%s" % d1.comments.replace(
"# ", "## ")
if plot_data:
idx = (d1.data > 0) # Remove Zeros on plot
ax.errorbar(d1.qgrid[idx],
d1.data[idx] * VOFFSET**n,
d1.err[idx] * VOFFSET**n,
label=d1.label)
for i in range(len(d1.overlaps)):
if plot_data:
ax.plot(d1.overlaps[i]['q_overlap'],
d1.overlaps[i]['raw_data1'] * VOFFSET**n, "v")
ax.plot(d1.overlaps[i]['q_overlap'],
d1.overlaps[i]['raw_data2'] * VOFFSET**n, "^")
d0.overlaps[i]['raw_data1'] += d1.overlaps[i]['raw_data1']
d0.overlaps[i]['raw_data2'] += d1.overlaps[i]['raw_data2']
n += 1
d0.label = common_name(d0.label, d1.label)
d0.trans /= n
d0.data /= n
d0.err /= np.sqrt(n)
if trans_mode == TRANS_FROM_BEAM_CENTER:
d0.roi /= n
for ov in d0.overlaps:
ov['raw_data1'] /= n
ov['raw_data2'] /= n
if debug == True:
print("\naveraged set re-named to %s." % d0.label)
if plot_data:
# plot the averaged data over each individual curve
for i in range(n):
if i == 0:
idx = (d0.data > 0) # Remove Zeros on plot
handles, labels = ax.get_legend_handles_labels()
lbl = "averaged" if "averaged" not in labels else ""
ax.plot(d0.qgrid[idx],
d0.data[idx] * VOFFSET**i,
color="gray",
lw=2,
ls="--",
label=lbl)
else:
idx = (d0.data > 0) # Remove Zeros on plot
ax.plot(d0.qgrid[idx],
d0.data[idx] * VOFFSET**i,
color="gray",
lw=2,
ls="--")
leg = ax.legend(loc='upper right', frameon=False)
for t in leg.get_texts():
t.set_fontsize('small')
return d0
def merge(self, d1, qmax=-1, qmin=-1, fix_scale=-1, debug=False):
"""
combine the data in self and d1
scale d1 intensity to match self
self and d1 should have the same qgrid
if qmax or qmin <0
simply keep the WAXS data that is beyond qmax for the SAXS data
this is useful for utilizing WAXS to normalize intensity but keep SAXS data only
"""
if debug == True:
print("merging data: %s and %s ..." % (self.label, d1.label))
if not (d1.qgrid == self.qgrid).all():
print("merging data sets should have the same qgrid.")
exit()
# this gives the overlapping region
idx = (self.data > 0) & (d1.data > 0)
if len(self.qgrid[idx]) > 0:
qmin0 = min(d1.qgrid[idx])
qmax0 = max(self.qgrid[idx])
# merge SAXS/WAXS based on intensity in the overlapping region
if qmax0 < qmax:
qmax = qmax0
if qmin0 > qmin:
qmin = qmin0
idx = (self.qgrid > qmin) & (self.qgrid < qmax)
# save the raw data in case needed, e.g. for ploting
self.overlaps.append({
'q_overlap': self.qgrid[idx],
'raw_data1': self.data[idx],
'raw_data2': d1.data[idx]
})
else:
# no overlap
# simply stack WAXS data to the high q end of SAXS data
qmin = qmax = max(self.qgrid[self.data > 0])
self.overlaps.append({
'q_overlap': np.empty(0),
'raw_data1': np.empty(0),
'raw_data2': np.empty(0)
})
# idx = np.asarray([],dtype=int)
if len(self.qgrid[idx]) == 0:
if debug != 'quiet':
print("data sets are not overlapping in the given q range.")
if fix_scale < 0:
fix_scale = 1
if debug != 'quiet':
print("forcing fix_scale=1.")
elif len(self.qgrid[idx]) < 5 and debug != 'quiet':
print("too few overlapping points: %d" % len(self.qgrid[idx]))
if fix_scale > 0:
# For a given experimental configuration, the intensity normlization
# factor between the SAXS and WAXS should be well-defined. This factor
# can be determined using scattering data with siginificant intensity
# in the overlapping q-range and applied to all data collected in the
# same configuration.
sc = fix_scale
else:
sc = np.linalg.lstsq(
np.asmatrix(self.data[idx]).T,
np.asmatrix(d1.data[idx]).T)[0]
sc = np.trace(sc)
d1.data /= sc
d1.err /= sc
if len(self.qgrid[idx]) > 0:
if debug == True:
print("Scaled Overlaps by 1/%f" % sc)
self.overlaps[-1]['raw_data2'] /= sc
self.overlaps[-1]['sc'] = sc
self.label = common_name(self.label, d1.label)
if debug == True:
print("set2 scaled by 1/%f" % sc)
print("merged set re-named %s." % self.label)
if len(self.qgrid[idx]) > 0:
self.data[idx] = (self.data[idx] + d1.data[idx]) / 2
# this won't work well if the merging data are mis-matched before bkg subtraction
# but match well after bkg subtraction
self.err[idx] = (self.err[idx] + d1.err[idx]) / 2
self.data[self.qgrid >= qmax] = d1.data[self.qgrid >= qmax]
self.err[self.qgrid >= qmax] = d1.err[self.qgrid >= qmax]
self.comments += "# merged with the following set by matching intensity within (%.4f, %.4f)," % (
qmin, qmax)
self.comments += " scaled by %f\n" % sc
self.comments += d1.comments.replace("# ", "## ")
def avg(self,
dsets,
weighted=False,
qmax_for_weight=0.3,
plot_data=False,
ax=None,
debug=False,
fontsize='large'):
"""
dsets is a collection of Data1d
weighted:
if False
the dsets are simply averaged together
errorbar is increased if there are discrepencies between individual values??
otherwise
weight should contain a list of weight factors, corresponding to each dset
each dset is first scaled by the
a weighted average (smaller errorbar has higher weight) is then performed
ax is the Axes to plot the data in
TODO: should calculate something like the cross-correlation between sets
to evaluate the consistency between them
"""
if debug != 'quiet':
print("averaging data with %s: \n" % self.label, end=' ')
i_fs = get_font_size(fontsize)[0]
n = 1
if plot_data:
if ax is None:
plt.figure()
plt.subplots_adjust(bottom=0.15)
ax = plt.gca()
ax.set_xlabel("$q (\AA^{-1})$", fontsize=get_font_size(i_fs)[1])
ax.set_ylabel("$I$", fontsize=get_font_size(i_fs)[1])
ax.set_xscale('log')
ax.set_yscale('log')
idx = (self.data > 0)
ax.errorbar(self.qgrid[idx],
self.data[idx],
self.err[idx],
label=self.label)
for ov in self.overlaps:
ax.plot(ov['q_overlap'], ov['raw_data1'], "v")
ax.plot(ov['q_overlap'], ov['raw_data2'], "^")
d0 = copy.deepcopy(self)
if len(dsets) == 0:
return d0
if weighted:
wt = []
for d in dsets + [self]:
w0 = np.sum(np.fabs(d.data[d.qgrid < qmax_for_weight]))
if w0 <= 0:
raise Exception(f"weight for averaging <0: {w0}")
wt.append(w0)
wt = np.asarray(wt)
wt /= wt.max()
er2 = (self.err / wt[-1])**2
d0.err = 1. / er2
d0.data = d0.data / er2
if debug == True:
print("weight factors: ", wt)
for i in range(len(dsets)):
d1 = dsets[i]
if debug == True:
print("%s " % d1.label, end=' ')
if not (d0.qgrid == d1.qgrid).all():
raise Exception("\n1D sets cannot be averaged: qgrid mismatch")
d0.trans += d1.trans
d0.trans_w += d1.trans_w
d0.trans_e += d1.trans_e
if weighted:
er2 = (d1.err / wt[i])**2
d0.err += 1 / er2
d0.data += d1.data / er2
else:
d0.data += d1.data
d0.err += d1.err
#if self.transMode == trans_mode.from_beam_center:
# d0.roi += d1.roi
d0.comments += "# averaged with \n%s" % d1.comments.replace(
"# ", "## ")
if plot_data:
idx = (d1.data > 0) # Remove Zeros on plot
ax.errorbar(d1.qgrid[idx],
d1.data[idx] * VOFFSET**n,
d1.err[idx] * VOFFSET**n,
label=d1.label)
for i in range(len(d1.overlaps)):
if plot_data:
ax.plot(d1.overlaps[i]['q_overlap'],
d1.overlaps[i]['raw_data1'] * VOFFSET**n, "v")
ax.plot(d1.overlaps[i]['q_overlap'],
d1.overlaps[i]['raw_data2'] * VOFFSET**n, "^")
d0.overlaps[i]['raw_data1'] += d1.overlaps[i]['raw_data1']
d0.overlaps[i]['raw_data2'] += d1.overlaps[i]['raw_data2']
n += 1
d0.label = common_name(d0.label, d1.label)
d0.trans /= n
d0.trans_w /= n
d0.trans_e /= n
if weighted:
d0.data /= d0.err
d0.err = 1. / np.sqrt(d0.err)
else:
d0.data /= n
d0.err /= (
n * np.sqrt(n)
) # should not be just sqrt(n), that would increase err after averaging
#if self.transMode == trans_mode.from_beam_center:
# d0.roi /= n
for ov in d0.overlaps:
ov['raw_data1'] /= n
ov['raw_data2'] /= n
if debug == True:
print("\naveraged set re-named to %s." % d0.label)
if plot_data:
# plot the averaged data over each individual curve
for i in range(n):
if i == 0:
idx = (d0.data > 0) # Remove Zeros on plot
handles, labels = ax.get_legend_handles_labels()
lbl = "averaged" if "averaged" not in labels else ""
ax.plot(d0.qgrid[idx],
d0.data[idx] * VOFFSET**i,
color="gray",
lw=2,
ls="--",
label=lbl)
else:
idx = (d0.data > 0) # Remove Zeros on plot
ax.plot(d0.qgrid[idx],
d0.data[idx] * VOFFSET**i,
color="gray",
lw=2,
ls="--")
leg = ax.legend(loc='upper right', frameon=False)
for t in leg.get_texts():
t.set_fontsize(get_font_size(i_fs - 2)[1])
return d0
