def subtract_background(signal, background, plot=False): x, y = smooth_spectrum.interpolate(background[0], background[1]) y_fitted = smoothing.savitzky_golay_filter(x, y, half_window=32, degree=3)[1] signal_x, signal_y = signal signal_x, signal_y = smooth_spectrum.interpolate(signal[0], signal[1]) x_interp_size = x.size() for i, x_i in enumerate(reversed(x)): if x_i not in signal[0]: assert x[x_interp_size - i - 1] == x_i del signal_x[x_interp_size - i - 1] del signal_y[x_interp_size - i - 1] del y_fitted[x_interp_size - i - 1] background_subtracted = signal_y - y_fitted if plot: from matplotlib import pyplot pyplot.plot(signal[0], signal[1], linewidth=2, label="signal+background") pyplot.plot(background[0], background[1], linewidth=2, label="background") pyplot.plot(signal_x, y_fitted, linewidth=2, label="background_fit") pyplot.plot(signal_x, background_subtracted, linewidth=2, label="signal") pyplot.legend(loc=2) pyplot.ylabel("Intensity", fontsize=15) pyplot.xlabel("Pixel column", fontsize=15) pyplot.show() return signal_x, background_subtracted
def run(args): processed = iotbx.phil.process_command_line( args=args, master_string=master_phil_str) args = processed.remaining_args work_params = processed.work.extract().xes processed.work.show() assert len(args) == 1 output_dirname = work_params.output_dirname roi = cspad_tbx.getOptROI(work_params.roi) bg_roi = cspad_tbx.getOptROI(work_params.bg_roi) gain_map_path = work_params.gain_map estimated_gain = work_params.estimated_gain nproc = work_params.nproc photon_threshold = work_params.photon_threshold method = work_params.method print output_dirname if output_dirname is None: output_dirname = os.path.join(os.path.dirname(args[0]), "finalise") print output_dirname hist_d = easy_pickle.load(args[0]) if len(hist_d.keys())==2: hist_d = hist_d['histogram'] pixel_histograms = view_pixel_histograms.pixel_histograms( hist_d, estimated_gain=estimated_gain) result = xes_from_histograms( pixel_histograms, output_dirname=output_dirname, gain_map_path=gain_map_path, estimated_gain=estimated_gain, method=method, nproc=nproc, photon_threshold=photon_threshold, roi=roi, run=work_params.run) if bg_roi is not None: bg_outdir = os.path.normpath(output_dirname)+"_bg" bg_result = xes_from_histograms( pixel_histograms, output_dirname=bg_outdir, gain_map_path=gain_map_path, estimated_gain=estimated_gain, method=method, nproc=nproc, photon_threshold=photon_threshold, roi=bg_roi) from xfel.command_line.subtract_background import subtract_background signal = result.spectrum background = bg_result.spectrum signal = (signal[0].as_double(), signal[1]) background = (background[0].as_double(), background[1]) signal_x, background_subtracted = subtract_background(signal, background, plot=True) f = open(os.path.join(output_dirname, "background_subtracted.txt"), "wb") print >> f, "\n".join(["%i %f" %(x, y) for x, y in zip(signal_x, background_subtracted)]) f.close() else: from xfel.command_line import smooth_spectrum from scitbx.smoothing import savitzky_golay_filter x, y = result.spectrum[0].as_double(), result.spectrum[1] x, y = smooth_spectrum.interpolate(x, y) x, y_smoothed = savitzky_golay_filter( x, y, 20, 4) smooth_spectrum.estimate_signal_to_noise(x, y, y_smoothed)
def exercise_savitzky_golay_smoothing(): plot = False def rms(flex_double): return math.sqrt(flex.mean(flex.pow2(flex_double))) for sigma_frac in (0.005, 0.01, 0.05, 0.1): mean = random.randint(-5, 5) scale = flex.random_double() * 10 sigma = flex.random_double() * 5 + 1 gaussian = scitbx.math.curve_fitting.gaussian(scale, mean, sigma) x = flex.double(frange(-20, 20, 0.1)) y = gaussian(x) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=sigma_frac * flex.max_absolute(y)) g = scitbx.random.variate(rand_norm) noise = g(y.size()) y_noisy = y + noise # according to numerical recipes the best results are obtained where the # full window width is between 1 and 2 times the number of points at fwhm # for polynomials of degree 4 half_window = int(round(0.5 * 2.355 * sigma * 10)) y_filtered = savitzky_golay_filter(x, y_noisy, half_window=half_window, degree=4)[1] extracted_noise = y_noisy - y_filtered rms_noise = rms(noise) rms_extracted_noise = rms(extracted_noise) assert is_below_limit(value=abs(rand_norm.sigma - rms_noise) / rand_norm.sigma, limit=0.15) assert is_below_limit( value=abs(rand_norm.sigma - rms_extracted_noise) / rand_norm.sigma, limit=0.15) diff = y_filtered - y assert is_below_limit(value=(rms(diff) / rand_norm.sigma), limit=0.4) if plot: from matplotlib import pyplot pyplot.plot(x, y) pyplot.plot(x, noise) pyplot.scatter(x, y_noisy, marker="x") pyplot.plot(x, y_filtered) pyplot.show() pyplot.plot(x, extracted_noise) pyplot.plot(x, noise) pyplot.show() return
def exercise_savitzky_golay_smoothing(): plot = False def rms(flex_double): return math.sqrt(flex.mean(flex.pow2(flex_double))) for sigma_frac in (0.005, 0.01, 0.05, 0.1): mean = random.randint(-5,5) scale = flex.random_double() * 10 sigma = flex.random_double() * 5 + 1 gaussian = curve_fitting.gaussian(scale, mean, sigma) x = flex.double(frange(-20,20,0.1)) y = gaussian(x) rand_norm = scitbx.random.normal_distribution( mean=0, sigma=sigma_frac*flex.max_absolute(y)) g = scitbx.random.variate(rand_norm) noise = g(y.size()) y_noisy = y + noise # according to numerical recipes the best results are obtained where the # full window width is between 1 and 2 times the number of points at fwhm # for polynomials of degree 4 half_window = int(round(0.5 * 2.355 * sigma * 10)) y_filtered = savitzky_golay_filter(x, y_noisy, half_window=half_window, degree=4)[1] extracted_noise = y_noisy - y_filtered rms_noise = rms(noise) rms_extracted_noise = rms(extracted_noise) assert is_below_limit( value=abs(rand_norm.sigma - rms_noise)/rand_norm.sigma, limit=0.15) assert is_below_limit( value=abs(rand_norm.sigma - rms_extracted_noise)/rand_norm.sigma, limit=0.15) diff = y_filtered - y assert is_below_limit( value=(rms(diff)/ rand_norm.sigma), limit=0.4) if plot: from matplotlib import pyplot pyplot.plot(x, y) pyplot.plot(x, noise) pyplot.scatter(x, y_noisy, marker="x") pyplot.plot(x, y_filtered) pyplot.show() pyplot.plot(x, extracted_noise) pyplot.plot(x, noise) pyplot.show() return
def estimate_signal_to_noise(x, y_noisy, y_smoothed, plot=False): """Estimate noise in spectra by subtracting a smoothed spectrum from the original noisy unsmoothed spectrum. See: The extraction of signal to noise values in x-ray absorption spectroscopy A. J. Dent, P. C. Stephenson, and G. N. Greaves Rev. Sci. Instrum. 63, 856 (1992); https://doi.org/10.1063/1.1142627 """ noise = y_noisy - y_smoothed noise_sq = flex.pow2(noise) from xfel.command_line.view_pixel_histograms import sliding_average sigma_sq = sliding_average(noise_sq, n=31) sigma_sq = smoothing.savitzky_golay_filter(x.as_double(), flex.pow2(noise), half_window=20, degree=1)[1] sigma_sq.set_selected(sigma_sq <= 0, flex.mean(sigma_sq)) # or do this instead to use the background region as the source of noise: #signal_to_noise = y_smoothed/math.sqrt(flex.mean(noise_sq[50:190])) signal_to_noise = y_smoothed / flex.sqrt(sigma_sq) #signal_to_noise.set_selected(x < 50, 0) #signal_to_noise.set_selected(x > 375, 0) if plot: from matplotlib import pyplot linewidth = 2 pyplot.plot(x, y_noisy, linewidth=linewidth) pyplot.plot(x, y_smoothed, linewidth=linewidth) pyplot_label_axes() pyplot.show() pyplot.plot(x, noise, linewidth=linewidth, label="noise") pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=linewidth, label="sigma") pyplot_label_axes() pyplot.legend(loc=2, prop={'size': 20}) pyplot.show() pyplot.plot(x, signal_to_noise, linewidth=linewidth) pyplot_label_axes() pyplot.show() return signal_to_noise
def estimate_signal_to_noise(x, y_noisy, y_smoothed, plot=False): """Estimate noise in spectra by subtracting a smoothed spectrum from the original noisy unsmoothed spectrum. See: The extraction of signal to noise values in x-ray absorption spectroscopy A. J. Dent, P. C. Stephenson, and G. N. Greaves Rev. Sci. Instrum. 63, 856 (1992); http://dx.doi.org/10.1063/1.1142627 """ noise = y_noisy - y_smoothed noise_sq = flex.pow2(noise) from xfel.command_line.view_pixel_histograms import sliding_average sigma_sq = sliding_average(noise_sq, n=31) sigma_sq = smoothing.savitzky_golay_filter( x.as_double(), flex.pow2(noise), half_window=20, degree=1)[1] sigma_sq.set_selected(sigma_sq <= 0, flex.mean(sigma_sq)) # or do this instead to use the background region as the source of noise: #signal_to_noise = y_smoothed/math.sqrt(flex.mean(noise_sq[50:190])) signal_to_noise = y_smoothed/flex.sqrt(sigma_sq) #signal_to_noise.set_selected(x < 50, 0) #signal_to_noise.set_selected(x > 375, 0) if plot: from matplotlib import pyplot linewidth=2 pyplot.plot(x, y_noisy, linewidth=linewidth) pyplot.plot(x, y_smoothed, linewidth=linewidth) pyplot_label_axes() pyplot.show() pyplot.plot(x, noise, linewidth=linewidth, label="noise") pyplot.plot(x, flex.sqrt(sigma_sq), linewidth=linewidth, label="sigma") pyplot_label_axes() pyplot.legend(loc=2, prop={'size':20}) pyplot.show() pyplot.plot(x, signal_to_noise, linewidth=linewidth) pyplot_label_axes() pyplot.show() return signal_to_noise
def run(args): master_phil = iotbx.phil.parse(master_phil_str) processed = iotbx.phil.process_command_line(args=args, master_string=master_phil_str) args = processed.remaining_args work_params = processed.work.extract() x_offsets = work_params.x_offsets bg_range_min, bg_range_max = work_params.bg_range if work_params.plot_range is not None: x_min, x_max = work_params.plot_range else: x_min, x_max = (0, 385) print bg_range_min, bg_range_max if x_offsets is None: x_offsets = [0] * len(args) legend = work_params.legend linewidth = 2 fontsize = 26 xy_pairs = [] colours = [ "cornflowerblue", "darkmagenta", "darkgreen", "black", "red", "blue", "pink" ] colours[2] = "orangered" colours[1] = "olivedrab" min_background = 1e16 #x_min, x_max = (0, 391) #x_min, x_max = (0, 360) #x_min, x_max = (200, 360) for i, filename in enumerate(args): print filename f = open(filename, 'rb') x, y = zip(*[ line.split() for line in f.readlines() if not line.startswith("#") ]) x = flex.double(flex.std_string(x)) y = flex.double(flex.std_string(y)) if work_params.smoothing.method is not None: savitzky_golay_half_window = work_params.smoothing.savitzky_golay.half_window savitzky_golay_degree = work_params.smoothing.savitzky_golay.degree fourier_cutoff = work_params.smoothing.fourier_filter_cutoff method = work_params.smoothing.method if method == "fourier_filter": assert work_params.smoothing.fourier_filter_cutoff is not None if method == "savitzky_golay": x, y = smoothing.savitzky_golay_filter( x, y, savitzky_golay_half_window, savitzky_golay_degree) elif method == "fourier_filter": x, y = smooth_spectrum.fourier_filter( x, y, cutoff_frequency=fourier_cutoff) x += x_offsets[i] y = y.select((x <= x_max) & (x > 0)) x = x.select((x <= x_max) & (x > 0)) bg_sel = (x > bg_range_min) & (x < bg_range_max) xy_pairs.append((x, y)) min_background = min(min_background, flex.mean(y.select(bg_sel)) / flex.max(y)) y -= min_background print "Peak maximum at: %i" % int(x[flex.max_index(y)]) for i, filename in enumerate(args): if legend is None: label = filename else: print legend assert len(legend) == len(args) label = legend[i] x, y = xy_pairs[i] if i == -1: x, y = interpolate(x, y) x, y = savitzky_golay_filter(x, y) #if i == 0: #y -= 10 bg_sel = (x > bg_range_min) & (x < bg_range_max) y -= (flex.mean(y.select(bg_sel)) - min_background * flex.max(y)) #y -= flex.min(y) y_min = flex.min(y.select(bg_sel)) if i == -2: y += 0.2 * flex.max(y) print "minimum at: %i" % int(x[flex.min_index(y)]), flex.min(y) #print "fwhm: %.2f" %full_width_half_max(x, y) y /= flex.max(y) if len(colours) > i: pyplot.plot(x, y, label=label, linewidth=linewidth, color=colours[i]) else: pyplot.plot(x, y, label=label, linewidth=linewidth) pyplot.ylabel("Intensity", fontsize=fontsize) pyplot.xlabel("Pixel column", fontsize=fontsize) if i > 0: # For some reason the line below causes a floating point error if we only # have one plot (i.e. i==0) legend = pyplot.legend(loc=2) for t in legend.get_texts(): t.set_fontsize(fontsize) axes = pyplot.axes() for tick in axes.xaxis.get_ticklabels(): tick.set_fontsize(20) for tick in axes.yaxis.get_ticklabels(): tick.set_fontsize(20) pyplot.ylim(0, 1) pyplot.xlim(x_min, x_max) ax = pyplot.axes() #ax.xaxis.set_minor_locator(pyplot.MultipleLocator(5)) #ax.yaxis.set_major_locator(pyplot.MultipleLocator(0.1)) #ax.yaxis.set_minor_locator(pyplot.MultipleLocator(0.05)) pyplot.show()
def run(args): processed = iotbx.phil.process_command_line( args=args, master_string=master_phil_str) args = processed.remaining_args work_params = processed.work.extract().smoothing savitzky_golay_half_window = work_params.savitzky_golay.half_window savitzky_golay_degree = work_params.savitzky_golay.degree fourier_cutoff = work_params.fourier_filter_cutoff #assert (fourier_cutoff is not None or #[savitzky_golay_degree, savitzky_golay_half_window].count(None) == 0) method = work_params.method if method == "fourier_filter": assert work_params.fourier_filter_cutoff is not None for i, filename in enumerate(args): print filename f = open(filename, 'rb') x, y = zip(*[line.split() for line in f.readlines() if not line.startswith("#")]) x = flex.double(flex.std_string(x)) y = flex.double(flex.std_string(y)) x = x[:-2] y = y[:-2] x_orig = x.deep_copy() y_orig = y.deep_copy() x, y = interpolate(x, y) if method == "savitzky_golay": x, y_smoothed = smoothing.savitzky_golay_filter( x, y, savitzky_golay_half_window, savitzky_golay_degree) elif method == "fourier_filter": x, y_smoothed = fourier_filter(x, y, cutoff_frequency=fourier_cutoff) from matplotlib import pyplot fontsize = 20 pyplot.plot(x_orig, y_orig, color='black', linestyle='dotted', linewidth=2) #pyplot.plot(x_orig, y_orig, color='black', linewidth=2) pyplot.plot(x, y_smoothed, linewidth=2, color='red') pyplot_label_axes() pyplot.show() #pyplot.plot(x[:385], (y - y_smoothed)[:385], linewidth=2) #pyplot_label_axes() #pyplot.show() filename = os.path.join(os.path.dirname(filename), "smoothed_spectrum.txt") f = open(filename, "wb") print >> f, "\n".join(["%i %f" %(xi, yi) for xi, yi in zip(x, y_smoothed)]) f.close() print "Smoothed spectrum written to %s" %filename x_interp_size = x.size() for i, x_i in enumerate(reversed(x)): if x_i not in x_orig: assert x[x_interp_size - i - 1] == x_i del x[x_interp_size - i - 1] del y[x_interp_size - i - 1] del y_smoothed[x_interp_size - i - 1] x = x[10:-10] y = y[10:-10] y_smoothed = y_smoothed[10:-10] signal_to_noise = estimate_signal_to_noise(x, y, y_smoothed, plot=False)
def run(args): processed = iotbx.phil.process_command_line(args=args, master_string=master_phil_str) args = processed.remaining_args work_params = processed.work.extract().xes processed.work.show() assert len(args) == 1 output_dirname = work_params.output_dirname roi = cspad_tbx.getOptROI(work_params.roi) bg_roi = cspad_tbx.getOptROI(work_params.bg_roi) gain_map_path = work_params.gain_map estimated_gain = work_params.estimated_gain nproc = work_params.nproc photon_threshold = work_params.photon_threshold method = work_params.method print output_dirname if output_dirname is None: output_dirname = os.path.join(os.path.dirname(args[0]), "finalise") print output_dirname hist_d = easy_pickle.load(args[0]) if len(hist_d.keys()) == 2: hist_d = hist_d['histogram'] pixel_histograms = view_pixel_histograms.pixel_histograms( hist_d, estimated_gain=estimated_gain) result = xes_from_histograms(pixel_histograms, output_dirname=output_dirname, gain_map_path=gain_map_path, estimated_gain=estimated_gain, method=method, nproc=nproc, photon_threshold=photon_threshold, roi=roi, run=work_params.run) if bg_roi is not None: bg_outdir = os.path.normpath(output_dirname) + "_bg" bg_result = xes_from_histograms(pixel_histograms, output_dirname=bg_outdir, gain_map_path=gain_map_path, estimated_gain=estimated_gain, method=method, nproc=nproc, photon_threshold=photon_threshold, roi=bg_roi) from xfel.command_line.subtract_background import subtract_background signal = result.spectrum background = bg_result.spectrum signal = (signal[0].as_double(), signal[1]) background = (background[0].as_double(), background[1]) signal_x, background_subtracted = subtract_background(signal, background, plot=True) f = open(os.path.join(output_dirname, "background_subtracted.txt"), "wb") print >> f, "\n".join([ "%i %f" % (x, y) for x, y in zip(signal_x, background_subtracted) ]) f.close() else: from xfel.command_line import smooth_spectrum from scitbx.smoothing import savitzky_golay_filter x, y = result.spectrum[0].as_double(), result.spectrum[1] x, y = smooth_spectrum.interpolate(x, y) x, y_smoothed = savitzky_golay_filter(x, y, 20, 4) smooth_spectrum.estimate_signal_to_noise(x, y, y_smoothed)
def run(args): master_phil = iotbx.phil.parse(master_phil_str) processed = iotbx.phil.process_command_line( args=args, master_string=master_phil_str) args = processed.remaining_args work_params = processed.work.extract() x_offsets = work_params.x_offsets bg_range_min, bg_range_max = work_params.bg_range if work_params.plot_range is not None: x_min, x_max = work_params.plot_range else: x_min, x_max = (0, 385) print bg_range_min, bg_range_max if x_offsets is None: x_offsets = [0]*len(args) legend = work_params.legend linewidth = 2 fontsize = 26 xy_pairs = [] colours = ["cornflowerblue", "darkmagenta", "darkgreen", "black", "red", "blue", "pink"] colours[2] = "orangered" colours[1] = "olivedrab" min_background = 1e16 #x_min, x_max = (0, 391) #x_min, x_max = (0, 360) #x_min, x_max = (200, 360) for i, filename in enumerate(args): print filename f = open(filename, 'rb') x, y = zip(*[line.split() for line in f.readlines() if not line.startswith("#")]) x = flex.double(flex.std_string(x)) y = flex.double(flex.std_string(y)) if work_params.smoothing.method is not None: savitzky_golay_half_window = work_params.smoothing.savitzky_golay.half_window savitzky_golay_degree = work_params.smoothing.savitzky_golay.degree fourier_cutoff = work_params.smoothing.fourier_filter_cutoff method = work_params.smoothing.method if method == "fourier_filter": assert work_params.smoothing.fourier_filter_cutoff is not None if method == "savitzky_golay": x, y = smoothing.savitzky_golay_filter( x, y, savitzky_golay_half_window, savitzky_golay_degree) elif method == "fourier_filter": x, y = smooth_spectrum.fourier_filter(x, y, cutoff_frequency=fourier_cutoff) x += x_offsets[i] y = y.select((x <= x_max) & (x > 0)) x = x.select((x <= x_max) & (x > 0)) bg_sel = (x > bg_range_min) & (x < bg_range_max) xy_pairs.append((x,y)) min_background = min(min_background, flex.mean(y.select(bg_sel))/flex.max(y)) y -= min_background print "Peak maximum at: %i" %int(x[flex.max_index(y)]) for i, filename in enumerate(args): if legend is None: label = filename else: print legend assert len(legend) == len(args) label = legend[i] x, y = xy_pairs[i] if i == -1: x, y = interpolate(x, y) x, y = savitzky_golay_filter(x, y) #if i == 0: #y -= 10 bg_sel = (x > bg_range_min) & (x < bg_range_max) y -= (flex.mean(y.select(bg_sel)) - min_background*flex.max(y)) #y -= flex.min(y) y_min = flex.min(y.select(bg_sel)) if i == -2: y += 0.2 * flex.max(y) print "minimum at: %i" %int(x[flex.min_index(y)]), flex.min(y) #print "fwhm: %.2f" %full_width_half_max(x, y) y /= flex.max(y) if len(colours) > i: pyplot.plot(x, y, label=label, linewidth=linewidth, color=colours[i]) else: pyplot.plot(x, y, label=label, linewidth=linewidth) pyplot.ylabel("Intensity", fontsize=fontsize) pyplot.xlabel("Pixel column", fontsize=fontsize) if i > 0: # For some reason the line below causes a floating point error if we only # have one plot (i.e. i==0) legend = pyplot.legend(loc=2) for t in legend.get_texts(): t.set_fontsize(fontsize) axes = pyplot.axes() for tick in axes.xaxis.get_ticklabels(): tick.set_fontsize(20) for tick in axes.yaxis.get_ticklabels(): tick.set_fontsize(20) pyplot.ylim(0,1) pyplot.xlim(x_min, x_max) ax = pyplot.axes() #ax.xaxis.set_minor_locator(pyplot.MultipleLocator(5)) #ax.yaxis.set_major_locator(pyplot.MultipleLocator(0.1)) #ax.yaxis.set_minor_locator(pyplot.MultipleLocator(0.05)) pyplot.show()