def event(self, evt, env):
"""The event() function is called for every L1Accept transition. It
outputs the detector image associated with the event @p evt to the
file system.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_dump, self).event(evt, env)
if (evt.get('skip_event')):
return
if self.cspad_img is None:
print "No image to save for %s"%self.timestamp
return
# Where the sample-detector distance is not available, set it to
# zero.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
distance = 0
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
output_filename = self._basename
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
output_filename = self._basename
elif device == 'marccd':
if distance == 0:
distance = evt.get('marccd_distance')
pixel_size = 0.079346
saturated_value = 2**16 - 1
output_filename = self._basename + evt.get(str, 'mccd_name') + "_"
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
if self._format == "pickle":
cspad_tbx.dwritef(d, self._dirname, output_filename)
elif self._format == "tiff":
cspad_tbx.write_tiff(d, self._dirname, output_filename)
output_filename = None
def event(self, evt, env):
"""The event() function is called for every L1Accept transition. It
outputs the detector image associated with the event @p evt to the
file system.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_dump, self).event(evt, env)
if (evt.get('skip_event')):
return
if self.cspad_img is None:
print("No image to save for %s"%self.timestamp)
return
# Where the sample-detector distance is not available, set it to
# zero.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
distance = 0
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
output_filename = self._basename
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
output_filename = self._basename
elif device == 'marccd':
if distance == 0:
distance = evt.get('marccd_distance')
pixel_size = 0.079346
saturated_value = 2**16 - 1
output_filename = self._basename + evt.get(str, 'mccd_name') + "_"
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
if self._format == "pickle":
cspad_tbx.dwritef(d, self._dirname, output_filename)
elif self._format == "tiff":
cspad_tbx.write_tiff(d, self._dirname, output_filename)
output_filename = None
def endjob(self, obj1, obj2=None):
"""The endjob() function writes the mean and standard deviation images
to disk.
@param evt Event object (psana only)
@param env Environment object
"""
if obj2 is None:
env = obj1
else:
evt = obj1
env = obj2
stats = super(mod_average, self).endjob(env)
if stats is None:
return
device = cspad_tbx.address_split(self.address)[2]
if device == 'Andor':
beam_center = (0, 0) # XXX Fiction!
pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
saturated_value = 10000
elif device == 'Cspad' or device == 'Cspad2x2':
beam_center = self.beam_center
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
beam_center = tuple(t // 2 for t in d['mean_img'].focus())
pixel_size = 0.079346
saturated_value = 2**16 - 1
if stats['nmemb'] > 0:
if self.avg_dirname is not None or \
self.avg_basename is not None or \
self._mean_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['mean_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._mean_out is not None:
p = cspad_tbx.dwritef2(d, self._mean_out)
else:
p = cspad_tbx.dwritef(d, self.avg_dirname,
self.avg_basename)
self.logger.info("Average written to %s" % p)
if self.stddev_dirname is not None or \
self.stddev_basename is not None or \
self._std_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['std_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._std_out is not None:
p = cspad_tbx.dwritef2(d, self._std_out)
else:
p = cspad_tbx.dwritef(d, self.stddev_dirname,
self.stddev_basename)
self.logger.info("Standard deviation written to %s" % p)
if self.max_dirname is not None or \
self.max_basename is not None or \
self._max_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['max_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._max_out is not None:
p = cspad_tbx.dwritef2(d, self._max_out)
else:
p = cspad_tbx.dwritef(d, self.max_dirname,
self.max_basename)
self.logger.info("Max written to %s" % p)
if stats['nfail'] == 0:
self.logger.info("%d images processed" % stats['nmemb'])
else:
self.logger.warning("%d images processed, %d failed" %
(stats['nmemb'], stats['nfail']))
def __init__(self, runs, output_dirname=".", roi=None):
avg_basename = "avg_"
stddev_basename = "stddev"
self.sum_img = None
self.sumsq_img = None
self.nmemb = 0
self.roi = cspad_tbx.getOptROI(roi)
self.unbound_pixel_mask = cspad_unbound_pixel_mask()
for i_run, run in enumerate(runs):
run_scratch_dir = run
result = finalise_one_run(run_scratch_dir)
if result.sum_img is None: continue
if self.sum_img is None:
self.sum_img = result.sum_img
self.sumsq_img = result.sumsq_img
else:
self.sum_img += result.sum_img
self.sumsq_img += result.sumsq_img
self.nmemb += result.nmemb
self.avg_img = self.sum_img.as_double() / self.nmemb
self.stddev_img = flex.sqrt(
(self.sumsq_img.as_double() -
self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1))
self.mask = flex.int(self.sum_img.accessor(), 0)
self.mask.set_selected(self.sum_img == 0, 1)
self.mask.set_selected(self.unbound_pixel_mask > 0, 1)
if (output_dirname is not None and avg_basename is not None):
if (not os.path.isdir(output_dirname)):
os.makedirs(output_dirname)
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.avg_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, avg_basename)
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.sum_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, "sum_")
if 1:
output_image(self.avg_img, "%s/avg.png" % output_dirname)
output_image(self.avg_img,
"%s/avg_inv.png" % output_dirname,
invert=True)
if 1:
output_matlab_form(self.sum_img, "%s/sum.m" % output_dirname)
output_matlab_form(self.avg_img, "%s/avg.m" % output_dirname)
output_matlab_form(self.stddev_img,
"%s/stddev.m" % output_dirname)
if (stddev_basename is not None):
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.stddev_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, stddev_basename)
# XXX we should really figure out automatically the area where the spectrum is
#write an integrated spectrum from lines 186-227
#spectrum_focus = self.sum_img.as_numpy_array()[186:228,:]
img = self.sum_img
if self.roi is None:
spectrum_focus = img
mask_focus = self.mask
else:
slices = (slice(self.roi[2],
self.roi[3]), slice(self.roi[0], self.roi[1]))
spectrum_focus = img[slices]
mask_focus = self.mask[slices]
if False:
from matplotlib import pylab
pylab.imshow(spectrum_focus.as_numpy_array())
pylab.show()
output_spectrum(spectrum_focus,
mask_focus=mask_focus,
output_dirname=output_dirname)
print "Total number of images used from %i runs: %i" % (i_run + 1,
self.nmemb)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print "Number of rows: %i" % spectrum_focus.all()[0]
print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus)
print "Number of images used: %i" % flex.sum(
pixel_histograms.histograms.values()[0].slots())
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr = "_%04d" % run
else: runstr = ""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)
if gain_map is None:
gain_map = flex.double(gain_img.accessor(), 0)
img_sel = (gain_img > 0).as_1d()
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_img,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
gain_map.as_1d().set_selected(img_sel.iselection(),
1 / gain_img.as_1d().select(img_sel))
gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_map,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'gain_map_')
class xes_from_histograms(object):
def __init__(self,
pixel_histograms,
output_dirname=".",
gain_map_path=None,
gain_map=None,
estimated_gain=30,
roi=None,
run=None):
self.sum_img = flex.double(flex.grid(
370, 391), 0) # XXX define the image size some other way?
gain_img = flex.double(self.sum_img.accessor(), 0)
assert [gain_map, gain_map_path].count(None) > 0
if gain_map_path is not None:
d = easy_pickle.load(gain_map_path)
gain_map = d["DATA"]
mask = flex.int(self.sum_img.accessor(), 0)
start_row = 370
end_row = 0
print len(pixel_histograms.histograms)
pixels = list(pixel_histograms.pixels())
n_pixels = len(pixels)
if roi is not None:
for k, (i, j) in enumerate(reversed(pixels)):
if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
del pixels[n_pixels - k - 1]
if gain_map is None:
fixed_func = pixel_histograms.fit_one_histogram
else:
def fixed_func(pixel):
return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)
chi_squared_list = flex.double()
for i, pixel in enumerate(pixels):
#print i,pixel
LEG = False
start_row = min(start_row, pixel[0])
end_row = max(end_row, pixel[0])
n_photons = 0
try:
if LEG:
gaussians, two_photon_flag = pixel_histograms.fit_one_histogram(
pixel)
alt_gaussians = pixel_histograms.fit_one_histogram_two_gaussians(
pixel)
except ZeroDivisionError:
print "HEY DIVIDE BY ZERO"
#pixel_histograms.plot_combo(pixel, gaussians)
mask[pixel] = 1
continue
except RuntimeError, e:
print "Error fitting pixel %s" % str(pixel)
print str(e)
mask[pixel] = 1
continue
hist = pixel_histograms.histograms[pixel]
if not LEG:
gs = alt_gaussians[1].params
fit_photons = gs[0] * gs[2] * math.sqrt(2. * math.pi)
n_photons = int(round(fit_photons, 0))
fit_interpretation = pixel_histograms.multiphoton_and_fit_residual(
pixel_histograms.histograms[pixel], alt_gaussians)
multi_photons = fit_interpretation.get_multiphoton_count()
total_photons = n_photons + multi_photons
if False and n_photons < 0: # Generally, do not mask negative values; if fit is still OK
print "\n%d pixel %s altrn %d photons from curvefitting" % (
i, pixel, n_photons)
pixel_histograms.plot_combo(
pixel,
alt_gaussians,
interpretation=fit_interpretation)
mask[
pixel] = 1 # do not mask out negative pixels if the Gaussian fit is good
continue
chi_squared_list.append(fit_interpretation.chi_squared())
suspect = False # don't know the optimal statistical test. Histograms vary primarily by total count & # photons
if total_photons <= 3:
if fit_interpretation.chi_squared(
) > 2.5 or fit_interpretation.quality_factor < 5:
suspect = True
elif 3 < total_photons <= 10:
if fit_interpretation.chi_squared(
) > 5 or fit_interpretation.quality_factor < 10:
suspect = True
elif 10 < total_photons <= 33:
if fit_interpretation.chi_squared(
) > 10 or fit_interpretation.quality_factor < 20:
suspect = True
elif 33 < total_photons <= 100:
if fit_interpretation.chi_squared(
) > 20 or fit_interpretation.quality_factor < 20:
suspect = True
elif 100 < total_photons <= 330:
if fit_interpretation.chi_squared(
) > 30 or fit_interpretation.quality_factor < 25:
suspect = True
elif 330 < total_photons <= 1000:
if fit_interpretation.chi_squared(
) > 40 or fit_interpretation.quality_factor < 30:
suspect = True
elif 1000 < total_photons:
if fit_interpretation.chi_squared(
) > 50 or fit_interpretation.quality_factor < 30:
suspect = True
if suspect:
print "\n%d pixel %s Bad quality 0/1-photon fit" % (
i, pixel), fit_interpretation.quality_factor
print " with chi-squared %10.5f" % fit_interpretation.chi_squared(
)
print " Suspect", suspect
print "%d fit photons, %d total photons" % (n_photons,
total_photons)
#pixel_histograms.plot_combo(pixel, alt_gaussians,
# interpretation=fit_interpretation)
mask[pixel] = 1
continue
self.sum_img[pixel] = n_photons + multi_photons
mask.set_selected(self.sum_img == 0, 1)
unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
mask.set_selected(unbound_pixel_mask > 0, 1)
bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
mask.set_selected(bad_pixel_mask > 0, 1)
for row in range(self.sum_img.all()[0]):
self.sum_img[row:row + 1, :].count(0)
spectrum_focus = self.sum_img[start_row:end_row, :]
mask_focus = mask[start_row:end_row, :]
spectrum_focus.set_selected(mask_focus > 0, 0)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print "Number of rows: %i" % spectrum_focus.all()[0]
print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus)
print "Number of images used: %i" % flex.sum(
pixel_histograms.histograms.values()[0].slots())
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr = "_%04d" % run
else: runstr = ""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)
plot_x, plot_y = xes_finalise.output_spectrum(
spectrum_focus.iround(),
mask_focus=mask_focus,
output_dirname=output_dirname,
run=run)
self.spectrum = (plot_x, plot_y)
self.spectrum_focus = spectrum_focus
xes_finalise.output_matlab_form(
spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
print output_dirname
print "Average chi squared is", flex.mean(
chi_squared_list), "on %d shots" % flex.sum(hist.slots())
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights, outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights >
self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((
peak_heights > self.m_threshold).__and__(
outvalue == 0)).count(True)
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
self.stats_logger.info("BRAGG %.3f %d" %
(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info(
"Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info(
"Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time,
number_of_accepted_peaks, distance, self.sifoil,
self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(
d,
integration_dirname=self.m_integration_dirname,
integration_basename=self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None and hasattr(info, 'spotfinder_results')
if self.m_progress_logging:
if self.m_db_version == 'v1':
if indexed:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best[
"identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host,
self.m_db_name,
self.m_db_user,
self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best[
"identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[
info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,
self.m_db_experiment_tag,
self.m_trial_id,
self.m_rungroup_id,
evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best[
"identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
elif self.m_db_version == 'v2':
key_low = 'cctbx.xfel.radial_average.two_theta_low'
key_high = 'cctbx.xfel.radial_average.two_theta_high'
tt_low = evt.get(key_low)
tt_high = evt.get(key_high)
from xfel.ui.db.dxtbx_db import log_frame
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
sfspots = evt.get('sfspots')
if sfspots is None:
if info is None or not isinstance(info, int):
n_spots = 0
else:
n_spots = info
else:
n_spots = sfspots
if indexed:
known_setting = info.horizons_phil.known_setting
indexed_setting = info.organizer.info[
'best_integration']['counter']
if known_setting is None or known_setting == indexed_setting:
from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
c = construct_reflection_table_and_experiment_list(
info.last_saved_best,
None,
pixel_size,
proceed_without_image=True)
c.assemble_experiments()
c.assemble_reflections()
log_frame(c.experiment_list, c.reflections,
self.db_params, evt.run(), n_spots,
self.timestamp, tt_low, tt_high)
else:
print(
"Not logging %s, wrong bravais setting (expecting %d, got %d)"
% (self.timestamp, known_setting,
indexed_setting))
else:
log_frame(None, None, self.db_params, evt.run(),
n_spots, self.timestamp, tt_low, tt_high)
if self.m_db_logging:
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get(
'ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get(
'ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get(
'ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
).parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time, n_spots,
distance, self.sifoil, self.wavelength, indexed,
mosaic_bloc_rotation, mosaic_block_size,
ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%" % detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(
horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%s" %
detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation, pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(
horizons_phil, imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead"
)
if (self.m_out_dirname is not None
or self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights,outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info("Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(d,
integration_dirname = self.m_integration_dirname,
integration_basename = self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None
if indexed and self.m_progress_logging:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best["identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best["identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
if self.m_db_logging:
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead")
if (self.m_out_dirname is not None or
self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def __init__(self,
pixel_histograms,
output_dirname=".",
gain_map_path=None,
gain_map=None,
method="photon_counting",
estimated_gain=30,
nproc=None,
photon_threshold=2 / 3,
roi=None,
run=None):
assert method in ("sum_adu", "photon_counting")
self.sum_img = flex.double(flex.grid(
370, 391), 0) # XXX define the image size some other way?
gain_img = flex.double(self.sum_img.accessor(), 0)
assert [gain_map, gain_map_path].count(None) > 0
if gain_map_path is not None:
d = easy_pickle.load(gain_map_path)
gain_map = d["DATA"]
two_photon_threshold = photon_threshold + 1
mask = flex.int(self.sum_img.accessor(), 0)
start_row = 370
end_row = 0
print(len(pixel_histograms.histograms))
pixels = list(pixel_histograms.pixels())
n_pixels = len(pixels)
if roi is not None:
for k, (i, j) in enumerate(reversed(pixels)):
if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
del pixels[n_pixels - k - 1]
if gain_map is None:
fixed_func = pixel_histograms.fit_one_histogram
else:
def fixed_func(pixel):
return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)
results = None
if nproc is None: nproc = easy_mp.Auto
nproc = easy_mp.get_processes(nproc)
print("nproc: ", nproc)
stdout_and_results = easy_mp.pool_map(
processes=nproc,
fixed_func=fixed_func,
args=pixels,
func_wrapper="buffer_stdout_stderr")
results = [r for so, r in stdout_and_results]
gains = flex.double()
for i, pixel in enumerate(pixels):
start_row = min(start_row, pixel[0])
end_row = max(end_row, pixel[0])
n_photons = 0
if results is None:
# i.e. not multiprocessing
try:
gaussians = pixel_histograms.fit_one_histogram(pixel)
except RuntimeError as e:
print("Error fitting pixel %s" % str(pixel))
print(str(e))
mask[pixel] = 1
continue
else:
gaussians = results[i]
hist = pixel_histograms.histograms[pixel]
if gaussians is None:
# Presumably the peak fitting failed in some way
print("Skipping pixel %s" % str(pixel))
continue
zero_peak_diff = gaussians[0].params[1]
if gain_map is None:
try:
view_pixel_histograms.check_pixel_histogram_fit(
hist, gaussians)
except view_pixel_histograms.PixelFitError as e:
print("PixelFitError:", str(pixel), str(e))
mask[pixel] = 1
continue
gain = gaussians[1].params[1] - gaussians[0].params[1]
gain_img[pixel] = gain
gain_ratio = gain / estimated_gain
else:
gain = gain_map[pixel]
if gain == 0:
print("bad gain!!!!!", pixel)
continue
gain = 30 / gain
gain_ratio = 1 / gain
gains.append(gain)
#for g in gaussians:
#sigma = abs(g.params[2])
#if sigma < 1 or sigma > 10:
#print "bad sigma!!!!!", pixel, sigma
#mask[pixel] = 1
#continue
if method == "sum_adu":
sum_adu = 0
one_photon_cutoff, two_photon_cutoff = [
(threshold * gain + zero_peak_diff)
for threshold in (photon_threshold, two_photon_threshold)
]
i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
slots = hist.slots().as_double()
slot_centers = hist.slot_centers()
slots -= gaussians[0](slot_centers)
for j in range(i_one_photon_cutoff, len(slots)):
center = slot_centers[j]
sum_adu += slots[j] * (center - zero_peak_diff) * 30 / gain
self.sum_img[pixel] = sum_adu
elif method == "photon_counting":
one_photon_cutoff, two_photon_cutoff = [
(threshold * gain + zero_peak_diff)
for threshold in (photon_threshold, two_photon_threshold)
]
i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
i_two_photon_cutoff = hist.get_i_slot(two_photon_cutoff)
slots = hist.slots()
for j in range(i_one_photon_cutoff, len(slots)):
if j == i_one_photon_cutoff:
center = hist.slot_centers()[j]
upper = center + 0.5 * hist.slot_width()
n_photons += int(
round((upper - one_photon_cutoff) /
hist.slot_width() * slots[j]))
elif j == i_two_photon_cutoff:
center = hist.slot_centers()[j]
upper = center + 0.5 * hist.slot_width()
n_photons += 2 * int(
round((upper - two_photon_cutoff) /
hist.slot_width() * slots[j]))
elif j < i_two_photon_cutoff:
n_photons += int(round(slots[j]))
else:
n_photons += 2 * int(round(slots[j]))
self.sum_img[pixel] = n_photons
stats = scitbx.math.basic_statistics(gains)
print("gain statistics:")
stats.show()
mask.set_selected(self.sum_img == 0, 1)
unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
mask.set_selected(unbound_pixel_mask > 0, 1)
bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
mask.set_selected(bad_pixel_mask > 0, 1)
for row in range(self.sum_img.all()[0]):
self.sum_img[row:row + 1, :].count(0)
spectrum_focus = self.sum_img[start_row:end_row, :]
mask_focus = mask[start_row:end_row, :]
spectrum_focus.set_selected(mask_focus > 0, 0)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print("Number of rows: %i" % spectrum_focus.all()[0])
print("Estimated no. photons counted: %i" % flex.sum(spectrum_focus))
print("Number of images used: %i" %
flex.sum(pixel_histograms.histograms.values()[0].slots()))
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr = "_%04d" % run
else: runstr = ""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)
if gain_map is None:
gain_map = flex.double(gain_img.accessor(), 0)
img_sel = (gain_img > 0).as_1d()
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_img,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
gain_map.as_1d().set_selected(img_sel.iselection(),
1 / gain_img.as_1d().select(img_sel))
gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_map,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'gain_map_')
plot_x, plot_y = xes_finalise.output_spectrum(
spectrum_focus.iround(),
mask_focus=mask_focus,
output_dirname=output_dirname,
run=run)
self.spectrum = (plot_x, plot_y)
self.spectrum_focus = spectrum_focus
xes_finalise.output_matlab_form(
spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
print(output_dirname)
def endjob(self, obj1, obj2=None):
"""The endjob() function writes the mean and standard deviation images
to disk.
@param evt Event object (psana only)
@param env Environment object
"""
if obj2 is None:
env = obj1
else:
evt = obj1
env = obj2
stats = super(mod_average, self).endjob(env)
if stats is None:
return
device = cspad_tbx.address_split(self.address)[2]
if device == 'Andor':
beam_center = (0, 0) # XXX Fiction!
pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
saturated_value = 10000
elif device == 'Cspad' or device == 'Cspad2x2':
beam_center = self.beam_center
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
beam_center = tuple(t // 2 for t in d['mean_img'].focus())
pixel_size = 0.079346
saturated_value = 2**16 - 1
if stats['nmemb'] > 0:
if self.avg_dirname is not None or \
self.avg_basename is not None or \
self._mean_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['mean_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._mean_out is not None:
p = cspad_tbx.dwritef2(d, self._mean_out)
else:
p = cspad_tbx.dwritef(d, self.avg_dirname, self.avg_basename)
self.logger.info("Average written to %s" % p)
if self.stddev_dirname is not None or \
self.stddev_basename is not None or \
self._std_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['std_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._std_out is not None:
p = cspad_tbx.dwritef2(d, self._std_out)
else:
p = cspad_tbx.dwritef(d, self.stddev_dirname, self.stddev_basename)
self.logger.info("Standard deviation written to %s" % p)
if self.max_dirname is not None or \
self.max_basename is not None or \
self._max_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['max_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._max_out is not None:
p = cspad_tbx.dwritef2(d, self._max_out)
else:
p = cspad_tbx.dwritef(d, self.max_dirname, self.max_basename)
self.logger.info("Max written to %s" % p)
if stats['nfail'] == 0:
self.logger.info("%d images processed" % stats['nmemb'])
else:
self.logger.warning(
"%d images processed, %d failed" % (stats['nmemb'], stats['nfail']))
def __init__(self,
runs,
output_dirname=".",
roi=None):
avg_basename="avg_"
stddev_basename="stddev"
self.sum_img = None
self.sumsq_img = None
self.nmemb = 0
self.roi = cspad_tbx.getOptROI(roi)
self.unbound_pixel_mask = cspad_unbound_pixel_mask()
for i_run, run in enumerate(runs):
run_scratch_dir = run
result = finalise_one_run(run_scratch_dir)
if result.sum_img is None: continue
if self.sum_img is None:
self.sum_img = result.sum_img
self.sumsq_img = result.sumsq_img
else:
self.sum_img += result.sum_img
self.sumsq_img += result.sumsq_img
self.nmemb += result.nmemb
self.avg_img = self.sum_img.as_double() / self.nmemb
self.stddev_img = flex.sqrt((self.sumsq_img.as_double() - self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1))
self.mask = flex.int(self.sum_img.accessor(), 0)
self.mask.set_selected(self.sum_img == 0, 1)
self.mask.set_selected(self.unbound_pixel_mask > 0, 1)
if (output_dirname is not None and
avg_basename is not None):
if (not os.path.isdir(output_dirname)):
os.makedirs(output_dirname)
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.avg_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, avg_basename)
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.sum_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, "sum_")
if 1:
output_image(self.avg_img, "%s/avg.png" %output_dirname)
output_image(self.avg_img, "%s/avg_inv.png" %output_dirname, invert=True)
if 1:
output_matlab_form(self.sum_img, "%s/sum.m" %output_dirname)
output_matlab_form(self.avg_img, "%s/avg.m" %output_dirname)
output_matlab_form(self.stddev_img, "%s/stddev.m" %output_dirname)
if (stddev_basename is not None):
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.stddev_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, stddev_basename)
# XXX we should really figure out automatically the area where the spectrum is
#write an integrated spectrum from lines 186-227
#spectrum_focus = self.sum_img.as_numpy_array()[186:228,:]
img = self.sum_img
if self.roi is None:
spectrum_focus = img
mask_focus = self.mask
else:
slices = (slice(self.roi[2],self.roi[3]), slice(self.roi[0],self.roi[1]))
spectrum_focus = img[slices]
mask_focus = self.mask[slices]
if False:
from matplotlib import pylab
pylab.imshow(spectrum_focus.as_numpy_array())
pylab.show()
output_spectrum(spectrum_focus, mask_focus=mask_focus,
output_dirname=output_dirname)
print "Total number of images used from %i runs: %i" %(i_run+1, self.nmemb)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print "Number of rows: %i" %spectrum_focus.all()[0]
print "Estimated no. photons counted: %i" %flex.sum(spectrum_focus)
print "Number of images used: %i" %flex.sum(
pixel_histograms.histograms.values()[0].slots())
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr="_%04d"%run
else: runstr=""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_'%runstr)
if gain_map is None:
gain_map = flex.double(gain_img.accessor(), 0)
img_sel = (gain_img > 0).as_1d()
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=gain_img,
distance=1
)
cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
gain_map.as_1d().set_selected(img_sel.iselection(), 1/gain_img.as_1d().select(img_sel))
gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=gain_map,