def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_radial_average, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = 0.079346
saturated_value = 2**16 - 1
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
from xfel.command_line.radial_average import run
args = [
"file_path=XTC stream",
"xfel_target=%s"%self.m_xtal_target,
"verbose=False"
]
t = self.timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]
if self._dirname is not None:
dest_path = os.path.join(self._dirname, self._basename + s + ".txt")
args.append("output_file=%s"%dest_path)
self.logger.info("Calculating radial average for image %s"%s)
xvals, results = run(args, d)
evt.put(xvals, "cctbx.xfel.radial_average.xvals")
evt.put(results, "cctbx.xfel.radial_average.results")
def do_work(img_no):
n_fails = 0
while True:
try:
raw_data = data.get_raw_data(img_no)
break
except (KeyError, ValueError):
n_fails += 1
print "Fail to read, attempt number", n_fails
if n_fails > 100:
raise Exception("Couldn't read the data")
import time
time.sleep(n_fails * 0.1)
imgdict = cspad_tbx.dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
address="Sacla.MPCCD.8tile",
active_areas=active_areas)
imgdict = crop_image_pickle(
imgdict,
preserve_active_areas_even_though_cropping_would_invalidate_them=True)
dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no)
print "Saving image", img_no, "to", dest_path
easy_pickle.dump(dest_path, imgdict)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_image_dict, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
if distance == 0:
distance = evt.get("marccd_distance")
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
evt.put(d, self.m_out_key)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def save_image(
command_line,
imgpath,
scan,
raw_data,
distance,
pixel_size,
wavelength,
beam_x,
beam_y,
overload,
timestamp,
image_number=None,
):
if image_number is None:
destpath = os.path.join(
os.path.dirname(imgpath),
os.path.splitext(os.path.basename(imgpath))[0] + ".pickle",
)
else:
destpath = os.path.join(
os.path.dirname(imgpath),
os.path.splitext(os.path.basename(imgpath))[0] +
"%05d.pickle" % image_number,
)
if command_line.options.skip_converted and os.path.isfile(destpath):
if command_line.options.verbose:
print("Skipping %s, file exists" % imgpath)
return
data = dpack(
data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp,
)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data["OSC_START"] = osc_start
data["OSC_RANGE"] = osc_range
data["TIME"] = scan.get_exposure_times()[0]
if command_line.options.crop:
data = crop_image_pickle(data)
if command_line.options.verbose:
print("Writing", destpath)
easy_pickle.dump(destpath, data)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition. It
outputs the detector image associated with the event @p evt to the
file system.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_dump, self).event(evt, env)
if (evt.get('skip_event')):
return
if self.cspad_img is None:
print "No image to save for %s"%self.timestamp
return
# Where the sample-detector distance is not available, set it to
# zero.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
distance = 0
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
output_filename = self._basename
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
output_filename = self._basename
elif device == 'marccd':
if distance == 0:
distance = evt.get('marccd_distance')
pixel_size = 0.079346
saturated_value = 2**16 - 1
output_filename = self._basename + evt.get(str, 'mccd_name') + "_"
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
if self._format == "pickle":
cspad_tbx.dwritef(d, self._dirname, output_filename)
elif self._format == "tiff":
cspad_tbx.write_tiff(d, self._dirname, output_filename)
output_filename = None
def event(self, evt, env):
"""The event() function is called for every L1Accept transition. It
outputs the detector image associated with the event @p evt to the
file system.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_dump, self).event(evt, env)
if (evt.get('skip_event')):
return
if self.cspad_img is None:
print("No image to save for %s"%self.timestamp)
return
# Where the sample-detector distance is not available, set it to
# zero.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
distance = 0
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
output_filename = self._basename
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
output_filename = self._basename
elif device == 'marccd':
if distance == 0:
distance = evt.get('marccd_distance')
pixel_size = 0.079346
saturated_value = 2**16 - 1
output_filename = self._basename + evt.get(str, 'mccd_name') + "_"
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
if self._format == "pickle":
cspad_tbx.dwritef(d, self._dirname, output_filename)
elif self._format == "tiff":
cspad_tbx.write_tiff(d, self._dirname, output_filename)
output_filename = None
def set_up_hitfinder(self):
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
img_dim = (1765, 1765)
pixel_size = cspad_tbx.pixel_size
elif device == 'marccd':
img_dim = (4500, 4500)
pixel_size = 0.079346
elif device == 'Rayonix':
img_dim = rayonix_tbx.get_rayonix_detector_dimensions(
self.bin_size)
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
else:
raise RuntimeError("Unsupported device %s" % self.address)
if self.beam_center is None:
self.beam_center = [0, 0]
self.hitfinder_d = cspad_tbx.dpack(
active_areas=self.active_areas,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
xtal_target=self.m_xtal_target)
if device == 'Cspad':
# Figure out which ASIC:s are on the central four sensors. This
# only applies to the CSPAD.
assert len(self.active_areas) % 4 == 0
distances = flex.double()
for i in range(0, len(self.active_areas), 4):
cenasic = (
(self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
(self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
distances.append(
math.hypot(cenasic[0] - self.beam_center[0],
cenasic[1] - self.beam_center[1]))
orders = flex.sort_permutation(distances)
# Use the central 8 ASIC:s (central 4 sensors).
flags = flex.int(len(self.active_areas) // 4, 0)
for i in range(8):
flags[orders[i]] = 1
self.asic_filter = "distl.tile_flags=" + ",".join(
["%1d" % b for b in flags])
elif device == 'marccd':
# There is only one active area for the MAR CCD, so use it.
self.asic_filter = "distl.tile_flags=1"
elif device == 'Rayonix':
# There is only one active area for the Rayonix, so use it.
self.asic_filter = "distl.tile_flags=1"
def set_up_hitfinder(self):
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
img_dim = (1765, 1765)
pixel_size = cspad_tbx.pixel_size
elif device == 'marccd':
img_dim = (4500, 4500)
pixel_size = 0.079346
elif device == 'Rayonix':
img_dim = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
else:
raise RuntimeError("Unsupported device %s" % self.address)
if self.beam_center is None:
self.beam_center = [0,0]
self.hitfinder_d = cspad_tbx.dpack(
active_areas=self.active_areas,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
xtal_target=self.m_xtal_target)
if device == 'Cspad':
# Figure out which ASIC:s are on the central four sensors. This
# only applies to the CSPAD.
assert len(self.active_areas) % 4 == 0
distances = flex.double()
for i in range(0, len(self.active_areas), 4):
cenasic = ((self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
(self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
distances.append(math.hypot(cenasic[0] - self.beam_center[0],
cenasic[1] - self.beam_center[1]))
orders = flex.sort_permutation(distances)
# Use the central 8 ASIC:s (central 4 sensors).
flags = flex.int(len(self.active_areas) // 4, 0)
for i in range(8):
flags[orders[i]] = 1
self.asic_filter = "distl.tile_flags=" + ",".join(
["%1d" % b for b in flags])
elif device == 'marccd':
# There is only one active area for the MAR CCD, so use it.
self.asic_filter = "distl.tile_flags=1"
elif device == 'Rayonix':
# There is only one active area for the Rayonix, so use it.
self.asic_filter = "distl.tile_flags=1"
def run(args):
assert len(args) == 3
d1 = easy_pickle.load(args[0])
d2 = easy_pickle.load(args[1])
image_1 = d1["DATA"]
image_2 = d2["DATA"]
assert image_1.all() == image_2.all()
diff_image = image_1 - image_2
d = cspad_tbx.dpack(
data=diff_image,
timestamp=cspad_tbx.evt_timestamp(),
distance=1,
)
easy_pickle.dump(args[2], d)
def run(args):
assert len(args) == 3
d1 = easy_pickle.load(args[0])
d2 = easy_pickle.load(args[1])
image_1 = d1["DATA"]
image_2 = d2["DATA"]
assert image_1.all() == image_2.all()
diff_image = image_1 - image_2
d = cspad_tbx.dpack(
data=diff_image,
timestamp=cspad_tbx.evt_timestamp(),
distance=1,
)
easy_pickle.dump(args[2], d)
def save_image(command_line, imgpath, scan, raw_data, distance, pixel_size, wavelength, beam_x, beam_y, overload, timestamp, image_number = None):
if image_number is None:
destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle")
else:
destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + "%05d.pickle"%image_number)
if command_line.options.skip_converted and os.path.isfile(destpath):
if command_line.options.verbose:
print "Skipping %s, file exists"%imgpath
return
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp
)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = osc_start
data['OSC_RANGE'] = osc_range
data['TIME'] = scan.get_exposure_times()[0]
if command_line.options.crop:
data = crop_image_pickle(data)
if command_line.options.verbose:
print "Writing", destpath
easy_pickle.dump(destpath, data)
def __init__(self,
runs,
output_dirname=".",
roi=None):
avg_basename="avg_"
stddev_basename="stddev"
self.sum_img = None
self.sumsq_img = None
self.nmemb = 0
self.roi = cspad_tbx.getOptROI(roi)
self.unbound_pixel_mask = cspad_unbound_pixel_mask()
for i_run, run in enumerate(runs):
run_scratch_dir = run
result = finalise_one_run(run_scratch_dir)
if result.sum_img is None: continue
if self.sum_img is None:
self.sum_img = result.sum_img
self.sumsq_img = result.sumsq_img
else:
self.sum_img += result.sum_img
self.sumsq_img += result.sumsq_img
self.nmemb += result.nmemb
self.avg_img = self.sum_img.as_double() / self.nmemb
self.stddev_img = flex.sqrt((self.sumsq_img.as_double() - self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1))
self.mask = flex.int(self.sum_img.accessor(), 0)
self.mask.set_selected(self.sum_img == 0, 1)
self.mask.set_selected(self.unbound_pixel_mask > 0, 1)
if (output_dirname is not None and
avg_basename is not None):
if (not os.path.isdir(output_dirname)):
os.makedirs(output_dirname)
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.avg_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, avg_basename)
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.sum_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, "sum_")
if 1:
output_image(self.avg_img, "%s/avg.png" %output_dirname)
output_image(self.avg_img, "%s/avg_inv.png" %output_dirname, invert=True)
if 1:
output_matlab_form(self.sum_img, "%s/sum.m" %output_dirname)
output_matlab_form(self.avg_img, "%s/avg.m" %output_dirname)
output_matlab_form(self.stddev_img, "%s/stddev.m" %output_dirname)
if (stddev_basename is not None):
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.stddev_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, stddev_basename)
# XXX we should really figure out automatically the area where the spectrum is
#write an integrated spectrum from lines 186-227
#spectrum_focus = self.sum_img.as_numpy_array()[186:228,:]
img = self.sum_img
if self.roi is None:
spectrum_focus = img
mask_focus = self.mask
else:
slices = (slice(self.roi[2],self.roi[3]), slice(self.roi[0],self.roi[1]))
spectrum_focus = img[slices]
mask_focus = self.mask[slices]
if False:
from matplotlib import pylab
pylab.imshow(spectrum_focus.as_numpy_array())
pylab.show()
output_spectrum(spectrum_focus, mask_focus=mask_focus,
output_dirname=output_dirname)
print "Total number of images used from %i runs: %i" %(i_run+1, self.nmemb)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_radial_average, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = 0.079346
saturated_value = 2**16 - 1
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
from xfel.command_line.radial_average import run
args = [
"file_path=XTC stream",
"xfel_target=%s" % self.m_xtal_target, "verbose=False"
]
t = self.timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
20:23]
if self._dirname is not None:
dest_path = os.path.join(self._dirname,
self._basename + s + ".txt")
args.append("output_file=%s" % dest_path)
self.logger.info("Calculating radial average for image %s" % s)
xvals, results = run(args, d)
evt.put(xvals, "cctbx.xfel.radial_average.xvals")
evt.put(results, "cctbx.xfel.radial_average.results")
def get_closest_idx(data, val):
from scitbx.array_family import flex
deltas = flex.abs(data - val)
return flex.first_index(deltas, flex.min(deltas))
if self._two_theta_low is not None:
i_low = results[get_closest_idx(xvals, self._two_theta_low)]
evt.put(i_low, "cctbx.xfel.radial_average.two_theta_low")
if self._two_theta_high is not None:
i_high = results[get_closest_idx(xvals, self._two_theta_high)]
evt.put(i_high, "cctbx.xfel.radial_average.two_theta_high")
def run(args):
command_line = (option_parser()
.option("-o", "--output_filename",
action="store",
type="string",
help="Filename for the output pickle file",
default="gain_map.pickle")
.option("-f", "--detector_format_version",
action="store",
type="string",
help="Detector format version to use for generating active areas and laying out tiles",
default=None)
.option("-m", "--optical_metrology_path",
action="store",
type="string",
help="Path to slac optical metrology file. If not set, use Run 4 metrology",
default=None)
.option("-d", "--distance",
action="store",
type="int",
help="Detector distance put into the gain pickle file. Not needed for processing.",
default="0")
.option("-w", "--wavelength",
action="store",
type="float",
help="Incident beam wavelength put into the gain pickle file. Not needed for processing.",
default="0")
).process(args=args)
output_filename = command_line.options.output_filename
detector_format_version = command_line.options.detector_format_version
if detector_format_version is None or 'XPP' not in detector_format_version:
beam_center_x = None
beam_center_y = None
else:
beam_center_x = 1765 // 2 * 0.11
beam_center_y = 1765 // 2 * 0.11
address, timestamp = address_and_timestamp_from_detector_format_version(detector_format_version)
# if no detector format version is provided, make sure to write no address to the image pickle
# but CsPadDetector (called later), needs an address, so give it a fake one
save_address = address is not None
if not save_address:
address = "CxiDs1-0|Cspad-0" # time stamp will still be None
timestamp = evt_timestamp((timestamp,0))
args = command_line.args
assert len(args) == 1
if args[0].endswith('.npy'):
data = numpy.load(args[0])
det, active_areas = convert_2x2(data, detector_format_version, address)
elif args[0].endswith('.txt') or args[0].endswith('.gain'):
raw_data = numpy.loadtxt(args[0])
assert raw_data.shape in [(5920, 388), (11840, 194)]
det, active_areas = convert_detector(raw_data, detector_format_version, address, command_line.options.optical_metrology_path)
img_diff = det
img_sel = (img_diff > 0).as_1d()
gain_map = flex.double(img_diff.accessor(), 0)
gain_map.as_1d().set_selected(img_sel.iselection(), 1/img_diff.as_1d().select(img_sel))
gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
if not save_address:
address = None
d = cspad_tbx.dpack(data=gain_map, address=address, active_areas=active_areas, timestamp=timestamp,
distance=command_line.options.distance,wavelength=command_line.options.wavelength,
beam_center_x = beam_center_x, beam_center_y = beam_center_y)
easy_pickle.dump(output_filename, d)
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except Exception as e:
print 'IOTA IMPORT ERROR:', e
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
img_type = 'pickle'
else:
img_type = 'raw'
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec,msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp
)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
else:
data = None
img_type = 'not imported'
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
from dxtbx.datablock import DataBlockFactory
from dials.command_line.estimate_gain import estimate_gain
with misc.Capturing() as junk_output:
try:
datablock = DataBlockFactory.from_filenames([self.raw_img])[0]
imageset = datablock.extract_imagesets()[0]
self.gain = estimate_gain(imageset)
except Exception as e:
self.gain = 1.0
else:
self.gain = 1.0
return data, img_type
class xes_from_histograms(object):
def __init__(self,
pixel_histograms,
output_dirname=".",
gain_map_path=None,
gain_map=None,
estimated_gain=30,
roi=None,
run=None):
self.sum_img = flex.double(flex.grid(
370, 391), 0) # XXX define the image size some other way?
gain_img = flex.double(self.sum_img.accessor(), 0)
assert [gain_map, gain_map_path].count(None) > 0
if gain_map_path is not None:
d = easy_pickle.load(gain_map_path)
gain_map = d["DATA"]
mask = flex.int(self.sum_img.accessor(), 0)
start_row = 370
end_row = 0
print len(pixel_histograms.histograms)
pixels = list(pixel_histograms.pixels())
n_pixels = len(pixels)
if roi is not None:
for k, (i, j) in enumerate(reversed(pixels)):
if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
del pixels[n_pixels - k - 1]
if gain_map is None:
fixed_func = pixel_histograms.fit_one_histogram
else:
def fixed_func(pixel):
return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)
chi_squared_list = flex.double()
for i, pixel in enumerate(pixels):
#print i,pixel
LEG = False
start_row = min(start_row, pixel[0])
end_row = max(end_row, pixel[0])
n_photons = 0
try:
if LEG:
gaussians, two_photon_flag = pixel_histograms.fit_one_histogram(
pixel)
alt_gaussians = pixel_histograms.fit_one_histogram_two_gaussians(
pixel)
except ZeroDivisionError:
print "HEY DIVIDE BY ZERO"
#pixel_histograms.plot_combo(pixel, gaussians)
mask[pixel] = 1
continue
except RuntimeError, e:
print "Error fitting pixel %s" % str(pixel)
print str(e)
mask[pixel] = 1
continue
hist = pixel_histograms.histograms[pixel]
if not LEG:
gs = alt_gaussians[1].params
fit_photons = gs[0] * gs[2] * math.sqrt(2. * math.pi)
n_photons = int(round(fit_photons, 0))
fit_interpretation = pixel_histograms.multiphoton_and_fit_residual(
pixel_histograms.histograms[pixel], alt_gaussians)
multi_photons = fit_interpretation.get_multiphoton_count()
total_photons = n_photons + multi_photons
if False and n_photons < 0: # Generally, do not mask negative values; if fit is still OK
print "\n%d pixel %s altrn %d photons from curvefitting" % (
i, pixel, n_photons)
pixel_histograms.plot_combo(
pixel,
alt_gaussians,
interpretation=fit_interpretation)
mask[
pixel] = 1 # do not mask out negative pixels if the Gaussian fit is good
continue
chi_squared_list.append(fit_interpretation.chi_squared())
suspect = False # don't know the optimal statistical test. Histograms vary primarily by total count & # photons
if total_photons <= 3:
if fit_interpretation.chi_squared(
) > 2.5 or fit_interpretation.quality_factor < 5:
suspect = True
elif 3 < total_photons <= 10:
if fit_interpretation.chi_squared(
) > 5 or fit_interpretation.quality_factor < 10:
suspect = True
elif 10 < total_photons <= 33:
if fit_interpretation.chi_squared(
) > 10 or fit_interpretation.quality_factor < 20:
suspect = True
elif 33 < total_photons <= 100:
if fit_interpretation.chi_squared(
) > 20 or fit_interpretation.quality_factor < 20:
suspect = True
elif 100 < total_photons <= 330:
if fit_interpretation.chi_squared(
) > 30 or fit_interpretation.quality_factor < 25:
suspect = True
elif 330 < total_photons <= 1000:
if fit_interpretation.chi_squared(
) > 40 or fit_interpretation.quality_factor < 30:
suspect = True
elif 1000 < total_photons:
if fit_interpretation.chi_squared(
) > 50 or fit_interpretation.quality_factor < 30:
suspect = True
if suspect:
print "\n%d pixel %s Bad quality 0/1-photon fit" % (
i, pixel), fit_interpretation.quality_factor
print " with chi-squared %10.5f" % fit_interpretation.chi_squared(
)
print " Suspect", suspect
print "%d fit photons, %d total photons" % (n_photons,
total_photons)
#pixel_histograms.plot_combo(pixel, alt_gaussians,
# interpretation=fit_interpretation)
mask[pixel] = 1
continue
self.sum_img[pixel] = n_photons + multi_photons
mask.set_selected(self.sum_img == 0, 1)
unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
mask.set_selected(unbound_pixel_mask > 0, 1)
bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
mask.set_selected(bad_pixel_mask > 0, 1)
for row in range(self.sum_img.all()[0]):
self.sum_img[row:row + 1, :].count(0)
spectrum_focus = self.sum_img[start_row:end_row, :]
mask_focus = mask[start_row:end_row, :]
spectrum_focus.set_selected(mask_focus > 0, 0)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print "Number of rows: %i" % spectrum_focus.all()[0]
print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus)
print "Number of images used: %i" % flex.sum(
pixel_histograms.histograms.values()[0].slots())
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr = "_%04d" % run
else: runstr = ""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)
plot_x, plot_y = xes_finalise.output_spectrum(
spectrum_focus.iround(),
mask_focus=mask_focus,
output_dirname=output_dirname,
run=run)
self.spectrum = (plot_x, plot_y)
self.spectrum_focus = spectrum_focus
xes_finalise.output_matlab_form(
spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
print output_dirname
print "Average chi squared is", flex.mean(
chi_squared_list), "on %d shots" % flex.sum(hist.slots())
def run(argv=None):
import libtbx.option_parser
if (argv is None):
argv = sys.argv
command_line = (libtbx.option_parser.option_parser(
usage=
"%s [-v] [-p poly_mask] [-c circle_mask] [-a avg_max] [-s stddev_max] [-m maxproj_min] [-x mask_pix_val] [-o output] avg_path stddev_path max_path"
% libtbx.env.dispatcher_name
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--poly_mask",
"-p",
type="string",
default=None,
dest="poly_mask",
help="Polygon to mask out. Comma-seperated string of xy pairs."
).option(
None,
"--circle_mask",
"-c",
type="string",
default=None,
dest="circle_mask",
help="Circle to mask out. Comma-seperated string of x, y, and radius."
).option(
None,
"--avg_max",
"-a",
type="float",
default=2000.0,
dest="avg_max",
help=
"Maximum ADU that pixels in the average image are allowed to have before masked out"
).option(
None,
"--stddev_max",
"-s",
type="float",
default=10.0,
dest="stddev_max",
help=
"Maximum ADU that pixels in the standard deviation image are allowed to have before masked out"
).option(
None,
"--maxproj_min",
"-m",
type="float",
default=300.0,
dest="maxproj_min",
help=
"Minimum ADU that pixels in the maximum projection image are allowed to have before masked out"
).option(None,
"--mask_pix_val",
"-x",
type="int",
default=-2,
dest="mask_pix_val",
help="Value for masked out pixels").option(
None,
"--detector_format_version",
"-d",
type="string",
default=None,
dest="detector_format_version",
help="detector format version string").option(
None,
"--output",
"-o",
type="string",
default="mask_.pickle",
dest="destpath",
help="output file path, should be *.pickle")).process(
args=argv[1:])
# Must have exactly three remaining arguments.
paths = command_line.args
if (len(paths) != 3):
command_line.parser.print_usage(file=sys.stderr)
return
if command_line.options.detector_format_version is None:
address = timestamp = None
else:
from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp
from iotbx.detectors.cspad_detector_formats import address_and_timestamp_from_detector_format_version
address, timestamp = address_and_timestamp_from_detector_format_version(
command_line.options.detector_format_version)
timestamp = evt_timestamp((timestamp, 0))
poly_mask = None
if not command_line.options.poly_mask == None:
poly_mask = []
poly_mask_tmp = command_line.options.poly_mask.split(",")
if len(poly_mask_tmp) % 2 != 0:
command_line.parser.print_usage(file=sys.stderr)
return
odd = True
for item in poly_mask_tmp:
try:
if odd:
poly_mask.append(int(item))
else:
poly_mask[-1] = (poly_mask[-1], int(item))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
odd = not odd
circle_mask = None
if command_line.options.circle_mask is not None:
circle_mask_tmp = command_line.options.circle_mask.split(",")
if len(circle_mask_tmp) != 3:
command_line.parser.print_usage(file=sys.stderr)
return
try:
circle_mask = (int(circle_mask_tmp[0]), int(circle_mask_tmp[1]),
int(circle_mask_tmp[2]))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
avg_path = paths[0]
stddev_path = paths[1]
max_path = paths[2]
# load the three images
format_class = Registry.find(avg_path)
avg_f = format_class(avg_path)
avg_i = avg_f.get_detectorbase()
avg_d = avg_i.get_raw_data()
stddev_f = format_class(stddev_path)
stddev_i = stddev_f.get_detectorbase()
stddev_d = stddev_i.get_raw_data()
max_f = format_class(max_path)
max_i = max_f.get_detectorbase()
max_d = max_i.get_raw_data()
# first find all the pixels in the average that are less than zero or greater
# than a cutoff and set them to the masking value
avg_d.set_selected((avg_d <= 0) | (avg_d > command_line.options.avg_max),
command_line.options.mask_pix_val)
# set all the rest of the pixels to zero. They will be accepted
avg_d.set_selected(avg_d != command_line.options.mask_pix_val, 0)
# mask out the overly noisy or flat pixels
avg_d.set_selected(stddev_d <= 0, command_line.options.mask_pix_val)
avg_d.set_selected(stddev_d >= command_line.options.stddev_max,
command_line.options.mask_pix_val)
# these are the non-bonded pixels
avg_d.set_selected(max_d < command_line.options.maxproj_min,
command_line.options.mask_pix_val)
# calculate the beam center
panel = avg_f.get_detector()[0]
bcx, bcy = panel.get_beam_centre(avg_f.get_beam().get_s0())
if poly_mask is not None or circle_mask is not None:
minx = miny = 0
maxx = avg_d.focus()[0]
maxy = avg_d.focus()[1]
if poly_mask is not None:
minx = min([x[0] for x in poly_mask])
miny = min([y[1] for y in poly_mask])
maxx = max([x[0] for x in poly_mask])
maxy = max([y[1] for y in poly_mask])
if circle_mask is not None:
circle_x, circle_y, radius = circle_mask
if circle_x - radius < minx: minx = circle_x - radius
if circle_y - radius < miny: miny = circle_y - radius
if circle_x + radius > maxx: maxx = circle_x + radius
if circle_y + radius > maxy: maxy = circle_y + radius
sel = avg_d == command_line.options.mask_pix_val
for j in range(miny, maxy):
for i in range(minx, maxx):
idx = j * avg_d.focus()[0] + i
if not sel[idx]:
if poly_mask is not None and point_in_polygon(
(i, j), poly_mask):
sel[idx] = True
elif circle_mask is not None and point_inside_circle(
i, j, circle_x, circle_y, radius):
sel[idx] = True
avg_d.set_selected(sel, command_line.options.mask_pix_val)
# have to re-layout the data to match how it was stored originally
shifted_int_data_old = avg_d
shifted_int_data_new = shifted_int_data_old.__class__(
flex.grid(shifted_int_data_old.focus()))
shifted_int_data_new += command_line.options.mask_pix_val
phil = avg_i.horizons_phil_cache
manager = avg_i.get_tile_manager(phil)
for i, shift in enumerate(manager.effective_translations()):
shift_slow = shift[0]
shift_fast = shift[1]
ur_slow = phil.distl.detector_tiling[4 * i + 0] + shift_slow
ur_fast = phil.distl.detector_tiling[4 * i + 1] + shift_fast
ll_slow = phil.distl.detector_tiling[4 * i + 2] + shift_slow
ll_fast = phil.distl.detector_tiling[4 * i + 3] + shift_fast
#print "Shifting tile at (%d, %d) by (%d, %d)" % (ur_slow-shift_slow, ur_fast-shift_fast, -shift_slow, -shift_fast)
shifted_int_data_new.matrix_paste_block_in_place(
block=shifted_int_data_old.matrix_copy_block(
i_row=ur_slow,
i_column=ur_fast,
n_rows=ll_slow - ur_slow,
n_columns=ll_fast - ur_fast),
i_row=ur_slow - shift_slow,
i_column=ur_fast - shift_fast)
d = dpack(active_areas=avg_i.parameters['ACTIVE_AREAS'],
address=address,
beam_center_x=bcx,
beam_center_y=bcy,
data=shifted_int_data_new,
distance=avg_i.distance,
timestamp=timestamp,
wavelength=avg_i.wavelength,
xtal_target=None,
pixel_size=avg_i.pixel_size,
saturated_value=avg_i.saturation)
dwritef2(d, command_line.options.destpath)
#the minimum number of pixels to mask out cooresponding to the interstitial regions for the CS-PAD
min_count = 818265 # (1765 * 1765) - (194 * 185 * 64)
masked_out = len(avg_d.as_1d().select(
(avg_d == command_line.options.mask_pix_val).as_1d()))
assert masked_out >= min_count
print "Masked out %d pixels out of %d (%.2f%%)"% \
(masked_out-min_count,len(avg_d)-min_count,(masked_out-min_count)*100/(len(avg_d)-min_count))
class SingleImage(object):
def __init__(self, img, init, verbose=True, imported_grid=None):
""" Constructor for the SingleImage object using a raw image file or pickle
"""
# Initialize parameters
self.params = init.params
self.args = init.args
self.raw_img = img[2]
self.conv_img = img[2]
self.img_index = img[0]
self.status = None
self.fail = None
self.final = None
self.log_info = []
self.gs_results = []
self.main_log = init.logfile
self.verbose = verbose
self.hmed = self.params.cctbx.grid_search.height_median
self.amed = self.params.cctbx.grid_search.area_median
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.viz_base = init.viz_base
self.tmp_base = init.tmp_base
self.abort_file = os.path.join(self.int_base, '.abort.tmp')
self.obj_path = None
self.obj_file = None
self.fin_path = None
self.fin_file = None
self.viz_path = None
# ============================== SELECTION-ONLY FUNCTIONS ============================== #
def import_int_file(self, init):
""" Replaces path settings in imported image object with new settings
NEED TO RE-DO LATER """
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Generate paths to output files
self.params = init.params
self.main_log = init.logfile
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.viz_base = init.viz_base
self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
self.obj_base)
self.obj_file = os.path.abspath(
os.path.join(
self.obj_path,
os.path.basename(self.conv_img).split('.')[0] + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base,
self.fin_base)
self.fin_file = os.path.abspath(
os.path.join(
self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.pickle"))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.viz_path = misc.make_image_path(self.conv_img, self.input_base,
self.viz_base)
self.viz_file = os.path.join(
self.viz_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.png")
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Grid search / integration log file
self.int_log = os.path.join(
self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
# Reset status to 'grid search' to pick up at selection (if no fail)
if self.fail == None:
self.status = 'bypass grid search'
return self
def determine_gs_result_file(self):
""" For 'selection-only' cctbx.xfel runs, determine where the image objects are """
if self.params.cctbx.selection.select_only.grid_search_path != None:
obj_path = os.path.abspath(
self.params.cctbx.selection.select_only.grid_search_path)
else:
run_number = int(os.path.basename(self.int_base)) - 1
obj_path = "{}/integration/{:03d}/image_objects"\
"".format(os.path.abspath(os.curdir), run_number)
gs_result_file = os.path.join(obj_path,
os.path.basename(self.obj_file))
return gs_result_file
# =============================== IMAGE IMPORT FUNCTIONS =============================== #
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except IOError, e:
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
img_type = 'pickle'
else:
img_type = 'raw'
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
else:
data = None
img_type = 'not imported'
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
try:
from dials.algorithms.image.threshold import KabschDebug
raw_data = [raw_data]
gain_value = 1
kernel_size = (10, 10)
gain_map = [
flex.double(raw_data[i].accessor(), gain_value)
for i in range(len(loaded_img.get_detector()))
]
mask = loaded_img.get_mask()
min_local = 0
# dummy values, shouldn't affect results: REPLACE WITH SETTINGS!
nsigma_b = 6
nsigma_s = 3
global_threshold = 0
kabsch_debug_list = []
for i_panel in range(len(loaded_img.get_detector())):
kabsch_debug_list.append(
KabschDebug(raw_data[i_panel].as_double(),
mask[i_panel], gain_map[i_panel],
kernel_size, nsigma_b, nsigma_s,
global_threshold, min_local))
dispersion = flex.double()
for kabsch in kabsch_debug_list:
dispersion.extend(
kabsch.coefficient_of_variation().as_1d())
sorted_dispersion = flex.sorted(dispersion)
from libtbx.math_utils import nearest_integer as nint
q1 = sorted_dispersion[nint(len(sorted_dispersion) / 4)]
q2 = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
q3 = sorted_dispersion[nint(len(sorted_dispersion) * 3 / 4)]
iqr = q3 - q1
inlier_sel = (sorted_dispersion >
(q1 - 1.5 * iqr)) & (sorted_dispersion <
(q3 + 1.5 * iqr))
sorted_dispersion = sorted_dispersion.select(inlier_sel)
self.gain = sorted_dispersion[nint(len(sorted_dispersion) / 2)]
except IndexError:
self.gain = 1.0
else:
self.gain = 1.0
return data, img_type
def run(argv=None):
import libtbx.option_parser
if (argv is None):
argv = sys.argv
command_line = (libtbx.option_parser.option_parser(
usage="%s [-v] [-p poly_mask] [-c circle_mask] [-a avg_max] [-s stddev_max] [-m maxproj_min] [-x mask_pix_val] [-o output] avg_path stddev_path max_path" % libtbx.env.dispatcher_name)
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--poly_mask", "-p",
type="string",
default=None,
dest="poly_mask",
help="Polygon to mask out. Comma-seperated string of xy pairs.")
.option(None, "--circle_mask", "-c",
type="string",
default=None,
dest="circle_mask",
help="Circle to mask out. Comma-seperated string of x, y, and radius.")
.option(None, "--avg_max", "-a",
type="float",
default=2000.0,
dest="avg_max",
help="Maximum ADU that pixels in the average image are allowed to have before masked out")
.option(None, "--stddev_max", "-s",
type="float",
default=10.0,
dest="stddev_max",
help="Maximum ADU that pixels in the standard deviation image are allowed to have before masked out")
.option(None, "--maxproj_min", "-m",
type="float",
default=300.0,
dest="maxproj_min",
help="Minimum ADU that pixels in the maximum projection image are allowed to have before masked out")
.option(None, "--mask_pix_val", "-x",
type="int",
default=-2,
dest="mask_pix_val",
help="Value for masked out pixels")
.option(None, "--detector_format_version", "-d",
type="string",
default=None,
dest="detector_format_version",
help="detector format version string")
.option(None, "--output", "-o",
type="string",
default="mask_.pickle",
dest="destpath",
help="output file path, should be *.pickle")
).process(args=argv[1:])
# Must have exactly three remaining arguments.
paths = command_line.args
if (len(paths) != 3):
command_line.parser.print_usage(file=sys.stderr)
return
if command_line.options.detector_format_version is None:
address = timestamp = None
else:
from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp
from xfel.detector_formats import address_and_timestamp_from_detector_format_version
address, timestamp = address_and_timestamp_from_detector_format_version(command_line.options.detector_format_version)
timestamp = evt_timestamp((timestamp,0))
poly_mask = None
if not command_line.options.poly_mask == None:
poly_mask = []
poly_mask_tmp = command_line.options.poly_mask.split(",")
if len(poly_mask_tmp) % 2 != 0:
command_line.parser.print_usage(file=sys.stderr)
return
odd = True
for item in poly_mask_tmp:
try:
if odd:
poly_mask.append(int(item))
else:
poly_mask[-1] = (poly_mask[-1],int(item))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
odd = not odd
circle_mask = None
if command_line.options.circle_mask is not None:
circle_mask_tmp = command_line.options.circle_mask.split(",")
if len(circle_mask_tmp) != 3:
command_line.parser.print_usage(file=sys.stderr)
return
try:
circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2]))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
avg_path = paths[0]
stddev_path = paths[1]
max_path = paths[2]
# load the three images
format_class = Registry.find(avg_path)
avg_f = format_class(avg_path)
avg_i = avg_f.get_detectorbase()
avg_d = avg_i.get_raw_data()
stddev_f = format_class(stddev_path)
stddev_i = stddev_f.get_detectorbase()
stddev_d = stddev_i.get_raw_data()
max_f = format_class(max_path)
max_i = max_f.get_detectorbase()
max_d = max_i.get_raw_data()
# first find all the pixels in the average that are less than zero or greater
# than a cutoff and set them to the masking value
avg_d.set_selected((avg_d <= 0) | (avg_d > command_line.options.avg_max), command_line.options.mask_pix_val)
# set all the rest of the pixels to zero. They will be accepted
avg_d.set_selected(avg_d != command_line.options.mask_pix_val, 0)
# mask out the overly noisy or flat pixels
avg_d.set_selected(stddev_d <= 0, command_line.options.mask_pix_val)
avg_d.set_selected(stddev_d >= command_line.options.stddev_max, command_line.options.mask_pix_val)
# these are the non-bonded pixels
avg_d.set_selected(max_d < command_line.options.maxproj_min, command_line.options.mask_pix_val)
# calculate the beam center
panel = avg_f.get_detector()[0]
bcx, bcy = panel.get_beam_centre(avg_f.get_beam().get_s0())
if poly_mask is not None or circle_mask is not None:
minx = miny = 0
maxx = avg_d.focus()[0]
maxy = avg_d.focus()[1]
if poly_mask is not None:
minx = min([x[0] for x in poly_mask])
miny = min([y[1] for y in poly_mask])
maxx = max([x[0] for x in poly_mask])
maxy = max([y[1] for y in poly_mask])
if circle_mask is not None:
circle_x, circle_y, radius = circle_mask
if circle_x - radius < minx: minx = circle_x - radius
if circle_y - radius < miny: miny = circle_y - radius
if circle_x + radius > maxx: maxx = circle_x + radius
if circle_y + radius > maxy: maxy = circle_y + radius
sel = avg_d == command_line.options.mask_pix_val
for j in xrange(miny, maxy):
for i in xrange(minx, maxx):
idx = j * avg_d.focus()[0] + i
if not sel[idx]:
if poly_mask is not None and point_in_polygon((i,j),poly_mask):
sel[idx] = True
elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius):
sel[idx] = True
avg_d.set_selected(sel,command_line.options.mask_pix_val)
# have to re-layout the data to match how it was stored originally
shifted_int_data_old = avg_d
shifted_int_data_new = shifted_int_data_old.__class__(
flex.grid(shifted_int_data_old.focus()))
shifted_int_data_new += command_line.options.mask_pix_val
phil = avg_i.horizons_phil_cache
manager = avg_i.get_tile_manager(phil)
for i,shift in enumerate(manager.effective_translations()):
shift_slow = shift[0]
shift_fast = shift[1]
ur_slow = phil.distl.detector_tiling[4 * i + 0] + shift_slow
ur_fast = phil.distl.detector_tiling[4 * i + 1] + shift_fast
ll_slow = phil.distl.detector_tiling[4 * i + 2] + shift_slow
ll_fast = phil.distl.detector_tiling[4 * i + 3] + shift_fast
#print "Shifting tile at (%d, %d) by (%d, %d)" % (ur_slow-shift_slow, ur_fast-shift_fast, -shift_slow, -shift_fast)
shifted_int_data_new.matrix_paste_block_in_place(
block = shifted_int_data_old.matrix_copy_block(
i_row=ur_slow,i_column=ur_fast,
n_rows=ll_slow-ur_slow, n_columns=ll_fast-ur_fast),
i_row = ur_slow - shift_slow,
i_column = ur_fast - shift_fast
)
d = dpack(
active_areas=avg_i.parameters['ACTIVE_AREAS'],
address=address,
beam_center_x=bcx,
beam_center_y=bcy,
data=shifted_int_data_new,
distance=avg_i.distance,
timestamp=timestamp,
wavelength=avg_i.wavelength,
xtal_target=None,
pixel_size=avg_i.pixel_size,
saturated_value=avg_i.saturation)
dwritef2(d, command_line.options.destpath)
#the minimum number of pixels to mask out cooresponding to the interstitial regions for the CS-PAD
min_count = 818265 # (1765 * 1765) - (194 * 185 * 64)
masked_out = len(avg_d.as_1d().select((avg_d == command_line.options.mask_pix_val).as_1d()))
assert masked_out >= min_count
print "Masked out %d pixels out of %d (%.2f%%)"% \
(masked_out-min_count,len(avg_d)-min_count,(masked_out-min_count)*100/(len(avg_d)-min_count))
break
except KeyError, ValueError:
n_fails += 1
print "Fail to read, attempt number", n_fails
if n_fails > 100:
raise Exception("Couldn't read the data")
import time
time.sleep(n_fails * 0.1)
imgdict = cspad_tbx.dpack(
data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
address="Sacla.MPCCD.8tile",
active_areas=active_areas,
)
imgdict = crop_image_pickle(imgdict, preserve_active_areas_even_though_cropping_would_invalidate_them=True)
dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no)
print "Saving image", img_no, "to", dest_path
easy_pickle.dump(dest_path, imgdict)
easy_mp.pool_map(args=range(data.get_num_images()), func=do_work, processes=4)
quad_asics.append(numpy.concatenate((a,b),axis=1))
quad_data = numpy.dstack(quad_asics)
quad_data = numpy.rollaxis(quad_data, 2,0)
data3d.append(fake_cspad_ElementV2(quad_data, i_quad))
env = fake_env(fake_config())
evt = fake_evt(data3d)
beam_center, active_areas = cbcaa(fake_config(),sections)
data = flex.int(CsPadDetector(address, evt, env, sections).astype(numpy.float64))
img_dict = dpack(
active_areas=active_areas,
address=address,
beam_center_x=beam_center[0]*pixel_size,
beam_center_y=beam_center[1]*pixel_size,
data=data,
distance=params.distance,
pixel_size=pixel_size,
timestamp=timestamp,
wavelength=params.wavelength)
img = NpyImage("", source_data=img_dict)
args = ["distl.detector_format_version=%s"%params.detector_format_version]
horizons_phil = cxi_phil.cxi_versioned_extract(args)
img.readHeader(horizons_phil)
img.translate_tiles(horizons_phil)
tm = img.get_tile_manager(horizons_phil)
effective_active_areas = tm.effective_tiling_as_flex_int()
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights,outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info("Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(d,
integration_dirname = self.m_integration_dirname,
integration_basename = self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None
if indexed and self.m_progress_logging:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best["identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best["identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
if self.m_db_logging:
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead")
if (self.m_out_dirname is not None or
self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(average_mixin, self).event(evt, env)
if evt.get('skip_event'):
return
# Get the distance for the detectors that should have it, and set
# it to NaN for those that should not.
if self.detector == 'CxiDs1' or \
self.detector == 'CxiDs2' or \
self.detector == 'CxiDsd' or \
self.detector == 'XppGon':
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self._nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
else:
distance = float('nan')
if ("skew" in self.flags):
# Take out inactive pixels
if self.roi is not None:
pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
pixels = pixels.as_1d().select(dark_mask.as_1d())
else:
pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double()
stats = scitbx.math.basic_statistics(pixels.as_double())
#stats.show()
self.logger.info("skew: %.3f" %stats.skew)
self.logger.info("kurtosis: %.3f" %stats.kurtosis)
if 0:
from matplotlib import pyplot
hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double())
print hist_min, hist_max
n_slots = 100
n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
pyplot.show()
# XXX This skew threshold probably needs fine-tuning
skew_threshold = 0.35
if stats.skew < skew_threshold:
self._nfail += 1
self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold)
evt.put(skip_event_flag(), 'skip_event')
return
#self.cspad_img *= stats.skew
if ("inactive" in self.flags):
self.cspad_img.set_selected(self.dark_stddev <= 0, 0)
if ("noelastic" in self.flags):
ELASTIC_THRESHOLD = self.elastic_threshold
self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0)
if self.hot_threshold is not None:
HOT_THRESHOLD = self.hot_threshold
self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0)
if self.gain_map is not None and self.gain_threshold is not None:
# XXX comparing each pixel to a moving average would probably be better
# since the gain should vary approximately smoothly over different areas
# of the detector
GAIN_THRESHOLD = self.gain_threshold
#self.logger.debug(
#"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True))
self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0)
if ("nonoise" in self.flags):
NOISE_THRESHOLD = self.noise_threshold
self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0)
if ("sigma_scaling" in self.flags):
self.do_sigma_scaling()
if ("symnoise" in self.flags):
SYMNOISE_THRESHOLD = self.symnoise_threshold
self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) &
( self.cspad_img < SYMNOISE_THRESHOLD), 0)
if ("output" in self.flags):
try:
import cPickle as pickle
except ImportError:
import pickle
import os
if (not os.path.isdir(self.pickle_dirname)):
os.makedirs(self.pickle_dirname)
flexdata = flex.int(self.cspad_img.astype(numpy.int32))
d = cspad_tbx.dpack(
address=self.address,
data=flexdata,
timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
)
G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename,
"ab")
pickle.dump(d,G,pickle.HIGHEST_PROTOCOL)
G.close()
if self.photon_threshold is not None and self.two_photon_threshold is not None:
self.do_photon_counting()
if self.background_path is not None:
self.cspad_img -= self.background_img
# t and self._sum_time are a two-long arrays of seconds and
# milliseconds which hold time with respect to the base time.
t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt),
self._metadata['time_base'])]
if self._nmemb == 0:
# The peers metadata item is a bit field where a bit is set if
# the partial sum from the corresponding worker process is
# pending. If this is the first frame a worker process sees,
# set its corresponding bit in the bit field since it will
# contribute a partial sum.
if env.subprocess() >= 0:
self._lock.acquire()
if 'peers' in self._metadata.keys():
self._metadata['peers'] |= (1 << env.subprocess())
else:
self._metadata['peers'] = (1 << env.subprocess())
self._lock.release()
self._sum_distance = distance
self._sum_time = (t[0], t[1])
self._sum_wavelength = self.wavelength
if self._have_max:
self._max_img = self.cspad_img.deep_copy()
if self._have_mean:
self._sum_img = self.cspad_img.deep_copy()
if self._have_std:
self._ssq_img = flex.pow2(self.cspad_img)
else:
self._sum_distance += distance
self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1])
self._sum_wavelength += self.wavelength
if self._have_max:
sel = (self.cspad_img > self._max_img).as_1d()
self._max_img.as_1d().set_selected(
sel, self.cspad_img.as_1d().select(sel))
if self._have_mean:
self._sum_img += self.cspad_img
if self._have_std:
self._ssq_img += flex.pow2(self.cspad_img)
self._nmemb += 1
try:
raw_data = data.get_raw_data(img_no)
break
except KeyError, ValueError:
n_fails += 1
print "Fail to read, attempt number", n_fails
if n_fails > 100:
raise Exception("Couldn't read the data")
import time
time.sleep(n_fails * 0.1)
imgdict = cspad_tbx.dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
address="Sacla.MPCCD.8tile",
active_areas=active_areas)
imgdict = crop_image_pickle(
imgdict,
preserve_active_areas_even_though_cropping_would_invalidate_them=True)
dest_path = os.path.join(dest_dir, dest_base + "_%06d.pickle" % img_no)
print "Saving image", img_no, "to", dest_path
easy_pickle.dump(dest_path, imgdict)
easy_mp.pool_map(args=range(data.get_num_images()), func=do_work, processes=4)
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except IOError:
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
if abs(beam_x - beam_y) <= 0.1 or self.params.image_conversion.square_mode == "None":
img_type = 'converted'
else:
img_type = 'unconverted'
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec,msec))
if self.params.image_conversion.beamstop != 0 or\
self.params.image_conversion.beam_center.x != 0 or\
self.params.image_conversion.beam_center.y != 0 or\
self.params.image_conversion.rename_pickle_prefix != 'Auto' or\
self.params.image_conversion.rename_pickle_prefix != None:
img_type = 'unconverted'
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp
)
#print "data: ", type(raw_data)
#print "pixel size: ", type(pixel_size)
#print 'wavelength: ', type(wavelength)
#print "beamX: ", type(beam_x)
#print "saturation: ", type(overload)
#print "timestamp: ", type(timestamp)
#for i in dir(raw_data): print i
#exit()
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
img_type = 'unconverted'
if osc_start != osc_range:
data['OSC_START'] = osc_start
data['OSC_RANGE'] = osc_range
data['TIME'] = scan.get_exposure_times()[0]
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
try:
from dials.algorithms.image.threshold import KabschDebug
raw_data = [raw_data]
gain_value = 1
kernel_size=(10,10)
gain_map = [flex.double(raw_data[i].accessor(), gain_value)
for i in range(len(loaded_img.get_detector()))]
mask = loaded_img.get_mask()
min_local = 0
# dummy values, shouldn't affect results
nsigma_b = 6
nsigma_s = 3
global_threshold = 0
kabsch_debug_list = []
for i_panel in range(len(loaded_img.get_detector())):
kabsch_debug_list.append(
KabschDebug(
raw_data[i_panel].as_double(), mask[i_panel], gain_map[i_panel],
kernel_size, nsigma_b, nsigma_s, global_threshold, min_local))
dispersion = flex.double()
for kabsch in kabsch_debug_list:
dispersion.extend(kabsch.coefficient_of_variation().as_1d())
sorted_dispersion = flex.sorted(dispersion)
from libtbx.math_utils import nearest_integer as nint
q1 = sorted_dispersion[nint(len(sorted_dispersion)/4)]
q2 = sorted_dispersion[nint(len(sorted_dispersion)/2)]
q3 = sorted_dispersion[nint(len(sorted_dispersion)*3/4)]
iqr = q3-q1
inlier_sel = (sorted_dispersion > (q1 - 1.5*iqr)) & (sorted_dispersion < (q3 + 1.5*iqr))
sorted_dispersion = sorted_dispersion.select(inlier_sel)
self.gain = sorted_dispersion[nint(len(sorted_dispersion)/2)]
except IndexError:
self.gain = 1.0
else:
self.gain = 1.0
else:
data = None
return data, img_type
def run(argv=None):
"""Compute mean, standard deviation, and maximum projection images
from a set of images given on the command line.
@param argv Command line argument list
@return @c 0 on successful termination, @c 1 on error, and @c 2
for command line syntax errors
"""
import libtbx.load_env
from libtbx import easy_pickle, option_parser
from scitbx.array_family import flex
from xfel.cxi.cspad_ana import cspad_tbx
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
if argv is None:
argv = sys.argv
command_line = (option_parser.option_parser(
usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
"image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
.option(None, "--average-path", "-a",
type="string",
default=None,
dest="avg_path",
metavar="PATH",
help="Write average image to PATH")
.option(None, "--maximum-path", "-m",
type="string",
default=None,
dest="max_path",
metavar="PATH",
help="Write maximum projection image to PATH")
.option(None, "--stddev-path", "-s",
type="string",
default=None,
dest="stddev_path",
metavar="PATH",
help="Write standard deviation image to PATH")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
).process(args=argv[1:])
# Note that it is not an error to omit the output paths, because
# certain statistics could still be printed, e.g. with the verbose
# option.
paths = command_line.args
if len(paths) == 0:
command_line.parser.print_usage(file=sys.stderr)
return 2
# Loop over all images and accumulate statistics.
nfail = 0
nmemb = 0
for path in paths:
if command_line.options.verbose:
sys.stdout.write("Processing %s...\n" % path)
try:
# Promote the image to double-precision floating point type.
# All real-valued flex arrays have the as_double() function.
d = easy_pickle.load(path)
distance = d['DISTANCE']
img = d['DATA'].as_1d().as_double()
wavelength = d['WAVELENGTH']
time_tuple = reverse_timestamp(d['TIMESTAMP'])
# Warn if the header items across the set of images do not match
# up. Note that discrepancies regarding the image size are
# fatal.
if 'active_areas' in locals():
if (active_areas != d['ACTIVE_AREAS']).count(True) != 0:
sys.stderr.write("Active areas do not match\n")
else:
active_areas = d['ACTIVE_AREAS']
if 'beam_center' in locals():
if beam_center != (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']):
sys.stderr.write("Beam centers do not match\n")
else:
beam_center = (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y'])
if 'detector_address' in locals():
if detector_address != d['DETECTOR_ADDRESS']:
sys.stderr.write("Detector addresses do not match\n")
else:
detector_address = d['DETECTOR_ADDRESS']
if 'saturated_value' in locals():
if saturated_value != d['SATURATED_VALUE']:
sys.stderr.write("Saturated values do not match\n")
else:
saturated_value = d['SATURATED_VALUE']
if 'size' in locals():
if size != (d['SIZE1'], d['SIZE2']):
sys.stderr.write("Image sizes do not match\n")
return 1
else:
size = (d['SIZE1'], d['SIZE2'])
if size != d['DATA'].focus():
sys.stderr.write("Image size does not match pixel array\n")
return 1
if 'pixel_size' in locals():
if pixel_size != d['PIXEL_SIZE']:
sys.stderr.write("Pixel sizes do not match\n")
return 1
else:
if 'PIXEL_SIZE' in d:
pixel_size = d['PIXEL_SIZE']
else:
pixel_size = None
except Exception:
try:
# Fall back on reading the image with dxtbx, and shoehorn the
# extracted information into what would have been found in a
# pickle file. XXX This code assumes a monolithic detector!
from dxtbx.format.Registry import Registry
format_class = Registry.find(path)
i = format_class(path)
beam = i.get_beam()
assert len(i.get_detector()) == 1
detector = i.get_detector()[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = i.get_raw_data().as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
time_tuple = (i.get_scan().get_epochs()[0], 0)
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
except Exception:
nfail += 1
continue
# See also event() in xfel.cxi.cspad_ana.average_tbx. Record the
# base time as the timestamp of the first image.
#
# The sum-of-squares image is accumulated using long integers, as
# this delays the point where overflow occurs. But really, this
# is just a band-aid...
if nmemb == 0:
max_img = img.deep_copy()
sum_distance = distance
sum_img = img.deep_copy()
ssq_img = flex.pow2(img)
sum_wavelength = wavelength
sum_time = (0, 0)
time_base = time_tuple
else:
sel = (img > max_img).as_1d()
max_img.set_selected(sel, img.select(sel))
sum_distance += distance
sum_img += img
ssq_img += flex.pow2(img)
sum_wavelength += wavelength
sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]),
sum_time[1] + (time_tuple[1] - time_base[1]))
nmemb += 1
# Early exit if no statistics were accumulated.
if command_line.options.verbose:
sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
if nmemb == 0:
return 0
# Calculate averages for measures where other statistics do not make
# sense. Note that avg_img is required for stddev_img.
avg_img = sum_img.as_double() / nmemb
avg_distance = sum_distance / nmemb
avg_timestamp = cspad_tbx.evt_timestamp(
(time_base[0] + int(round(sum_time[0] / nmemb)),
time_base[1] + int(round(sum_time[1] / nmemb))))
avg_wavelength = sum_wavelength / nmemb
# Output the average image, maximum projection image, and standard
# deviation image, if requested.
if command_line.options.avg_path is not None:
avg_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=avg_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.avg_path, d)
if command_line.options.max_path is not None:
max_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=max_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.max_path, d)
if command_line.options.stddev_path is not None:
stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img
# Accumulating floating-point numbers introduces errors, which may
# cause negative variances. Since a two-pass approach is
# unacceptable, the standard deviation is clamped at zero.
stddev_img.set_selected(stddev_img < 0, 0)
if nmemb == 1:
stddev_img = flex.sqrt(stddev_img)
else:
stddev_img = flex.sqrt(stddev_img / (nmemb - 1))
stddev_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=stddev_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.stddev_path, d)
return 0
def HitFinder(IO,XSetup,HFParams,Frelon,DataCorr,AI,index):
hit=0
fname=IO.fname_list[index]
peaks=[]
peakslist=[]
try :
img = fabio.open(fname)
except:
print 'Warning : problem while opening file %s, file skiped '%fname
return
if img.data.shape == Frelon.resolution:
#Apply the dark, flatfield and distortion correction (as specified by the user)
img.data=DataCorr.apply_correction(img.data,HFParams.DoDarkCorr,HFParams.DoFlatCorr,HFParams.DoDist)
#Remove beam stop area (i.e = 0)
extend=15
#BkgCorr with pyFAI Azimuthal Integrator
working=AI.ai.separate(img.data,npt_rad=1024, npt_azim=512, unit="2th_deg",percentile=50, mask=AI.mask,restore_mask=False)[0]
imgmax,imgmin,imgmed = np.max(working), np.min(working), np.median(working)
working[XSetup.beam_y-extend:XSetup.beam_y+extend,XSetup.beam_x-extend:XSetup.beam_x+extend]=0
# Get number of peaks above threshold in the current frame
cropped=working[20:Frelon.resolution[1]-20,20:Frelon.resolution[1]-20]
#print cropped.shape
peaks=cropped[np.where(cropped>float(HFParams.threshold))]
#peaks=working[np.where(working[20:Frelon.resolution[1]-20][20:Frelon.resolution[0]-20]>float(HFParams.threshold))]
#peaks=working[np.where(working>float(HFParams.threshold))]# If enough peaks in current frame - This is a hit - Save it !!
if len(peaks) >= HFParams.npixels:
hit = 1
root=os.path.basename(fname)
root=os.path.splitext(root)[0]
if HFParams.DoPeakSearch:
local_max=pf.find_local_max(working[0:1023,0:1004].astype(np.float),d_rad=1,threshold=HFParams.threshold)
local_max_crop=pf.local_max_crop(working[0:1023,0:1004].astype(np.float), local_max, 3)
peakslist=np.array(pf.subpixel_centroid(working[0:1023,0:1004],local_max_crop,3))
if IO.edf:
OutputFileName =os.path.join(IO.procdir, IO.EDFDir,"%s.edf" %root)
img.data = working
img.write(OutputFileName)
#Conversion to H5
if IO.H5:
OutputFileName =os.path.join(IO.procdir, IO.H5Dir,"%s.h5" %root)
#OutputFileName =os.path.join(IO.procdir, "HDF5/%s.h5" %root)
OutputFile = h5py.File(OutputFileName,'w')
working[0:1023,1004:1023]=0
OutputFile.create_dataset("data",data=working.astype(np.int32))
if HFParams.DoPeakSearch:
OutputFile.create_dataset("processing/hitfinder/peakinfo",data=peakslist.astype(np.int32))
#g1=OutputFile.create_group("processing")
#g2=g1.create_group("hitfinder")
#g2.create_dataset("peakinfo",data=peakslist.astype(np.int32))
OutputFile.close()
#if conversion to Pickle
if IO.pickle:
pixels=flex.int(working.astype(np.int32))
pixel_size=Frelon.pixel_size
data = dpack(data=pixels,
distance=XSetup.distance,
pixel_size=pixel_size,
wavelength=XSetup.wavelength,
beam_center_x=XSetup.beam_y*pixel_size,
beam_center_y=XSetup.beam_x*pixel_size,
ccd_image_saturation=Frelon.overload,
saturated_value=Frelon.overload)
data=crop_image_pickle(data)
OutputFileName =os.path.join(IO.procdir, IO.PicklesDir , "%s.pickle" %root)
easy_pickle.dump(OutputFileName,data)
else: working =0
else:
print 'Warning : data shape problem for file %s, file skiped '%fname
#continue
return [hit,imgmax,imgmin,imgmed,index,peakslist,fname,working]
#beamX=602.015+(ds.env().epicsStore().value("robot_x"))
#beamY=51.682+(ds.env().epicsStore().value("robot_y"))
#beamDist=-563.083+60.+ (ds.env().epicsStore().value("robot_z"))
beamX = 87.96
beamY = 85.57
beamDist = 150
print 'beam: ',beamX,beamY,beamDist
# Assemble data pack
data = cspad_tbx.dpack(data=image,
distance=beamDist,
pixel_size=pixelSize,
wavelength=lambda_wavelength,
beam_center_x=beamX,
beam_center_y=beamY,
ccd_image_saturation=saturation,
saturated_value=saturation,
timestamp = evt_timestamp)
# output image pickle
fpklname='/reg/d/psdm/xpp/%s/ftc/Run%s/Run_%s_idx_{}.pickle'%(expname,run,run)
ep.dump(fpklname.format(ievt), data)
print 'Importing image {}, beamX = {}, beamY = {}, Z-offset = {}, wavelength = {} '.format(fpklname, beamX, beamY, beamDist, lambda_wavelength)
# if ievt>5:
# break
print 'is done '
def endjob(self, obj1, obj2=None):
"""The endjob() function writes the mean and standard deviation images
to disk.
@param evt Event object (psana only)
@param env Environment object
"""
if obj2 is None:
env = obj1
else:
evt = obj1
env = obj2
stats = super(mod_average, self).endjob(env)
if stats is None:
return
device = cspad_tbx.address_split(self.address)[2]
if device == 'Andor':
beam_center = (0, 0) # XXX Fiction!
pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
saturated_value = 10000
elif device == 'Cspad' or device == 'Cspad2x2':
beam_center = self.beam_center
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
beam_center = tuple(t // 2 for t in d['mean_img'].focus())
pixel_size = 0.079346
saturated_value = 2**16 - 1
if stats['nmemb'] > 0:
if self.avg_dirname is not None or \
self.avg_basename is not None or \
self._mean_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['mean_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._mean_out is not None:
p = cspad_tbx.dwritef2(d, self._mean_out)
else:
p = cspad_tbx.dwritef(d, self.avg_dirname, self.avg_basename)
self.logger.info("Average written to %s" % p)
if self.stddev_dirname is not None or \
self.stddev_basename is not None or \
self._std_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['std_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._std_out is not None:
p = cspad_tbx.dwritef2(d, self._std_out)
else:
p = cspad_tbx.dwritef(d, self.stddev_dirname, self.stddev_basename)
self.logger.info("Standard deviation written to %s" % p)
if self.max_dirname is not None or \
self.max_basename is not None or \
self._max_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['max_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._max_out is not None:
p = cspad_tbx.dwritef2(d, self._max_out)
else:
p = cspad_tbx.dwritef(d, self.max_dirname, self.max_basename)
self.logger.info("Max written to %s" % p)
if stats['nfail'] == 0:
self.logger.info("%d images processed" % stats['nmemb'])
else:
self.logger.warning(
"%d images processed, %d failed" % (stats['nmemb'], stats['nfail']))
spectrum_focus = self.sum_img[start_row:end_row, :]
mask_focus = mask[start_row:end_row, :]
spectrum_focus.set_selected(mask_focus > 0, 0)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print "Number of rows: %i" % spectrum_focus.all()[0]
print "Estimated no. photons counted: %i" % flex.sum(spectrum_focus)
print "Number of images used: %i" % flex.sum(
pixel_histograms.histograms.values()[0].slots())
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr = "_%04d" % run
else: runstr = ""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)
if gain_map is None:
gain_map = flex.double(gain_img.accessor(), 0)
img_sel = (gain_img > 0).as_1d()
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_img,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
gain_map.as_1d().set_selected(img_sel.iselection(),
1 / gain_img.as_1d().select(img_sel))
def run(argv=None):
import libtbx.option_parser
if (argv is None):
argv = sys.argv
command_line = (libtbx.option_parser.option_parser(
usage="%s [-v] [-p poly_mask] [-c circle_mask] [-x mask_pix_val] [-o output] -W mask_width -H mask_height" % libtbx.env.dispatcher_name)
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--poly_mask", "-p",
type="string",
default=None,
dest="poly_mask",
help="Polygon to mask out. Comma-seperated string of xy pairs.")
.option(None, "--circle_mask", "-c",
type="string",
default=None,
dest="circle_mask",
help="Circle to mask out. Comma-seperated string of x, y, and radius.")
.option(None, "--mask_pix_val", "-x",
type="int",
default=-2,
dest="mask_pix_val",
help="Value for masked out pixels")
.option(None, "--mask_width", "-W",
type="int",
default=None,
dest="mask_width",
help="Width of output mask")
.option(None, "--mask_height", "-H",
type="int",
default=None,
dest="mask_height",
help="Height of output mask")
.option(None, "--output", "-o",
type="string",
default="mask.pickle",
dest="destpath",
help="Output file path, should be *.pickle")
.option(None, "--pixel_size", "-s",
type="float",
default=None,
dest="pixel_size",
help="Pixel size for detector")
).process(args=argv[1:])
# Must have width and height set
if command_line.options.mask_height is None or command_line.options.mask_width is None:
command_line.parser.print_usage(file=sys.stderr)
return
poly_mask = None
if not command_line.options.poly_mask == None:
poly_mask = []
poly_mask_tmp = command_line.options.poly_mask.split(",")
if len(poly_mask_tmp) % 2 != 0:
command_line.parser.print_usage(file=sys.stderr)
return
odd = True
for item in poly_mask_tmp:
try:
if odd:
poly_mask.append(int(item))
else:
poly_mask[-1] = (poly_mask[-1],int(item))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
odd = not odd
circle_mask = None
if command_line.options.circle_mask is not None:
circle_mask_tmp = command_line.options.circle_mask.split(",")
if len(circle_mask_tmp) != 3:
command_line.parser.print_usage(file=sys.stderr)
return
try:
circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2]))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
mask = flex.int(flex.grid(command_line.options.mask_width,
command_line.options.mask_height))
if poly_mask is not None or circle_mask is not None:
minx = miny = 0
maxx = mask.focus()[0]
maxy = mask.focus()[1]
if poly_mask is not None:
minx = min([x[0] for x in poly_mask])
miny = min([y[1] for y in poly_mask])
maxx = max([x[0] for x in poly_mask])
maxy = max([y[1] for y in poly_mask])
if circle_mask is not None:
circle_x, circle_y, radius = circle_mask
if circle_x - radius < minx: minx = circle_x - radius
if circle_y - radius < miny: miny = circle_y - radius
if circle_x + radius > maxx: maxx = circle_x + radius
if circle_y + radius > maxy: maxy = circle_y + radius
sel = mask == command_line.options.mask_pix_val
for j in xrange(miny, maxy):
for i in xrange(minx, maxx):
idx = j * mask.focus()[0] + i
if not sel[idx]:
if poly_mask is not None and point_in_polygon((i,j),poly_mask):
sel[idx] = True
elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius):
sel[idx] = True
mask.set_selected(sel,command_line.options.mask_pix_val)
masked_out = len(mask.as_1d().select((mask == command_line.options.mask_pix_val).as_1d()))
print "Masked out %d pixels out of %d (%.2f%%)"% \
(masked_out,len(mask),(masked_out)*100/(len(mask)))
easy_pickle.dump(command_line.options.destpath, mask)
d = dpack(
active_areas=[0,0,command_line.options.mask_width,command_line.options.mask_height],
address=None,
beam_center_x=None,
beam_center_y=None,
data=mask,
distance=None,
timestamp=None,
wavelength=1,
xtal_target=None,
pixel_size=command_line.options.pixel_size,
saturated_value=None)
dwritef2(d, command_line.options.destpath)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(average_mixin, self).event(evt, env)
if evt.get('skip_event'):
return
# Get the distance for the detectors that should have it, and set
# it to NaN for those that should not.
if self.detector == 'CxiDs1' or \
self.detector == 'CxiDs2' or \
self.detector == 'CxiDsd' or \
self.detector == 'XppGon':
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self._nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
else:
distance = float('nan')
if ("skew" in self.flags):
# Take out inactive pixels
if self.roi is not None:
pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
pixels = pixels.as_1d().select(dark_mask.as_1d())
else:
pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double()
stats = scitbx.math.basic_statistics(pixels.as_double())
#stats.show()
self.logger.info("skew: %.3f" %stats.skew)
self.logger.info("kurtosis: %.3f" %stats.kurtosis)
if 0:
from matplotlib import pyplot
hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double())
print hist_min, hist_max
n_slots = 100
n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
pyplot.show()
# XXX This skew threshold probably needs fine-tuning
skew_threshold = 0.35
if stats.skew < skew_threshold:
self._nfail += 1
self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold)
evt.put(skip_event_flag(), 'skip_event')
return
#self.cspad_img *= stats.skew
if ("inactive" in self.flags):
self.cspad_img.set_selected(self.dark_stddev <= 0, 0)
if ("noelastic" in self.flags):
ELASTIC_THRESHOLD = self.elastic_threshold
self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0)
if self.hot_threshold is not None:
HOT_THRESHOLD = self.hot_threshold
self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0)
if self.gain_map is not None and self.gain_threshold is not None:
# XXX comparing each pixel to a moving average would probably be better
# since the gain should vary approximately smoothly over different areas
# of the detector
GAIN_THRESHOLD = self.gain_threshold
#self.logger.debug(
#"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True))
self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0)
if ("nonoise" in self.flags):
NOISE_THRESHOLD = self.noise_threshold
self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0)
if ("sigma_scaling" in self.flags):
self.do_sigma_scaling()
if ("symnoise" in self.flags):
SYMNOISE_THRESHOLD = self.symnoise_threshold
self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) &
( self.cspad_img < SYMNOISE_THRESHOLD), 0)
if ("output" in self.flags):
import pickle,os
if (not os.path.isdir(self.pickle_dirname)):
os.makedirs(self.pickle_dirname)
flexdata = flex.int(self.cspad_img.astype(numpy.int32))
d = cspad_tbx.dpack(
address=self.address,
data=flexdata,
timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
)
G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename,
"ab")
pickle.dump(d,G,pickle.HIGHEST_PROTOCOL)
G.close()
if self.photon_threshold is not None and self.two_photon_threshold is not None:
self.do_photon_counting()
if self.background_path is not None:
self.cspad_img -= self.background_img
# t and self._sum_time are a two-long arrays of seconds and
# milliseconds which hold time with respect to the base time.
t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt),
self._metadata['time_base'])]
if self._nmemb == 0:
# The peers metadata item is a bit field where a bit is set if
# the partial sum from the corresponding worker process is
# pending. If this is the first frame a worker process sees,
# set its corresponding bit in the bit field since it will
# contribute a partial sum.
if env.subprocess() >= 0:
self._lock.acquire()
if 'peers' in self._metadata.keys():
self._metadata['peers'] |= (1 << env.subprocess())
else:
self._metadata['peers'] = (1 << env.subprocess())
self._lock.release()
self._sum_distance = distance
self._sum_time = (t[0], t[1])
self._sum_wavelength = self.wavelength
if self._have_max:
self._max_img = self.cspad_img.deep_copy()
if self._have_mean:
self._sum_img = self.cspad_img.deep_copy()
if self._have_std:
self._ssq_img = flex.pow2(self.cspad_img)
else:
self._sum_distance += distance
self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1])
self._sum_wavelength += self.wavelength
if self._have_max:
sel = (self.cspad_img > self._max_img).as_1d()
self._max_img.as_1d().set_selected(
sel, self.cspad_img.as_1d().select(sel))
if self._have_mean:
self._sum_img += self.cspad_img
if self._have_std:
self._ssq_img += flex.pow2(self.cspad_img)
self._nmemb += 1
def run(argv=None):
if argv is None:
argv = sys.argv[1:]
command_line = (
libtbx.option_parser.option_parser(
usage="%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files"
% libtbx.env.dispatcher_name
)
.option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress",
)
.option(
None,
"--crop",
"-c",
action="store_true",
default=False,
dest="crop",
help="Crop the image such that the beam center is in the middle",
)
.option(
None,
"--skip_converted",
"-s",
action="store_true",
default=False,
dest="skip_converted",
help="Skip converting if an image already exist that matches the destination file name",
)
.option(
None,
"--wavelength",
"-w",
type="float",
default=None,
dest="wavelength",
help="Override the image's wavelength (angstroms)",
)
.option(
None,
"--distance",
"-d",
type="float",
default=None,
dest="distance",
help="Override the detector distance (mm)",
)
.option(
None,
"--pixel_size",
"-p",
type="float",
default=None,
dest="pixel_size",
help="Override the detector pixel size (mm)",
)
.option(
None,
"--beam_x",
"-x",
type="float",
default=None,
dest="beam_center_x",
help="Override the beam x position (pixels)",
)
.option(
None,
"--beam_y",
"-y",
type="float",
default=None,
dest="beam_center_y",
help="Override the beam y position (pixels)",
)
.option(
None,
"--overload",
"-o",
type="float",
default=None,
dest="overload",
help="Override the detector overload value (ADU)",
)
).process(args=argv)
paths = command_line.args
if len(paths) <= 0:
raise Usage("No files specified")
for imgpath in paths:
destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle")
if command_line.options.skip_converted and os.path.isfile(destpath):
if command_line.options.verbose:
print "Skipping %s, file exists" % imgpath
continue
if command_line.options.verbose:
print "Converting %s to %s..." % (imgpath, destpath)
try:
img = dxtbx.load(imgpath)
except IOError:
img = None
pass
if img is None:
import numpy as np
try:
raw_data = np.loadtxt(imgpath)
from scitbx.array_family import flex
raw_data = flex.double(raw_data.astype(np.double))
except ValueError:
raise Usage("Couldn't load %s, no supported readers" % imgpath)
detector = None
beam = None
scan = None
else:
raw_data = img.get_raw_data()
detector = img.get_detector()
beam = img.get_beam()
scan = img.get_scan()
if detector is None:
if command_line.options.distance is None:
raise Usage("Can't get distance from image. Override with -d")
if command_line.options.pixel_size is None:
raise Usage("Can't get pixel size from image. Override with -p")
if command_line.options.overload is None:
raise Usage("Can't get overload value from image. Override with -o")
distance = command_line.options.distance
pixel_size = command_line.options.pixel_size
overload = command_line.options.overload
else:
detector = detector[0]
if command_line.options.distance is None:
distance = detector.get_distance()
else:
distance = command_line.options.distance
if command_line.options.pixel_size is None:
pixel_size = detector.get_pixel_size()[0]
else:
pixel_size = command_line.options.pixel_size
if command_line.options.overload is None:
overload = detector.get_trusted_range()[1]
else:
overload = command_line.options.overload
if beam is None:
if command_line.options.wavelength is None:
raise Usage("Can't get wavelength from image. Override with -w")
wavelength = command_line.options.wavelength
else:
if command_line.options.wavelength is None:
wavelength = beam.get_wavelength()
else:
wavelength = command_line.options.wavelength
if beam is None and detector is None:
if command_line.options.beam_center_x is None:
print "Can't get beam x position from image. Using image center. Override with -x"
beam_x = raw_data.focus()[0] * pixel_size
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
print "Can't get beam y position from image. Using image center. Override with -y"
beam_y = raw_data.focus()[1] * pixel_size
else:
beam_y = command_line.options.beam_center_y * pixel_size
else:
if command_line.options.beam_center_x is None:
beam_x = detector.get_beam_centre(beam.get_s0())[0]
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
beam_y = detector.get_beam_centre(beam.get_s0())[1]
else:
beam_y = command_line.options.beam_center_y * pixel_size
if scan is None:
timestamp = None
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
data = dpack(
data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp,
)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data["OSC_START"] = osc_start
data["OSC_RANGE"] = osc_range
data["TIME"] = scan.get_exposure_times()[0]
if command_line.options.crop:
data = crop_image_pickle(data)
easy_pickle.dump(destpath, data)
quad_asics.append(numpy.concatenate((a, b), axis=1))
quad_data = numpy.dstack(quad_asics)
quad_data = numpy.rollaxis(quad_data, 2, 0)
data3d.append(fake_cspad_ElementV2(quad_data, i_quad))
env = fake_env(fake_config())
evt = fake_evt(data3d)
beam_center, active_areas = cbcaa(fake_config(), sections)
data = flex.int(
CsPadDetector(address, evt, env, sections).astype(numpy.float64))
img_dict = dpack(active_areas=active_areas,
address=address,
beam_center_x=beam_center[0] * pixel_size,
beam_center_y=beam_center[1] * pixel_size,
data=data,
distance=params.distance,
pixel_size=pixel_size,
timestamp=timestamp,
wavelength=params.wavelength)
img = NpyImage("", source_data=img_dict)
args = [
"distl.detector_format_version=%s" % params.detector_format_version
]
horizons_phil = cxi_phil.cxi_versioned_extract(args)
img.readHeader(horizons_phil)
img.translate_tiles(horizons_phil)
tm = img.get_tile_manager(horizons_phil)
def run(argv=None):
import libtbx.option_parser
if (argv is None):
argv = sys.argv
command_line = (libtbx.option_parser.option_parser(
usage="%s [-v] [-p poly_mask] [-c circle_mask] [-x mask_pix_val] [-o output] -W mask_width -H mask_height" % libtbx.env.dispatcher_name)
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--poly_mask", "-p",
type="string",
default=None,
dest="poly_mask",
help="Polygon to mask out. Comma-seperated string of xy pairs.")
.option(None, "--circle_mask", "-c",
type="string",
default=None,
dest="circle_mask",
help="Circle to mask out. Comma-seperated string of x, y, and radius.")
.option(None, "--mask_pix_val", "-x",
type="int",
default=-2,
dest="mask_pix_val",
help="Value for masked out pixels")
.option(None, "--mask_width", "-W",
type="int",
default=None,
dest="mask_width",
help="Width of output mask")
.option(None, "--mask_height", "-H",
type="int",
default=None,
dest="mask_height",
help="Height of output mask")
.option(None, "--output", "-o",
type="string",
default="mask.pickle",
dest="destpath",
help="Output file path, should be *.pickle")
.option(None, "--pixel_size", "-s",
type="float",
default=None,
dest="pixel_size",
help="Pixel size for detector")
).process(args=argv[1:])
# Must have width and height set
if command_line.options.mask_height is None or command_line.options.mask_width is None:
command_line.parser.print_usage(file=sys.stderr)
return
poly_mask = None
if not command_line.options.poly_mask == None:
poly_mask = []
poly_mask_tmp = command_line.options.poly_mask.split(",")
if len(poly_mask_tmp) % 2 != 0:
command_line.parser.print_usage(file=sys.stderr)
return
odd = True
for item in poly_mask_tmp:
try:
if odd:
poly_mask.append(int(item))
else:
poly_mask[-1] = (poly_mask[-1],int(item))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
odd = not odd
circle_mask = None
if command_line.options.circle_mask is not None:
circle_mask_tmp = command_line.options.circle_mask.split(",")
if len(circle_mask_tmp) != 3:
command_line.parser.print_usage(file=sys.stderr)
return
try:
circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2]))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
mask = flex.int(flex.grid(command_line.options.mask_width,
command_line.options.mask_height))
if poly_mask is not None or circle_mask is not None:
minx = miny = 0
maxx = mask.focus()[0]
maxy = mask.focus()[1]
if poly_mask is not None:
minx = min([x[0] for x in poly_mask])
miny = min([y[1] for y in poly_mask])
maxx = max([x[0] for x in poly_mask])
maxy = max([y[1] for y in poly_mask])
if circle_mask is not None:
circle_x, circle_y, radius = circle_mask
if circle_x - radius < minx: minx = circle_x - radius
if circle_y - radius < miny: miny = circle_y - radius
if circle_x + radius > maxx: maxx = circle_x + radius
if circle_y + radius > maxy: maxy = circle_y + radius
sel = mask == command_line.options.mask_pix_val
for j in range(miny, maxy):
for i in range(minx, maxx):
idx = j * mask.focus()[0] + i
if not sel[idx]:
if poly_mask is not None and point_in_polygon((i,j),poly_mask):
sel[idx] = True
elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius):
sel[idx] = True
mask.set_selected(sel,command_line.options.mask_pix_val)
masked_out = len(mask.as_1d().select((mask == command_line.options.mask_pix_val).as_1d()))
print("Masked out %d pixels out of %d (%.2f%%)"% \
(masked_out,len(mask),(masked_out)*100/(len(mask))))
easy_pickle.dump(command_line.options.destpath, mask)
d = dpack(
active_areas=[0,0,command_line.options.mask_width,command_line.options.mask_height],
address=None,
beam_center_x=None,
beam_center_y=None,
data=mask,
distance=None,
timestamp=None,
wavelength=1,
xtal_target=None,
pixel_size=command_line.options.pixel_size,
saturated_value=None)
dwritef2(d, command_line.options.destpath)
def load_image_file(self, filepath):
""" Reads a single image file (e.g. a CBF or MCCD) and returns raw data,
experiment information, and an image type (e.g. pickle, raw image, or none)
:param img: path to image file
:return: data: raw image data and experiment info
img_type: which type of image this was, or None if not loaded
"""
error = None
try:
with util.Capturing() as junk_output:
loaded_img = dxtbx.load(filepath)
except Exception as e:
error = 'IOTA IMPORTER ERROR: DXTBX failed! {}'.format(e)
print(e)
loaded_img = None
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
# Populate image object information
self.img_object.final['pixel_size'] = pixel_size
self.img_object.final['img_size'] = (data['SIZE1'], data['SIZE2'])
self.img_object.final['beamX'] = beam_x
self.img_object.final['beamY'] = beam_y
self.img_object.final['gain'] = detector.get_gain()
self.img_object.final['distance'] = distance
self.img_object.final['wavelength'] = wavelength
else:
data = None
return data, error
spectrum_focus = self.sum_img[start_row:end_row,:]
mask_focus = mask[start_row:end_row,:]
spectrum_focus.set_selected(mask_focus > 0, 0)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print "Number of rows: %i" %spectrum_focus.all()[0]
print "Estimated no. photons counted: %i" %flex.sum(spectrum_focus)
print "Number of images used: %i" %flex.sum(
pixel_histograms.histograms.values()[0].slots())
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr="_%04d"%run
else: runstr=""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_'%runstr)
if gain_map is None:
gain_map = flex.double(gain_img.accessor(), 0)
img_sel = (gain_img > 0).as_1d()
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=gain_img,
distance=1
)
cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
def run(argv=None):
"""Compute mean, standard deviation, and maximum projection images
from a set of images given on the command line.
@param argv Command line argument list
@return @c 0 on successful termination, @c 1 on error, and @c 2
for command line syntax errors
"""
import libtbx.load_env
from libtbx import easy_pickle, option_parser
from scitbx.array_family import flex
from xfel.cxi.cspad_ana import cspad_tbx
if argv is None:
argv = sys.argv
command_line = (option_parser.option_parser(
usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
"image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
.option(None, "--average-path", "-a",
type="string",
default=None,
dest="avg_path",
metavar="PATH",
help="Write average image to PATH")
.option(None, "--maximum-path", "-m",
type="string",
default=None,
dest="max_path",
metavar="PATH",
help="Write maximum projection image to PATH")
.option(None, "--stddev-path", "-s",
type="string",
default=None,
dest="stddev_path",
metavar="PATH",
help="Write standard deviation image to PATH")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
).process(args=argv[1:])
# Note that it is not an error to omit the output paths, because
# certain statistics could still be printed, e.g. with the verbose
# option.
paths = command_line.args
if len(paths) == 0:
command_line.parser.print_usage(file=sys.stderr)
return 2
# Loop over all images and accumulate statistics.
nfail = 0
nmemb = 0
if len(paths) == 1:
# test if the iamge is a multi-image
from dxtbx.format.Registry import Registry
from dxtbx.format.FormatMultiImage import FormatMultiImage
format_class = Registry.find(paths[0])
if not issubclass(format_class, FormatMultiImage):
from libtbx.utils import Usage
raise Usage("Supply more than one image")
print "Loading image..."
i = format_class(paths[0])
print "Loaded"
def read_single_image(n):
if command_line.options.verbose:
sys.stdout.write("Processing %s: %d...\n" % (paths[0], n))
beam = i.get_beam(n)
assert len(i.get_detector(n)) == 1
detector = i.get_detector(n)[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = i.get_raw_data(n).as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
scan = i.get_scan(n)
if scan is None:
time_tuple = (0, 0)
else:
time_tuple = (scan.get_epochs()[0], 0)
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas
iterable = xrange(i.get_num_images())
else:
def read_single_image(path):
if command_line.options.verbose:
sys.stdout.write("Processing %s...\n" % path)
from dxtbx.format.Registry import Registry
format_class = Registry.find(path)
i = format_class(path)
beam = i.get_beam()
assert len(i.get_detector()) == 1
detector = i.get_detector()[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = i.get_raw_data().as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
scan = i.get_scan()
if scan is None:
time_tuple = (0, 0)
else:
time_tuple = (scan.get_epochs()[0], 0)
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas
iterable = paths
for item in iterable:
try:
#XXX This code assumes a monolithic detector!
beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas = \
read_single_image(item)
except Exception:
nfail += 1
continue
# See also event() in xfel.cxi.cspad_ana.average_tbx. Record the
# base time as the timestamp of the first image.
#
# The sum-of-squares image is accumulated using long integers, as
# this delays the point where overflow occurs. But really, this
# is just a band-aid...
if nmemb == 0:
max_img = img.deep_copy()
sum_distance = distance
sum_img = img.deep_copy()
ssq_img = flex.pow2(img)
sum_wavelength = wavelength
sum_time = (0, 0)
time_base = time_tuple
else:
sel = (img > max_img).as_1d()
max_img.set_selected(sel, img.select(sel))
sum_distance += distance
sum_img += img
ssq_img += flex.pow2(img)
sum_wavelength += wavelength
sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]),
sum_time[1] + (time_tuple[1] - time_base[1]))
nmemb += 1
# Early exit if no statistics were accumulated.
if command_line.options.verbose:
sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
if nmemb == 0:
return 0
# Calculate averages for measures where other statistics do not make
# sense. Note that avg_img is required for stddev_img.
avg_img = sum_img.as_double() / nmemb
avg_distance = sum_distance / nmemb
avg_timestamp = cspad_tbx.evt_timestamp(
(time_base[0] + int(round(sum_time[0] / nmemb)),
time_base[1] + int(round(sum_time[1] / nmemb))))
avg_wavelength = sum_wavelength / nmemb
# Output the average image, maximum projection image, and standard
# deviation image, if requested.
if command_line.options.avg_path is not None:
avg_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=avg_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.avg_path, d)
if command_line.options.max_path is not None:
max_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=max_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.max_path, d)
if command_line.options.stddev_path is not None:
stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img
# Accumulating floating-point numbers introduces errors, which may
# cause negative variances. Since a two-pass approach is
# unacceptable, the standard deviation is clamped at zero.
stddev_img.set_selected(stddev_img < 0, 0)
if nmemb == 1:
stddev_img = flex.sqrt(stddev_img)
else:
stddev_img = flex.sqrt(stddev_img / (nmemb - 1))
stddev_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=stddev_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.stddev_path, d)
return 0
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(
usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name)
.option(None, "--as_pickle", "-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files")
.option(None, "--config", "-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file")
.option(None, "--experiment", "-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)")
.option(None, "--run", "-r",
type="int",
default=None,
dest="run",
help="run number")
.option(None, "--address", "-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)")
.option(None, "--detz_offset", "-d",
type="float",
default=None,
dest="detz_offset",
help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)")
.option(None, "--outputdir", "-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files")
.option(None, "--averagebase", "-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help="Path to output average image without extension. String substitution allowed")
.option(None, "--stddevbase", "-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help="Path to output standard deviation image without extension. String substitution allowed")
.option(None, "--maxbase", "-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help="Path to output maximum projection image without extension. String substitution allowed")
.option(None, "--numevents", "-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all")
.option(None, "--skipevents", "-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--pickle-optical-metrology", "-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help="If writing pickle files, use the optical metrology in the experiment's calib directory")
.option(None, "--bin_size", "-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size")
.option(None, "--override_beam_x", "-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate")
.option(None, "--override_beam_y", "-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate")
.option(None, "--calib_dir", "-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory")
.option(None, "--xtc_dir", "-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory")
.option(None, "--use_ffb", "-f",
action="store_true",
default=False,
dest="use_ffb",
help="Use the fast feedback filesystem at LCLS. Only for the active experiment!")
).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxint
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s"%command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)'%address)
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
nevent = np.array([0.])
for run in ds.runs():
runnumber = run.run()
# list of all events
if command_line.options.skipevents > 0:
print "Skipping first %d events"%command_line.options.skipevents
times = run.times()[command_line.options.skipevents:]
nevents = min(len(times),maxevents)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0]
for i in xrange(len(mytimes)):
if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes)
evt = run.event(mytimes[i])
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address:
data = evt.get(Camera.FrameV1,src)
if data is None:
print "No data"
continue
data=data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations
if data is None:
print "No data"
continue
d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset)
if d is None:
print "No distance, skipping shot"
continue
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print "No wavelength, skipping shot"
continue
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print "No timestamp, skipping shot"
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1]/1000)
else:
timestamp = np.array([t[0] + (t[1]/1000)])
if 'sum' in locals():
sum+=data
else:
sum=np.array(data, copy=True)
if 'sumsq' in locals():
sumsq+=data*data
else:
sumsq=data*data
if 'maximum' in locals():
maximum=np.maximum(maximum,data)
else:
maximum=np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print "Synchronizing rank", rank
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent,totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum,sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq,sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum,maxall, op=MPI.MAX)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength,waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance,distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp,timeall)
if rank==0:
if size > 1:
print "Synchronized"
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env())]
dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths]
if 'Rayonix' in command_line.options.address:
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size)
beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y]
detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size)
active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from xfel.detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.calib_dir is not None:
metro_path = command_line.options.calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
maxall = flex.double(maxall.astype(np.float64))
else:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
maxall = flex.double(maxall.astype(np.float64))
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d"%run.run())
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def run(argv=None):
"""Compute mean, standard deviation, and maximum projection images
from a set of images given on the command line.
@param argv Command line argument list
@return @c 0 on successful termination, @c 1 on error, and @c 2
for command line syntax errors
"""
import libtbx.load_env
from libtbx import easy_pickle, option_parser
from xfel.cxi.cspad_ana import cspad_tbx
if argv is None:
argv = sys.argv
command_line = (option_parser.option_parser(
usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
"image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
.option(None, "--average-path", "-a",
type="string",
default=None,
dest="avg_path",
metavar="PATH",
help="Write average image to PATH")
.option(None, "--maximum-path", "-m",
type="string",
default=None,
dest="max_path",
metavar="PATH",
help="Write maximum projection image to PATH")
.option(None, "--stddev-path", "-s",
type="string",
default=None,
dest="stddev_path",
metavar="PATH",
help="Write standard deviation image to PATH")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--nproc", "-n",
type="int",
default=1,
dest="nproc",
help="Number of processors")
.option(None, "--num-images-max", "-N",
type="int",
default=None,
dest="num_images_max",
help="Maximum number of frames to average")
).process(args=argv[1:])
# Note that it is not an error to omit the output paths, because
# certain statistics could still be printed, e.g. with the verbose
# option.
paths = command_line.args
if len(paths) == 0:
command_line.parser.print_usage(file=sys.stderr)
return 2
if len(paths) == 1:
# test if the iamge is a multi-image
from dxtbx.datablock import DataBlockFactory
datablocks = DataBlockFactory.from_filenames([paths[0]])
assert len(datablocks) == 1
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
assert len(imagesets) == 1
imageset = imagesets[0]
if not imageset.reader().is_single_file_reader():
from libtbx.utils import Usage
raise Usage("Supply more than one image")
worker = multi_image_worker(command_line, paths[0], imageset)
if command_line.options.num_images_max is not None and command_line.options.num_images_max < len(
imageset):
iterable = range(command_line.options.num_images_max)
else:
iterable = range(len(imageset))
else:
# Multiple images provided
worker = single_image_worker(command_line)
if command_line.options.num_images_max is not None and command_line.options.num_images_max < len(
paths):
iterable = paths[:command_line.options.num_images_max]
else:
iterable = paths
if command_line.options.nproc > 1:
iterable = splitit(iterable, command_line.options.nproc)
from libtbx import easy_mp
if command_line.options.nproc == 1:
results = [worker(iterable)]
else:
results = easy_mp.parallel_map(func=worker,
iterable=iterable,
processes=command_line.options.nproc)
nfail = 0
nmemb = 0
for i, (r_nfail, r_nmemb, r_max_img, r_sum_distance, r_sum_img, r_ssq_img,
r_sum_wavelength, size, active_areas, detector_address,
beam_center, pixel_size, saturated_value) in enumerate(results):
nfail += r_nfail
nmemb += r_nmemb
if i == 0:
max_img = r_max_img
sum_distance = r_sum_distance
sum_img = r_sum_img
ssq_img = r_ssq_img
sum_wavelength = r_sum_wavelength
else:
sel = (r_max_img > max_img).as_1d()
max_img.set_selected(sel, r_max_img.select(sel))
sum_distance += r_sum_distance
sum_img += r_sum_img
ssq_img += r_ssq_img
sum_wavelength += r_sum_wavelength
# Early exit if no statistics were accumulated.
if command_line.options.verbose:
sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
if nmemb == 0:
return 0
# Calculate averages for measures where other statistics do not make
# sense. Note that avg_img is required for stddev_img.
avg_img = sum_img.as_double() / nmemb
avg_distance = sum_distance / nmemb
avg_wavelength = sum_wavelength / nmemb
# Output the average image, maximum projection image, and standard
# deviation image, if requested.
if command_line.options.avg_path is not None:
avg_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=avg_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.avg_path, d)
if command_line.options.max_path is not None:
max_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=max_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.max_path, d)
if command_line.options.stddev_path is not None:
stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img
# Accumulating floating-point numbers introduces errors, which may
# cause negative variances. Since a two-pass approach is
# unacceptable, the standard deviation is clamped at zero.
stddev_img.set_selected(stddev_img < 0, 0)
if nmemb == 1:
stddev_img = flex.sqrt(stddev_img)
else:
stddev_img = flex.sqrt(stddev_img / (nmemb - 1))
stddev_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=stddev_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.stddev_path, d)
return 0
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights, outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights >
self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((
peak_heights > self.m_threshold).__and__(
outvalue == 0)).count(True)
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
self.stats_logger.info("BRAGG %.3f %d" %
(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info(
"Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info(
"Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time,
number_of_accepted_peaks, distance, self.sifoil,
self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(
d,
integration_dirname=self.m_integration_dirname,
integration_basename=self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None and hasattr(info, 'spotfinder_results')
if self.m_progress_logging:
if self.m_db_version == 'v1':
if indexed:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best[
"identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host,
self.m_db_name,
self.m_db_user,
self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best[
"identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[
info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,
self.m_db_experiment_tag,
self.m_trial_id,
self.m_rungroup_id,
evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best[
"identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
elif self.m_db_version == 'v2':
key_low = 'cctbx.xfel.radial_average.two_theta_low'
key_high = 'cctbx.xfel.radial_average.two_theta_high'
tt_low = evt.get(key_low)
tt_high = evt.get(key_high)
from xfel.ui.db.dxtbx_db import log_frame
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
sfspots = evt.get('sfspots')
if sfspots is None:
if info is None or not isinstance(info, int):
n_spots = 0
else:
n_spots = info
else:
n_spots = sfspots
if indexed:
known_setting = info.horizons_phil.known_setting
indexed_setting = info.organizer.info[
'best_integration']['counter']
if known_setting is None or known_setting == indexed_setting:
from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
c = construct_reflection_table_and_experiment_list(
info.last_saved_best,
None,
pixel_size,
proceed_without_image=True)
c.assemble_experiments()
c.assemble_reflections()
log_frame(c.experiment_list, c.reflections,
self.db_params, evt.run(), n_spots,
self.timestamp, tt_low, tt_high)
else:
print(
"Not logging %s, wrong bravais setting (expecting %d, got %d)"
% (self.timestamp, known_setting,
indexed_setting))
else:
log_frame(None, None, self.db_params, evt.run(),
n_spots, self.timestamp, tt_low, tt_high)
if self.m_db_logging:
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get(
'ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get(
'ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get(
'ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
).parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time, n_spots,
distance, self.sifoil, self.wavelength, indexed,
mosaic_bloc_rotation, mosaic_block_size,
ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%" % detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(
horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%s" %
detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation, pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(
horizons_phil, imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead"
)
if (self.m_out_dirname is not None
or self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
None,
"--as_pickle",
"-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files"
).option(
None,
"--raw_data",
"-R",
action="store_true",
default=False,
dest="raw_data",
help=
"Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
).option(
None,
"--background_pickle",
"-B",
default=None,
dest="background_pickle",
help=""
).option(
None,
"--config",
"-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file"
).option(
None,
"--experiment",
"-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)"
).option(
None,
"--run",
"-r",
type="int",
default=None,
dest="run",
help="run number"
).option(
None,
"--address",
"-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)"
).option(
None,
"--detz_offset",
"-d",
type="float",
default=None,
dest="detz_offset",
help=
"offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
).option(
None,
"--outputdir",
"-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files"
).option(
None,
"--averagebase",
"-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help=
"Path to output average image without extension. String substitution allowed"
).option(
None,
"--stddevbase",
"-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help=
"Path to output standard deviation image without extension. String substitution allowed"
).option(
None,
"--maxbase",
"-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help=
"Path to output maximum projection image without extension. String substitution allowed"
).option(
None,
"--numevents",
"-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all"
).option(
None,
"--skipevents",
"-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--pickle-optical-metrology",
"-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help=
"If writing pickle files, use the optical metrology in the experiment's calib directory"
).option(
None,
"--bin_size",
"-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size"
).option(
None,
"--override_beam_x",
"-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate"
).option(
None,
"--override_beam_y",
"-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate"
).option(
None,
"--calib_dir",
"-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory"
).option(
None,
"--pickle_calib_dir",
"-P",
type="string",
default=None,
dest="pickle_calib_dir",
metavar="PATH",
help=
"pickle calibration directory specification. Replaces --calib_dir functionality."
).option(
None,
"--xtc_dir",
"-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory"
).option(
None,
"--use_ffb",
"-f",
action="store_true",
default=False,
dest="use_ffb",
help=
"Use the fast feedback filesystem at LCLS. Only for the active experiment!"
).option(
None,
"--gain_mask_value",
"-g",
type="float",
default=None,
dest="gain_mask_value",
help=
"Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
).option(
None,
"--min",
None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection"
).option(
None,
"--minpath",
None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help=
"Path to output minimum image without extension. String substitution allowed"
)).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxsize
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
command_line.options.experiment[0:3],
command_line.options.experiment)
if command_line.options.calib_dir is not None:
psana.setOption('psana.calib-dir', command_line.options.calib_dir)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)' % address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(
command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print("Skipping first %d events" % command_line.options.skipevents)
elif "Rayonix" in command_line.options.address:
print("Skipping first image in the Rayonix detector"
) # Shuttering issue
command_line.options.skipevents = 1
for i, evt in enumerate(run.events()):
if i % size != rank: continue
if i < command_line.options.skipevents: continue
if i >= maxevents: break
if i % 10 == 0: print('Rank', rank, 'processing event', i)
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1, src)
if data is None:
print("No data")
continue
data = data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det,
evt,
use_default=False,
dark=False,
common_mode=None,
apply_gain_mask=False,
per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det,
evt,
use_default=True)
else:
data = get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=True,
gain_mask_value=command_line.options.
gain_mask_value,
per_pixel_gain=False)
if data is None:
print("No data")
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(),
command_line.options.detz_offset)
if d is None:
print("No distance, using distance",
command_line.options.detz_offset)
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print("No wavelength")
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print("No timestamp, skipping shot")
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1] / 1000)
else:
timestamp = np.array([t[0] + (t[1] / 1000)])
if 'sum' in locals():
sum += data
else:
sum = np.array(data, copy=True)
if 'sumsq' in locals():
sumsq += data * data
else:
sumsq = data * data
if 'maximum' in locals():
maximum = np.maximum(maximum, data)
else:
maximum = np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum = np.minimum(minimum, data)
else:
minimum = np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print("Synchronizing rank", rank)
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent, totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum, sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq, sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum, maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum, minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength, waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance, distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp, timeall)
if rank == 0:
if size > 1:
print("Synchronized")
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [
cspad_tbx.pathsubst(command_line.options.averagepath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
ds.env())
]
if command_line.options.do_minimum_projection:
dest_paths.append(
cspad_tbx.pathsubst(command_line.options.minpath + extension,
evt, ds.env()))
dest_paths = [
os.path.join(command_line.options.outputdir, path)
for path in dest_paths
]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
command_line.options.bin_size)
beam_center = [
command_line.options.override_beam_x,
command_line.options.override_beam_y
]
active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(address=old_style_address,
data=flex.double(data),
distance=distance,
pixel_size=0.1,
timestamp=timestamp,
wavelength=wavelength)
print("Saving", path)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(
evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
old_style_address,
reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.pickle_calib_dir is not None:
metro_path = command_line.options.pickle_calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
address,
round_to_int=False)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def __init__(self,
pixel_histograms,
output_dirname=".",
gain_map_path=None,
gain_map=None,
method="photon_counting",
estimated_gain=30,
nproc=None,
photon_threshold=2 / 3,
roi=None,
run=None):
assert method in ("sum_adu", "photon_counting")
self.sum_img = flex.double(flex.grid(
370, 391), 0) # XXX define the image size some other way?
gain_img = flex.double(self.sum_img.accessor(), 0)
assert [gain_map, gain_map_path].count(None) > 0
if gain_map_path is not None:
d = easy_pickle.load(gain_map_path)
gain_map = d["DATA"]
two_photon_threshold = photon_threshold + 1
mask = flex.int(self.sum_img.accessor(), 0)
start_row = 370
end_row = 0
print(len(pixel_histograms.histograms))
pixels = list(pixel_histograms.pixels())
n_pixels = len(pixels)
if roi is not None:
for k, (i, j) in enumerate(reversed(pixels)):
if (i < roi[2] or i > roi[3] or j < roi[0] or j > roi[1]):
del pixels[n_pixels - k - 1]
if gain_map is None:
fixed_func = pixel_histograms.fit_one_histogram
else:
def fixed_func(pixel):
return pixel_histograms.fit_one_histogram(pixel, n_gaussians=1)
results = None
if nproc is None: nproc = easy_mp.Auto
nproc = easy_mp.get_processes(nproc)
print("nproc: ", nproc)
stdout_and_results = easy_mp.pool_map(
processes=nproc,
fixed_func=fixed_func,
args=pixels,
func_wrapper="buffer_stdout_stderr")
results = [r for so, r in stdout_and_results]
gains = flex.double()
for i, pixel in enumerate(pixels):
start_row = min(start_row, pixel[0])
end_row = max(end_row, pixel[0])
n_photons = 0
if results is None:
# i.e. not multiprocessing
try:
gaussians = pixel_histograms.fit_one_histogram(pixel)
except RuntimeError as e:
print("Error fitting pixel %s" % str(pixel))
print(str(e))
mask[pixel] = 1
continue
else:
gaussians = results[i]
hist = pixel_histograms.histograms[pixel]
if gaussians is None:
# Presumably the peak fitting failed in some way
print("Skipping pixel %s" % str(pixel))
continue
zero_peak_diff = gaussians[0].params[1]
if gain_map is None:
try:
view_pixel_histograms.check_pixel_histogram_fit(
hist, gaussians)
except view_pixel_histograms.PixelFitError as e:
print("PixelFitError:", str(pixel), str(e))
mask[pixel] = 1
continue
gain = gaussians[1].params[1] - gaussians[0].params[1]
gain_img[pixel] = gain
gain_ratio = gain / estimated_gain
else:
gain = gain_map[pixel]
if gain == 0:
print("bad gain!!!!!", pixel)
continue
gain = 30 / gain
gain_ratio = 1 / gain
gains.append(gain)
#for g in gaussians:
#sigma = abs(g.params[2])
#if sigma < 1 or sigma > 10:
#print "bad sigma!!!!!", pixel, sigma
#mask[pixel] = 1
#continue
if method == "sum_adu":
sum_adu = 0
one_photon_cutoff, two_photon_cutoff = [
(threshold * gain + zero_peak_diff)
for threshold in (photon_threshold, two_photon_threshold)
]
i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
slots = hist.slots().as_double()
slot_centers = hist.slot_centers()
slots -= gaussians[0](slot_centers)
for j in range(i_one_photon_cutoff, len(slots)):
center = slot_centers[j]
sum_adu += slots[j] * (center - zero_peak_diff) * 30 / gain
self.sum_img[pixel] = sum_adu
elif method == "photon_counting":
one_photon_cutoff, two_photon_cutoff = [
(threshold * gain + zero_peak_diff)
for threshold in (photon_threshold, two_photon_threshold)
]
i_one_photon_cutoff = hist.get_i_slot(one_photon_cutoff)
i_two_photon_cutoff = hist.get_i_slot(two_photon_cutoff)
slots = hist.slots()
for j in range(i_one_photon_cutoff, len(slots)):
if j == i_one_photon_cutoff:
center = hist.slot_centers()[j]
upper = center + 0.5 * hist.slot_width()
n_photons += int(
round((upper - one_photon_cutoff) /
hist.slot_width() * slots[j]))
elif j == i_two_photon_cutoff:
center = hist.slot_centers()[j]
upper = center + 0.5 * hist.slot_width()
n_photons += 2 * int(
round((upper - two_photon_cutoff) /
hist.slot_width() * slots[j]))
elif j < i_two_photon_cutoff:
n_photons += int(round(slots[j]))
else:
n_photons += 2 * int(round(slots[j]))
self.sum_img[pixel] = n_photons
stats = scitbx.math.basic_statistics(gains)
print("gain statistics:")
stats.show()
mask.set_selected(self.sum_img == 0, 1)
unbound_pixel_mask = xes_finalise.cspad_unbound_pixel_mask()
mask.set_selected(unbound_pixel_mask > 0, 1)
bad_pixel_mask = xes_finalise.cspad2x2_bad_pixel_mask_cxi_run7()
mask.set_selected(bad_pixel_mask > 0, 1)
for row in range(self.sum_img.all()[0]):
self.sum_img[row:row + 1, :].count(0)
spectrum_focus = self.sum_img[start_row:end_row, :]
mask_focus = mask[start_row:end_row, :]
spectrum_focus.set_selected(mask_focus > 0, 0)
xes_finalise.filter_outlying_pixels(spectrum_focus, mask_focus)
print("Number of rows: %i" % spectrum_focus.all()[0])
print("Estimated no. photons counted: %i" % flex.sum(spectrum_focus))
print("Number of images used: %i" %
flex.sum(pixel_histograms.histograms.values()[0].slots()))
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=spectrum_focus,
distance=1,
ccd_image_saturation=2e8, # XXX
)
if run is not None: runstr = "_%04d" % run
else: runstr = ""
cspad_tbx.dwritef(d, output_dirname, 'sum%s_' % runstr)
if gain_map is None:
gain_map = flex.double(gain_img.accessor(), 0)
img_sel = (gain_img > 0).as_1d()
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_img,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'raw_gain_map_')
gain_map.as_1d().set_selected(img_sel.iselection(),
1 / gain_img.as_1d().select(img_sel))
gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
d = cspad_tbx.dpack(address='CxiSc1-0|Cspad2x2-0',
data=gain_map,
distance=1)
cspad_tbx.dwritef(d, output_dirname, 'gain_map_')
plot_x, plot_y = xes_finalise.output_spectrum(
spectrum_focus.iround(),
mask_focus=mask_focus,
output_dirname=output_dirname,
run=run)
self.spectrum = (plot_x, plot_y)
self.spectrum_focus = spectrum_focus
xes_finalise.output_matlab_form(
spectrum_focus, "%s/sum%s.m" % (output_dirname, runstr))
print(output_dirname)
def __init__(self, runs, output_dirname=".", roi=None):
avg_basename = "avg_"
stddev_basename = "stddev"
self.sum_img = None
self.sumsq_img = None
self.nmemb = 0
self.roi = cspad_tbx.getOptROI(roi)
self.unbound_pixel_mask = cspad_unbound_pixel_mask()
for i_run, run in enumerate(runs):
run_scratch_dir = run
result = finalise_one_run(run_scratch_dir)
if result.sum_img is None: continue
if self.sum_img is None:
self.sum_img = result.sum_img
self.sumsq_img = result.sumsq_img
else:
self.sum_img += result.sum_img
self.sumsq_img += result.sumsq_img
self.nmemb += result.nmemb
self.avg_img = self.sum_img.as_double() / self.nmemb
self.stddev_img = flex.sqrt(
(self.sumsq_img.as_double() -
self.sum_img.as_double() * self.avg_img) / (self.nmemb - 1))
self.mask = flex.int(self.sum_img.accessor(), 0)
self.mask.set_selected(self.sum_img == 0, 1)
self.mask.set_selected(self.unbound_pixel_mask > 0, 1)
if (output_dirname is not None and avg_basename is not None):
if (not os.path.isdir(output_dirname)):
os.makedirs(output_dirname)
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.avg_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, avg_basename)
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.sum_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, "sum_")
if 1:
output_image(self.avg_img, "%s/avg.png" % output_dirname)
output_image(self.avg_img,
"%s/avg_inv.png" % output_dirname,
invert=True)
if 1:
output_matlab_form(self.sum_img, "%s/sum.m" % output_dirname)
output_matlab_form(self.avg_img, "%s/avg.m" % output_dirname)
output_matlab_form(self.stddev_img,
"%s/stddev.m" % output_dirname)
if (stddev_basename is not None):
d = cspad_tbx.dpack(
address='CxiSc1-0|Cspad2x2-0',
data=self.stddev_img,
distance=1,
)
cspad_tbx.dwritef(d, output_dirname, stddev_basename)
# XXX we should really figure out automatically the area where the spectrum is
#write an integrated spectrum from lines 186-227
#spectrum_focus = self.sum_img.as_numpy_array()[186:228,:]
img = self.sum_img
if self.roi is None:
spectrum_focus = img
mask_focus = self.mask
else:
slices = (slice(self.roi[2],
self.roi[3]), slice(self.roi[0], self.roi[1]))
spectrum_focus = img[slices]
mask_focus = self.mask[slices]
if False:
from matplotlib import pylab
pylab.imshow(spectrum_focus.as_numpy_array())
pylab.show()
output_spectrum(spectrum_focus,
mask_focus=mask_focus,
output_dirname=output_dirname)
print "Total number of images used from %i runs: %i" % (i_run + 1,
self.nmemb)
def run(args):
command_line = (option_parser().option(
"-o",
"--output_filename",
action="store",
type="string",
help="Filename for the output pickle file",
default="gain_map.pickle"
).option(
"-f",
"--detector_format_version",
action="store",
type="string",
help=
"Detector format version to use for generating active areas and laying out tiles",
default=None
).option(
"-m",
"--optical_metrology_path",
action="store",
type="string",
help=
"Path to slac optical metrology file. If not set, use Run 4 metrology",
default=None
).option(
"-d",
"--distance",
action="store",
type="int",
help=
"Detector distance put into the gain pickle file. Not needed for processing.",
default="0"
).option(
"-w",
"--wavelength",
action="store",
type="float",
help=
"Incident beam wavelength put into the gain pickle file. Not needed for processing.",
default="0")).process(args=args)
output_filename = command_line.options.output_filename
detector_format_version = command_line.options.detector_format_version
if detector_format_version is None or 'XPP' not in detector_format_version:
beam_center_x = None
beam_center_y = None
else:
beam_center_x = 1765 // 2 * 0.11
beam_center_y = 1765 // 2 * 0.11
address, timestamp = address_and_timestamp_from_detector_format_version(
detector_format_version)
# if no detector format version is provided, make sure to write no address to the image pickle
# but CsPadDetector (called later), needs an address, so give it a fake one
save_address = address is not None
if not save_address:
address = "CxiDs1-0|Cspad-0" # time stamp will still be None
timestamp = evt_timestamp((timestamp, 0))
args = command_line.args
assert len(args) == 1
if args[0].endswith('.npy'):
data = numpy.load(args[0])
det, active_areas = convert_2x2(data, detector_format_version, address)
elif args[0].endswith('.txt') or args[0].endswith('.gain'):
raw_data = numpy.loadtxt(args[0])
assert raw_data.shape in [(5920, 388), (11840, 194)]
det, active_areas = convert_detector(
raw_data, detector_format_version, address,
command_line.options.optical_metrology_path)
img_diff = det
img_sel = (img_diff > 0).as_1d()
gain_map = flex.double(img_diff.accessor(), 0)
gain_map.as_1d().set_selected(img_sel.iselection(),
1 / img_diff.as_1d().select(img_sel))
gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
if not save_address:
address = None
d = cspad_tbx.dpack(data=gain_map,
address=address,
active_areas=active_areas,
timestamp=timestamp,
distance=command_line.options.distance,
wavelength=command_line.options.wavelength,
beam_center_x=beam_center_x,
beam_center_y=beam_center_y)
easy_pickle.dump(output_filename, d)
class SingleImage(object):
def __init__(self, img, init, verbose=True, imported_grid=None):
""" Constructor for the SingleImage object using a raw image file or pickle
"""
# Initialize parameters
self.params = init.params
self.args = init.args
self.user_id = init.user_id
self.raw_img = img[2]
self.conv_img = img[2]
self.img_index = img[0]
self.center_int = 0
self.status = None
self.fail = None
self.final = None
self.log_info = []
self.gs_results = []
self.main_log = init.logfile
self.verbose = verbose
self.hmed = self.params.cctbx.grid_search.height_median
self.amed = self.params.cctbx.grid_search.area_median
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.viz_base = init.viz_base
self.log_base = init.log_base
self.tmp_base = init.tmp_base
self.abort_file = os.path.join(self.int_base, '.abort.tmp')
self.obj_path = None
self.obj_file = None
self.fin_path = None
self.fin_file = None
self.viz_path = None
# ============================== SELECTION-ONLY FUNCTIONS ============================== #
def import_int_file(self, init):
""" Replaces path settings in imported image object with new settings
NEED TO RE-DO LATER """
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Generate paths to output files
self.params = init.params
self.main_log = init.logfile
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.log_base = init.log_base
self.viz_base = init.viz_base
self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
self.obj_base)
self.obj_file = os.path.abspath(
os.path.join(
self.obj_path,
os.path.basename(self.conv_img).split('.')[0] + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base,
self.fin_base)
self.log_path = misc.make_image_path(self.conv_img, self.input_base,
self.log_base)
self.fin_file = os.path.abspath(
os.path.join(
self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.pickle"))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.viz_path = misc.make_image_path(self.conv_img, self.input_base,
self.viz_base)
self.viz_file = os.path.join(
self.viz_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.png")
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Grid search / integration log file
self.int_log = os.path.join(
self.log_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
# Reset status to 'grid search' to pick up at selection (if no fail)
if self.fail == None:
self.status = 'bypass grid search'
return self
def determine_gs_result_file(self):
""" For 'selection-only' cctbx.xfel runs, determine where the image objects are """
if self.params.cctbx.selection.select_only.grid_search_path != None:
obj_path = os.path.abspath(
self.params.cctbx.selection.select_only.grid_search_path)
else:
run_number = int(os.path.basename(self.int_base)) - 1
obj_path = "{}/integration/{:03d}/image_objects"\
"".format(os.path.abspath(os.curdir), run_number)
gs_result_file = os.path.join(obj_path,
os.path.basename(self.obj_file))
return gs_result_file
# =============================== IMAGE IMPORT FUNCTIONS =============================== #
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except Exception, e:
print 'IOTA IMPORT ERROR:', e
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
img_type = 'pickle'
else:
img_type = 'raw'
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
else:
data = None
img_type = 'not imported'
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
from dxtbx.datablock import DataBlockFactory
from dials.command_line.estimate_gain import estimate_gain
with misc.Capturing() as junk_output:
try:
datablock = DataBlockFactory.from_filenames([self.raw_img
])[0]
imageset = datablock.extract_imagesets()[0]
self.gain = estimate_gain(imageset)
except Exception, e:
self.gain = 1.0
def run(argv=None):
if argv is None:
argv = sys.argv[1:]
command_line = (libtbx.option_parser.option_parser(
usage=
"%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files"
% libtbx.env.dispatcher_name
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--crop",
"-c",
action="store_true",
default=False,
dest="crop",
help="Crop the image such that the beam center is in the middle"
).option(
None,
"--skip_converted",
"-s",
action="store_true",
default=False,
dest="skip_converted",
help=
"Skip converting if an image already exist that matches the destination file name"
).option(
None,
"--wavelength",
"-w",
type="float",
default=None,
dest="wavelength",
help="Override the image's wavelength (angstroms)").option(
None,
"--distance",
"-d",
type="float",
default=None,
dest="distance",
help="Override the detector distance (mm)").option(
None,
"--pixel_size",
"-p",
type="float",
default=None,
dest="pixel_size",
help="Override the detector pixel size (mm)").option(
None,
"--beam_x",
"-x",
type="float",
default=None,
dest="beam_center_x",
help="Override the beam x position (pixels)").option(
None,
"--beam_y",
"-y",
type="float",
default=None,
dest="beam_center_y",
help="Override the beam y position (pixels)").option(
None,
"--overload",
"-o",
type="float",
default=None,
dest="overload",
help="Override the detector overload value (ADU)")
).process(args=argv)
paths = command_line.args
if len(paths) <= 0:
raise Usage("No files specified")
for imgpath in paths:
destpath = os.path.join(
os.path.dirname(imgpath),
os.path.splitext(os.path.basename(imgpath))[0] + ".pickle")
if command_line.options.skip_converted and os.path.isfile(destpath):
if command_line.options.verbose:
print "Skipping %s, file exists" % imgpath
continue
if command_line.options.verbose:
print "Converting %s to %s..." % (imgpath, destpath)
try:
img = dxtbx.load(imgpath)
except IOError:
img = None
pass
if img is None:
import numpy as np
try:
raw_data = np.loadtxt(imgpath)
from scitbx.array_family import flex
raw_data = flex.double(raw_data.astype(np.double))
except ValueError:
raise Usage("Couldn't load %s, no supported readers" % imgpath)
detector = None
beam = None
scan = None
else:
raw_data = img.get_raw_data()
detector = img.get_detector()
beam = img.get_beam()
scan = img.get_scan()
if detector is None:
if command_line.options.distance is None:
raise Usage("Can't get distance from image. Override with -d")
if command_line.options.pixel_size is None:
raise Usage(
"Can't get pixel size from image. Override with -p")
if command_line.options.overload is None:
raise Usage(
"Can't get overload value from image. Override with -o")
distance = command_line.options.distance
pixel_size = command_line.options.pixel_size
overload = command_line.options.overload
else:
detector = detector[0]
if command_line.options.distance is None:
distance = detector.get_directed_distance()
else:
distance = command_line.options.distance
if command_line.options.pixel_size is None:
pixel_size = detector.get_pixel_size()[0]
else:
pixel_size = command_line.options.pixel_size
if command_line.options.overload is None:
overload = detector.get_trusted_range()[1]
else:
overload = command_line.options.overload
if beam is None:
if command_line.options.wavelength is None:
raise Usage(
"Can't get wavelength from image. Override with -w")
wavelength = command_line.options.wavelength
else:
if command_line.options.wavelength is None:
wavelength = beam.get_wavelength()
else:
wavelength = command_line.options.wavelength
if beam is None and detector is None:
if command_line.options.beam_center_x is None:
print "Can't get beam x position from image. Using image center. Override with -x"
beam_x = raw_data.focus()[0] * pixel_size
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
print "Can't get beam y position from image. Using image center. Override with -y"
beam_y = raw_data.focus()[1] * pixel_size
else:
beam_y = command_line.options.beam_center_y * pixel_size
else:
if command_line.options.beam_center_x is None:
beam_x = detector.get_beam_centre(beam.get_s0())[0]
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
beam_y = detector.get_beam_centre(beam.get_s0())[1]
else:
beam_y = command_line.options.beam_center_y * pixel_size
if scan is None:
timestamp = None
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = osc_start
data['OSC_RANGE'] = osc_range
data['TIME'] = scan.get_exposure_times()[0]
if command_line.options.crop:
data = crop_image_pickle(data)
easy_pickle.dump(destpath, data)
def endjob(self, obj1, obj2=None):
"""The endjob() function writes the mean and standard deviation images
to disk.
@param evt Event object (psana only)
@param env Environment object
"""
if obj2 is None:
env = obj1
else:
evt = obj1
env = obj2
stats = super(mod_average, self).endjob(env)
if stats is None:
return
device = cspad_tbx.address_split(self.address)[2]
if device == 'Andor':
beam_center = (0, 0) # XXX Fiction!
pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
saturated_value = 10000
elif device == 'Cspad' or device == 'Cspad2x2':
beam_center = self.beam_center
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
beam_center = tuple(t // 2 for t in d['mean_img'].focus())
pixel_size = 0.079346
saturated_value = 2**16 - 1
if stats['nmemb'] > 0:
if self.avg_dirname is not None or \
self.avg_basename is not None or \
self._mean_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['mean_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._mean_out is not None:
p = cspad_tbx.dwritef2(d, self._mean_out)
else:
p = cspad_tbx.dwritef(d, self.avg_dirname,
self.avg_basename)
self.logger.info("Average written to %s" % p)
if self.stddev_dirname is not None or \
self.stddev_basename is not None or \
self._std_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['std_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._std_out is not None:
p = cspad_tbx.dwritef2(d, self._std_out)
else:
p = cspad_tbx.dwritef(d, self.stddev_dirname,
self.stddev_basename)
self.logger.info("Standard deviation written to %s" % p)
if self.max_dirname is not None or \
self.max_basename is not None or \
self._max_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['max_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._max_out is not None:
p = cspad_tbx.dwritef2(d, self._max_out)
else:
p = cspad_tbx.dwritef(d, self.max_dirname,
self.max_basename)
self.logger.info("Max written to %s" % p)
if stats['nfail'] == 0:
self.logger.info("%d images processed" % stats['nmemb'])
else:
self.logger.warning("%d images processed, %d failed" %
(stats['nmemb'], stats['nfail']))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_image_dict, self).event(evt, env)
if (evt.get("skip_event")):
return
if self.cspad_img is None:
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
if distance == 0:
distance = evt.get("marccd_distance")
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
evt.put(d, self.m_out_key)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))