def _detector(self):
import psana
self._env = self._ds.env()
self._det = psana.Detector(self._src,self._env)
return self._detector_factory.simple(
sensor = 'UNKNOWN',
distance = 100.0,
beam_centre = (50., 50.),
fast_direction = '+x',
slow_direction = '-y',
pixel_size = (rayonix_tbx.get_rayonix_pixel_size(2), rayonix_tbx.get_rayonix_pixel_size(2)),
image_size = self._det.shape(),
trusted_range = (rayonix_tbx.rayonix_min_trusted_value, rayonix_tbx.rayonix_saturated_value),
mask = [])
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_image_dict, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
if distance == 0:
distance = evt.get("marccd_distance")
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
evt.put(d, self.m_out_key)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition. It
outputs the detector image associated with the event @p evt to the
file system.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_dump, self).event(evt, env)
if (evt.get('skip_event')):
return
if self.cspad_img is None:
print "No image to save for %s"%self.timestamp
return
# Where the sample-detector distance is not available, set it to
# zero.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
distance = 0
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
output_filename = self._basename
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
output_filename = self._basename
elif device == 'marccd':
if distance == 0:
distance = evt.get('marccd_distance')
pixel_size = 0.079346
saturated_value = 2**16 - 1
output_filename = self._basename + evt.get(str, 'mccd_name') + "_"
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
if self._format == "pickle":
cspad_tbx.dwritef(d, self._dirname, output_filename)
elif self._format == "tiff":
cspad_tbx.write_tiff(d, self._dirname, output_filename)
output_filename = None
def event(self, evt, env):
"""The event() function is called for every L1Accept transition. It
outputs the detector image associated with the event @p evt to the
file system.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_dump, self).event(evt, env)
if (evt.get('skip_event')):
return
if self.cspad_img is None:
print("No image to save for %s"%self.timestamp)
return
# Where the sample-detector distance is not available, set it to
# zero.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
distance = 0
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
output_filename = self._basename
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
output_filename = self._basename
elif device == 'marccd':
if distance == 0:
distance = evt.get('marccd_distance')
pixel_size = 0.079346
saturated_value = 2**16 - 1
output_filename = self._basename + evt.get(str, 'mccd_name') + "_"
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
if self._format == "pickle":
cspad_tbx.dwritef(d, self._dirname, output_filename)
elif self._format == "tiff":
cspad_tbx.write_tiff(d, self._dirname, output_filename)
output_filename = None
def set_up_hitfinder(self):
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
img_dim = (1765, 1765)
pixel_size = cspad_tbx.pixel_size
elif device == 'marccd':
img_dim = (4500, 4500)
pixel_size = 0.079346
elif device == 'Rayonix':
img_dim = rayonix_tbx.get_rayonix_detector_dimensions(
self.bin_size)
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
else:
raise RuntimeError("Unsupported device %s" % self.address)
if self.beam_center is None:
self.beam_center = [0, 0]
self.hitfinder_d = cspad_tbx.dpack(
active_areas=self.active_areas,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
xtal_target=self.m_xtal_target)
if device == 'Cspad':
# Figure out which ASIC:s are on the central four sensors. This
# only applies to the CSPAD.
assert len(self.active_areas) % 4 == 0
distances = flex.double()
for i in range(0, len(self.active_areas), 4):
cenasic = (
(self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
(self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
distances.append(
math.hypot(cenasic[0] - self.beam_center[0],
cenasic[1] - self.beam_center[1]))
orders = flex.sort_permutation(distances)
# Use the central 8 ASIC:s (central 4 sensors).
flags = flex.int(len(self.active_areas) // 4, 0)
for i in range(8):
flags[orders[i]] = 1
self.asic_filter = "distl.tile_flags=" + ",".join(
["%1d" % b for b in flags])
elif device == 'marccd':
# There is only one active area for the MAR CCD, so use it.
self.asic_filter = "distl.tile_flags=1"
elif device == 'Rayonix':
# There is only one active area for the Rayonix, so use it.
self.asic_filter = "distl.tile_flags=1"
def __init__(self, image_file, **kwargs):
super().__init__(image_file, locator_scope=rayonix_locator_scope, **kwargs)
cfgs = self._ds.env().configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix"))
if self.params.rayonix.bin_size is None:
assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s()
bin_size = rayonix_cfg.binning_f()
else:
bin_size = self.params.rayonix.bin_size
self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size)
self._image_size = rayonix_tbx.get_rayonix_detector_dimensions(self._ds.env())
def _detector(self):
import psana
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.params.rayonix.bin_size)
return self._detector_factory.simple(
sensor = 'UNKNOWN',
distance = 100.0,
beam_centre = (50., 50.),
fast_direction = '+x',
slow_direction = '-y',
pixel_size = (pixel_size, pixel_size),
image_size = rayonix_tbx.get_rayonix_detector_dimensions(self.params.rayonix.bin_size),
trusted_range = (rayonix_tbx.rayonix_min_trusted_value, rayonix_tbx.rayonix_saturated_value),
mask = [])
def set_up_hitfinder(self):
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
img_dim = (1765, 1765)
pixel_size = cspad_tbx.pixel_size
elif device == 'marccd':
img_dim = (4500, 4500)
pixel_size = 0.079346
elif device == 'Rayonix':
img_dim = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
else:
raise RuntimeError("Unsupported device %s" % self.address)
if self.beam_center is None:
self.beam_center = [0,0]
self.hitfinder_d = cspad_tbx.dpack(
active_areas=self.active_areas,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=flex.int(flex.grid(img_dim[0], img_dim[1]), 0),
xtal_target=self.m_xtal_target)
if device == 'Cspad':
# Figure out which ASIC:s are on the central four sensors. This
# only applies to the CSPAD.
assert len(self.active_areas) % 4 == 0
distances = flex.double()
for i in range(0, len(self.active_areas), 4):
cenasic = ((self.active_areas[i + 0] + self.active_areas[i + 2]) / 2,
(self.active_areas[i + 1] + self.active_areas[i + 3]) / 2)
distances.append(math.hypot(cenasic[0] - self.beam_center[0],
cenasic[1] - self.beam_center[1]))
orders = flex.sort_permutation(distances)
# Use the central 8 ASIC:s (central 4 sensors).
flags = flex.int(len(self.active_areas) // 4, 0)
for i in range(8):
flags[orders[i]] = 1
self.asic_filter = "distl.tile_flags=" + ",".join(
["%1d" % b for b in flags])
elif device == 'marccd':
# There is only one active area for the MAR CCD, so use it.
self.asic_filter = "distl.tile_flags=1"
elif device == 'Rayonix':
# There is only one active area for the Rayonix, so use it.
self.asic_filter = "distl.tile_flags=1"
def __init__(self, params, run, detector, **kwargs):
from xfel.cxi.cspad_ana import rayonix_tbx
import psana
FormatXTCSingleEvent.__init__(self, params, run, detector, **kwargs)
self._cached_detector = {}
cfgs = run.env().configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix"))
if params.rayonix.bin_size is None:
assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s()
bin_size = rayonix_cfg.binning_f()
else:
bin_size = params.rayonix.bin_size
self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size)
self._image_size = rayonix_tbx.get_rayonix_detector_dimensions(
run.env())
def __init__(self, image_file, **kwargs):
super(FormatXTCRayonix, self).__init__(
image_file, locator_scope=rayonix_locator_scope, **kwargs
)
self._ds = FormatXTCRayonix._get_datasource(image_file, self.params)
self._env = self._ds.env()
self.populate_events()
self.n_images = len(self.times)
cfgs = self._ds.env().configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source("Rayonix"))
if self.params.rayonix.bin_size is None:
assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s()
bin_size = rayonix_cfg.binning_f()
else:
bin_size = self.params.rayonix.bin_size
self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(bin_size)
self._image_size = rayonix_tbx.get_rayonix_detector_dimensions(self._ds.env())
def __init__(self, image_file, **kwargs):
import psana
assert (self.understand(image_file))
FormatXTC.__init__(self,
image_file,
locator_scope=rayonix_locator_scope,
**kwargs)
self._ds = FormatXTCRayonix._get_datasource(image_file, self.params)
self._env = self._ds.env()
self.populate_events()
self.n_images = len(self.times)
cfgs = self._ds.env().configStore()
rayonix_cfg = cfgs.get(psana.Rayonix.ConfigV2, psana.Source('Rayonix'))
if self.params.rayonix.bin_size is None:
assert rayonix_cfg.binning_f() == rayonix_cfg.binning_s()
self._bin_size = rayonix_cfg.binning_f()
else:
self._bin_size = self.params.rayonix.bin_size
self._pixel_size = rayonix_tbx.get_rayonix_pixel_size(self._bin_size)
self._image_size = rayonix_cfg.width(), rayonix_cfg.height()
def submit_job(app, job):
import os, libtbx.load_env
from xfel.ui import settings_dir
configs_dir = os.path.join(settings_dir, "cfgs")
if not os.path.exists(configs_dir):
os.makedirs(configs_dir)
target_phil_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d_params.phil" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
dispatcher = app.params.dispatcher
phil_str = job.trial.target_phil_str
if job.rungroup.extra_phil_str is not None:
phil_str += "\n" + job.rungroup.extra_phil_str
from xfel.ui import known_dials_dispatchers
if dispatcher in known_dials_dispatchers:
import importlib
orig_phil_scope = importlib.import_module(
known_dials_dispatchers[dispatcher]).phil_scope
from iotbx.phil import parse
if job.rungroup.two_theta_low is not None or job.rungroup.two_theta_high is not None:
override_str = """
radial_average {
enable = True
show_plots = False
verbose = False
output_bins = False
}
"""
phil_scope = orig_phil_scope.fetch(parse(override_str))
else:
phil_scope = orig_phil_scope
trial_params = phil_scope.fetch(parse(phil_str)).extract()
image_format = job.rungroup.format
assert image_format in ['cbf', 'pickle']
if image_format == 'cbf':
if "rayonix" in job.rungroup.detector_address.lower():
mode = "rayonix"
elif "cspad" in job.rungroup.detector_address.lower():
mode = "cspad"
elif "jungfrau" in job.rungroup.detector_address.lower():
mode = "jungfrau"
else:
assert False, "Couldn't figure out what kind of detector is specified by address %s" % job.rungroup.detector_address
if dispatcher == 'cctbx.xfel.xtc_process':
trial_params.format.file_format = image_format
trial_params.format.cbf.mode = mode
elif dispatcher == "cxi.xtc_process":
image_format = 'pickle'
else:
raise RuntimeError("Unknown dispatcher: %s" % dispatcher)
if job.rungroup.calib_dir is not None or job.rungroup.config_str is not None or dispatcher == 'cxi.xtc_process' or image_format == 'pickle':
config_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d.cfg" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
else:
config_path = None
# Dictionary for formating the submit phil and, if used, the labelit cfg file
d = dict(
# Generally for the LABELIT backend or image pickles
address=job.rungroup.detector_address,
default_calib_dir=libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"),
dark_avg_path=job.rungroup.dark_avg_path,
dark_stddev_path=job.rungroup.dark_stddev_path,
untrusted_pixel_mask_path=job.rungroup.untrusted_pixel_mask_path,
detz_parameter=job.rungroup.detz_parameter,
gain_map_path=job.rungroup.gain_map_path,
gain_mask_level=job.rungroup.gain_mask_level,
beamx=job.rungroup.beamx,
beamy=job.rungroup.beamy,
energy=job.rungroup.energy,
binning=job.rungroup.binning,
two_theta_low=job.rungroup.two_theta_low,
two_theta_high=job.rungroup.two_theta_high,
# Generally for job submission
dry_run=app.params.dry_run,
dispatcher=app.params.dispatcher,
cfg=config_path,
experiment=app.params.experiment,
run_num=job.run.run,
output_dir=app.params.output_folder,
use_ffb=app.params.use_ffb,
# Generally for both
trial=job.trial.trial,
rungroup=job.rungroup.rungroup_id,
experiment_tag=app.params.experiment_tag,
calib_dir=job.rungroup.calib_dir,
nproc=app.params.mp.nproc,
queue=app.params.mp.queue,
target=target_phil_path,
host=app.params.db.host,
dbname=app.params.db.name,
user=app.params.db.user,
)
if app.params.db.password is not None and len(app.params.db.password) == 0:
d['password'] = None
else:
d['password'] = app.params.db.password
phil = open(target_phil_path, "w")
if dispatcher == 'cxi.xtc_process':
phil.write(phil_str)
elif dispatcher in known_dials_dispatchers:
extra_scope = None
if dispatcher == 'cctbx.xfel.xtc_process':
if image_format == "cbf":
trial_params.input.address = job.rungroup.detector_address
trial_params.format.cbf.detz_offset = job.rungroup.detz_parameter
trial_params.format.cbf.override_energy = job.rungroup.energy
trial_params.format.cbf.invalid_pixel_mask = job.rungroup.untrusted_pixel_mask_path
if mode == 'cspad':
trial_params.format.cbf.cspad.gain_mask_value = job.rungroup.gain_mask_level
elif mode == 'rayonix':
trial_params.format.cbf.rayonix.bin_size = job.rungroup.binning
trial_params.format.cbf.rayonix.override_beam_x = job.rungroup.beamx
trial_params.format.cbf.rayonix.override_beam_y = job.rungroup.beamy
trial_params.dispatch.process_percent = job.trial.process_percent
if trial_params.input.known_orientations_folder is not None:
trial_params.input.known_orientations_folder = trial_params.input.known_orientations_folder.format(
run=job.run.run)
else:
trial_params.spotfinder.lookup.mask = job.rungroup.untrusted_pixel_mask_path
trial_params.integration.lookup.mask = job.rungroup.untrusted_pixel_mask_path
locator_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d.loc" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
locator = open(locator_path, 'w')
locator.write("experiment=%s\n" % app.params.experiment)
locator.write("run=%d\n" % job.run.run)
locator.write("detector_address=%s\n" %
job.rungroup.detector_address)
if image_format == "cbf":
if mode == 'rayonix':
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
job.rungroup.binning)
fast, slow = rayonix_tbx.get_rayonix_detector_dimensions(
job.rungroup.binning)
extra_scope = parse(
"geometry { detector { panel { origin = (%f, %f, %f) } } }"
%
(-job.rungroup.beamx * pixel_size, job.rungroup.beamy *
pixel_size, -job.rungroup.detz_parameter))
locator.write("rayonix.bin_size=%s\n" %
job.rungroup.binning)
locator.close()
d['locator'] = locator_path
if job.rungroup.two_theta_low is not None or job.rungroup.two_theta_high is not None:
try:
trial_params.radial_average.two_theta_low = job.rungroup.two_theta_low
trial_params.radial_average.two_theta_high = job.rungroup.two_theta_high
except AttributeError:
pass # not all dispatchers support radial averaging
working_phil = phil_scope.format(python_object=trial_params)
if extra_scope:
working_phil = working_phil.fetch(extra_scope)
diff_phil = orig_phil_scope.fetch_diff(source=working_phil)
phil.write(diff_phil.as_str())
phil.close()
if config_path is not None:
if dispatcher != 'cxi.xtc_process':
d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will
# will be used during dials processing directly
config_str = "[psana]\n"
if job.rungroup.calib_dir is not None:
config_str += "calib-dir=%s\n" % job.rungroup.calib_dir
modules = []
if job.rungroup.config_str is not None:
for line in job.rungroup.config_str.split("\n"):
if line.startswith('['):
modules.append(line.lstrip('[').rstrip(']'))
if dispatcher == 'cxi.xtc_process':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(
['my_ana_pkg.mod_hitfind:index', 'my_ana_pkg.mod_dump:index'])
elif image_format == 'pickle':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(['my_ana_pkg.mod_image_dict'])
if app.params.dump_shots:
modules.insert(0, 'my_ana_pkg.mod_dump:shot')
if len(modules) > 0:
config_str += "modules = %s\n" % (" ".join(modules))
if job.rungroup.config_str is not None:
config_str += job.rungroup.config_str + "\n"
if dispatcher == 'cxi.xtc_process' or image_format == 'pickle':
d['address'] = d['address'].replace('.', '-').replace(
':', '|') # old style address
if dispatcher == 'cxi.xtc_process':
template = open(
os.path.join(
libtbx.env.find_in_repositories("xfel/ui/db/cfgs"),
"index_all.cfg"))
elif image_format == 'pickle':
template = open(
os.path.join(
libtbx.env.find_in_repositories("xfel/ui/db/cfgs"),
"image_dict.cfg"))
for line in template.readlines():
config_str += line.format(**d)
template.close()
d['address'] = job.rungroup.detector_address
cfg = open(config_path, 'w')
cfg.write(config_str)
cfg.close()
if dispatcher != 'cxi.xtc_process':
d['untrusted_pixel_mask_path'] = job.rungroup.untrusted_pixel_mask_path
submit_phil_path = os.path.join(
configs_dir, "%s_%s_r%04d_t%03d_rg%03d_submit.phil" %
(app.params.experiment, app.params.experiment_tag, job.run.run,
job.trial.trial, job.rungroup.id))
submit_root = libtbx.env.find_in_repositories("xfel/ui/db/cfgs")
if dispatcher in ['cxi.xtc_process', 'cctbx.xfel.xtc_process']:
template = open(os.path.join(submit_root, "submit_xtc_process.phil"))
elif dispatcher == 'cctbx.xfel.process':
template = open(os.path.join(submit_root, "submit_xfel_process.phil"))
else:
test_root = os.path.join(submit_root, "submit_" + dispatcher + ".phil")
if os.path.exists(test_root):
template = open(test_root)
else:
template = open(os.path.joins(submit_root, "submit.phil"))
phil = open(submit_phil_path, "w")
if dispatcher == 'cxi.xtc_process':
d['target'] = None # any target phil will be in mod_hitfind
for line in template.readlines():
phil.write(line.format(**d))
d['target'] = target_phil_path
template.close()
phil.close()
from xfel.command_line.cxi_mpi_submit import Script as submit_script
return submit_script().run([submit_phil_path])
destroot_n,
os.path.splitext(picklename)[0] + "_n.pickle")
#if os.path.exists(destpath_l): continue
try:
data = easy_pickle.load(picklepath)
except Exception, e:
print "Pickle failed to load", picklepath
continue
if not "fuller_kapton_absorption_correction" in data:
continue
corr = data["fuller_kapton_absorption_correction"]
from xfel.cxi.cspad_ana.rayonix_tbx import get_rayonix_pixel_size
from scitbx.array_family import flex
pixel_size = get_rayonix_pixel_size(2)
bx = data['xbeam'] / pixel_size
by = data['ybeam'] / pixel_size
preds = data['mapped_predictions']
sel_l = []
sel_r = []
sel_mid = []
sel_nomid = []
all_good = True
for i in xrange(len(preds)):
# all preds left of the beam center
p1_sel = preds[i].parts()[1] < bx
# mostly will be preds right of the beam center, but includes a few to the left of middle strip
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_radial_average, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = 0.079346
saturated_value = 2**16 - 1
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
from xfel.command_line.radial_average import run
args = [
"file_path=XTC stream",
"xfel_target=%s" % self.m_xtal_target, "verbose=False"
]
t = self.timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
20:23]
if self._dirname is not None:
dest_path = os.path.join(self._dirname,
self._basename + s + ".txt")
args.append("output_file=%s" % dest_path)
self.logger.info("Calculating radial average for image %s" % s)
xvals, results = run(args, d)
evt.put(xvals, "cctbx.xfel.radial_average.xvals")
evt.put(results, "cctbx.xfel.radial_average.results")
def get_closest_idx(data, val):
from scitbx.array_family import flex
deltas = flex.abs(data - val)
return flex.first_index(deltas, flex.min(deltas))
if self._two_theta_low is not None:
i_low = results[get_closest_idx(xvals, self._two_theta_low)]
evt.put(i_low, "cctbx.xfel.radial_average.two_theta_low")
if self._two_theta_high is not None:
i_high = results[get_closest_idx(xvals, self._two_theta_high)]
evt.put(i_high, "cctbx.xfel.radial_average.two_theta_high")
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_image_dict, self).event(evt, env)
if (evt.get("skip_event")):
return
if self.cspad_img is None:
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
if distance == 0:
distance = evt.get("marccd_distance")
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
evt.put(d, self.m_out_key)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
destpath_m = os.path.join(destroot_m, os.path.splitext(picklename)[0] + "_m.pickle")
destpath_n = os.path.join(destroot_n, os.path.splitext(picklename)[0] + "_n.pickle")
#if os.path.exists(destpath_l): continue
try:
data = easy_pickle.load(picklepath)
except Exception, e:
print "Pickle failed to load", picklepath
continue
if not "applied_absorption_correction" in data:
continue
corr = data["applied_absorption_correction"]
from xfel.cxi.cspad_ana.rayonix_tbx import get_rayonix_pixel_size
from scitbx.array_family import flex
pixel_size = get_rayonix_pixel_size(2)
bx = data['xbeam'] / pixel_size
by = data['ybeam'] / pixel_size
preds = data['mapped_predictions']
sel_l = []
sel_r = []
sel_mid = []
sel_nomid = []
all_good = True
for i in xrange(len(preds)):
# all preds left of the beam center
p1_sel = preds[i].parts()[1] < bx
# mostly will be preds right of the beam center, but includes a few to the left of middle strip
def submit(self, previous_job = None):
import libtbx.load_env
configs_dir = os.path.join(settings_dir, "cfgs")
if not os.path.exists(configs_dir):
os.makedirs(configs_dir)
identifier_string = self.get_identifier_string()
target_phil_path = os.path.join(configs_dir, identifier_string + "_params.phil")
dispatcher = self.app.params.dispatcher
phil_str = self.trial.target_phil_str
if phil_str is None: phil_str = ""
if self.rungroup.extra_phil_str is not None:
phil_str += "\n" + self.rungroup.extra_phil_str
from xfel.ui import load_phil_scope_from_dispatcher
if dispatcher == "cxi.xtc_process":
image_format = 'pickle'
else:
orig_phil_scope = load_phil_scope_from_dispatcher(dispatcher)
if os.path.isfile(dispatcher):
dispatcher = 'libtbx.python ' + dispatcher
from iotbx.phil import parse
if self.rungroup.two_theta_low is not None or self.rungroup.two_theta_high is not None:
override_str = """
radial_average {
enable = True
show_plots = False
verbose = False
output_bins = False
mask = %s
}
"""%(self.rungroup.untrusted_pixel_mask_path)
phil_scope = orig_phil_scope.fetch(parse(override_str))
else:
phil_scope = orig_phil_scope
trial_params = phil_scope.fetch(parse(phil_str)).extract()
image_format = self.rungroup.format
mode = "other"
if self.app.params.facility.name == 'lcls':
if "rayonix" in self.rungroup.detector_address.lower():
mode = "rayonix"
elif "cspad" in self.rungroup.detector_address.lower():
mode = "cspad"
elif "jungfrau" in self.rungroup.detector_address.lower():
mode = "jungfrau"
if hasattr(trial_params, 'format'):
trial_params.format.file_format = image_format
trial_params.format.cbf.mode = mode
if self.rungroup.calib_dir is not None or self.rungroup.config_str is not None or dispatcher == 'cxi.xtc_process' or image_format == 'pickle':
config_path = os.path.join(configs_dir, identifier_string + ".cfg")
else:
config_path = None
if hasattr(trial_params.dispatch, 'process_percent'):
trial_params.dispatch.process_percent = self.trial.process_percent
# Dictionary for formating the submit phil and, if used, the labelit cfg file
d = dict(
# Generally for the LABELIT backend or image pickles
address = self.rungroup.detector_address,
default_calib_dir = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0"),
dark_avg_path = self.rungroup.dark_avg_path,
dark_stddev_path = self.rungroup.dark_stddev_path,
untrusted_pixel_mask_path = self.rungroup.untrusted_pixel_mask_path,
detz_parameter = self.rungroup.detz_parameter,
gain_map_path = self.rungroup.gain_map_path,
gain_mask_level = self.rungroup.gain_mask_level,
beamx = self.rungroup.beamx,
beamy = self.rungroup.beamy,
energy = self.rungroup.energy,
binning = self.rungroup.binning,
two_theta_low = self.rungroup.two_theta_low,
two_theta_high = self.rungroup.two_theta_high,
# Generally for job submission
dry_run = self.app.params.dry_run,
dispatcher = dispatcher,
cfg = config_path,
experiment = self.app.params.facility.lcls.experiment, # LCLS specific parameter
run_num = self.run.run,
output_dir = self.app.params.output_folder,
use_ffb = self.app.params.facility.lcls.use_ffb, # LCLS specific parameter
# Generally for both
trial = self.trial.trial,
rungroup = self.rungroup.rungroup_id,
experiment_tag = self.app.params.experiment_tag,
calib_dir = self.rungroup.calib_dir,
nproc = self.app.params.mp.nproc,
nnodes = self.app.params.mp.nnodes_index or self.app.params.mp.nnodes,
nproc_per_node = self.app.params.mp.nproc_per_node,
queue = self.app.params.mp.queue or None,
env_script = self.app.params.mp.env_script[0] if self.app.params.mp.env_script is not None and len(self.app.params.mp.env_script) > 0 and len(self.app.params.mp.env_script[0]) > 0 else None,
phenix_script = self.app.params.mp.phenix_script[0] if self.app.params.mp.phenix_script is not None and len(self.app.params.mp.phenix_script) > 0 and len(self.app.params.mp.phenix_script[0]) > 0 else None,
method = self.app.params.mp.method,
wall_time = self.app.params.mp.wall_time,
htcondor_executable_path = self.app.params.mp.htcondor.executable_path,
nersc_shifter_image = self.app.params.mp.shifter.shifter_image,
sbatch_script_template = self.app.params.mp.shifter.sbatch_script_template,
srun_script_template = self.app.params.mp.shifter.srun_script_template,
nersc_partition = self.app.params.mp.shifter.partition,
nersc_jobname = self.app.params.mp.shifter.jobname,
nersc_project = self.app.params.mp.shifter.project,
nersc_constraint = self.app.params.mp.shifter.constraint,
nersc_reservation = self.app.params.mp.shifter.reservation,
nersc_staging = self.app.params.mp.shifter.staging,
target = target_phil_path,
host = self.app.params.db.host,
dbname = self.app.params.db.name,
user = self.app.params.db.user,
port = self.app.params.db.port,
# always use mpi for 'lcls'
use_mpi = self.app.params.mp.method != 'local' or (self.app.params.mp.method == 'local' and self.app.params.facility.name == 'lcls'),
mpi_command = self.app.params.mp.mpi_command,
extra_options = "\n".join(["extra_options = %s"%opt for opt in self.app.params.mp.extra_options]),
)
if self.app.params.mp.method == 'sge':
d['use_mpi'] = False
if self.app.params.db.password is not None and len(self.app.params.db.password) == 0:
d['password'] = None
else:
d['password'] = self.app.params.db.password
phil = open(target_phil_path, "w")
if dispatcher == 'cxi.xtc_process':
phil.write(phil_str)
else:
extra_scope = None
if hasattr(trial_params, 'format'):
if image_format == "cbf":
trial_params.input.address = self.rungroup.detector_address
trial_params.format.cbf.detz_offset = self.rungroup.detz_parameter
trial_params.format.cbf.override_energy = self.rungroup.energy
trial_params.format.cbf.invalid_pixel_mask = self.rungroup.untrusted_pixel_mask_path
if mode == 'cspad':
trial_params.format.cbf.cspad.gain_mask_value = self.rungroup.gain_mask_level
elif mode == 'rayonix':
trial_params.format.cbf.rayonix.bin_size = self.rungroup.binning
trial_params.format.cbf.rayonix.override_beam_x = self.rungroup.beamx
trial_params.format.cbf.rayonix.override_beam_y = self.rungroup.beamy
if trial_params.input.known_orientations_folder is not None:
trial_params.input.known_orientations_folder = trial_params.input.known_orientations_folder.format(run=self.run.run)
else:
if trial_params.spotfinder.lookup.mask is None:
trial_params.spotfinder.lookup.mask = self.rungroup.untrusted_pixel_mask_path
if trial_params.integration.lookup.mask is None:
trial_params.integration.lookup.mask = self.rungroup.untrusted_pixel_mask_path
if self.app.params.facility.name == 'lcls':
locator_path = os.path.join(configs_dir, identifier_string + ".loc")
locator = open(locator_path, 'w')
locator.write("experiment=%s\n"%self.app.params.facility.lcls.experiment) # LCLS specific parameter
locator.write("run=%s\n"%self.run.run)
locator.write("detector_address=%s\n"%self.rungroup.detector_address)
if self.rungroup.wavelength_offset:
locator.write("wavelength_offset=%s\n"%self.rungroup.wavelength_offset)
if self.rungroup.spectrum_eV_per_pixel:
locator.write("spectrum_eV_per_pixel=%s\n"%self.rungroup.spectrum_eV_per_pixel)
if self.rungroup.spectrum_eV_offset:
locator.write("spectrum_eV_offset=%s\n"%self.rungroup.spectrum_eV_offset)
if self.app.params.facility.lcls.use_ffb:
locator.write("use_ffb=True\n")
if mode == 'rayonix':
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.rungroup.binning)
extra_scope = parse("geometry { detector { panel { origin = (%f, %f, %f) } } }"%(-self.rungroup.beamx * pixel_size,
self.rungroup.beamy * pixel_size,
-self.rungroup.detz_parameter))
locator.write("rayonix.bin_size=%s\n"%self.rungroup.binning)
elif mode == 'cspad':
locator.write("cspad.detz_offset=%s\n"%self.rungroup.detz_parameter)
locator.close()
d['locator'] = locator_path
else:
d['locator'] = None
if self.rungroup.two_theta_low is not None or self.rungroup.two_theta_high is not None:
try:
trial_params.radial_average.two_theta_low = self.rungroup.two_theta_low
trial_params.radial_average.two_theta_high = self.rungroup.two_theta_high
except AttributeError:
pass # not all dispatchers support radial averaging
working_phil = phil_scope.format(python_object=trial_params)
if extra_scope:
working_phil = working_phil.fetch(extra_scope)
diff_phil = orig_phil_scope.fetch_diff(source=working_phil)
phil.write(diff_phil.as_str())
phil.close()
if config_path is not None:
if dispatcher != 'cxi.xtc_process':
d['untrusted_pixel_mask_path'] = None # Don't pass a pixel mask to mod_image_dict as it will
# will be used during dials processing directly
config_str = "[psana]\n"
if self.rungroup.calib_dir is not None:
config_str += "calib-dir=%s\n"%self.rungroup.calib_dir
modules = []
if self.rungroup.config_str is not None:
for line in self.rungroup.config_str.split("\n"):
if line.startswith('['):
modules.append(line.lstrip('[').rstrip(']'))
if dispatcher == 'cxi.xtc_process':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(['my_ana_pkg.mod_hitfind:index','my_ana_pkg.mod_dump:index'])
elif image_format == 'pickle':
modules.insert(0, 'my_ana_pkg.mod_radial_average')
modules.extend(['my_ana_pkg.mod_image_dict'])
if self.app.params.facility.lcls.dump_shots:
modules.insert(0, 'my_ana_pkg.mod_dump:shot')
if len(modules) > 0:
config_str += "modules = %s\n"%(" ".join(modules))
if self.rungroup.config_str is not None:
config_str += self.rungroup.config_str + "\n"
if dispatcher == 'cxi.xtc_process' or image_format == 'pickle':
d['address'] = d['address'].replace('.','-').replace(':','|') # old style address
if dispatcher == 'cxi.xtc_process':
template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "index_all.cfg"))
elif image_format == 'pickle':
template = open(os.path.join(libtbx.env.find_in_repositories("xfel/ui/db/cfgs"), "image_dict.cfg"))
for line in template.readlines():
config_str += line.format(**d)
template.close()
d['address'] = self.rungroup.detector_address
cfg = open(config_path, 'w')
cfg.write(config_str)
cfg.close()
if dispatcher != 'cxi.xtc_process':
d['untrusted_pixel_mask_path'] = self.rungroup.untrusted_pixel_mask_path
submit_phil_path = os.path.join(configs_dir, identifier_string + "_submit.phil")
submit_root = libtbx.env.find_in_repositories("xfel/ui/db/cfgs")
if dispatcher in ['cxi.xtc_process', 'cctbx.xfel.xtc_process']:
template = open(os.path.join(submit_root, "submit_xtc_process.phil"))
else:
test_root = os.path.join(submit_root, "submit_" + dispatcher + ".phil")
if os.path.exists(test_root):
template = open(test_root)
else:
if hasattr(trial_params, 'format'):
template = open(os.path.join(submit_root, "submit_xtc_process.phil"))
else:
template = open(os.path.join(submit_root, "submit_xfel_process.phil"))
phil = open(submit_phil_path, "w")
if dispatcher == 'cxi.xtc_process':
d['target'] = None # any target phil will be in mod_hitfind
for line in template.readlines():
phil.write(line.format(**d))
d['target'] = target_phil_path
template.close()
phil.close()
from xfel.command_line.cxi_mpi_submit import Script as submit_script
args = [submit_phil_path]
if self.app.params.facility.name not in ['lcls']:
args.append(self.run.path)
return submit_script().run(args)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights,outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info("Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(d,
integration_dirname = self.m_integration_dirname,
integration_basename = self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None
if indexed and self.m_progress_logging:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best["identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best["identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
if self.m_db_logging:
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead")
if (self.m_out_dirname is not None or
self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(
usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name)
.option(None, "--as_pickle", "-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files")
.option(None, "--config", "-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file")
.option(None, "--experiment", "-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)")
.option(None, "--run", "-r",
type="int",
default=None,
dest="run",
help="run number")
.option(None, "--address", "-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)")
.option(None, "--detz_offset", "-d",
type="float",
default=None,
dest="detz_offset",
help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)")
.option(None, "--outputdir", "-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files")
.option(None, "--averagebase", "-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help="Path to output average image without extension. String substitution allowed")
.option(None, "--stddevbase", "-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help="Path to output standard deviation image without extension. String substitution allowed")
.option(None, "--maxbase", "-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help="Path to output maximum projection image without extension. String substitution allowed")
.option(None, "--numevents", "-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all")
.option(None, "--skipevents", "-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--pickle-optical-metrology", "-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help="If writing pickle files, use the optical metrology in the experiment's calib directory")
.option(None, "--bin_size", "-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size")
.option(None, "--override_beam_x", "-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate")
.option(None, "--override_beam_y", "-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate")
.option(None, "--calib_dir", "-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory")
.option(None, "--xtc_dir", "-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory")
.option(None, "--use_ffb", "-f",
action="store_true",
default=False,
dest="use_ffb",
help="Use the fast feedback filesystem at LCLS. Only for the active experiment!")
).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxint
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s"%command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)'%address)
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
nevent = np.array([0.])
for run in ds.runs():
runnumber = run.run()
# list of all events
if command_line.options.skipevents > 0:
print "Skipping first %d events"%command_line.options.skipevents
times = run.times()[command_line.options.skipevents:]
nevents = min(len(times),maxevents)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0]
for i in xrange(len(mytimes)):
if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes)
evt = run.event(mytimes[i])
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address:
data = evt.get(Camera.FrameV1,src)
if data is None:
print "No data"
continue
data=data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations
if data is None:
print "No data"
continue
d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset)
if d is None:
print "No distance, skipping shot"
continue
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print "No wavelength, skipping shot"
continue
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print "No timestamp, skipping shot"
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1]/1000)
else:
timestamp = np.array([t[0] + (t[1]/1000)])
if 'sum' in locals():
sum+=data
else:
sum=np.array(data, copy=True)
if 'sumsq' in locals():
sumsq+=data*data
else:
sumsq=data*data
if 'maximum' in locals():
maximum=np.maximum(maximum,data)
else:
maximum=np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print "Synchronizing rank", rank
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent,totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum,sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq,sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum,maxall, op=MPI.MAX)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength,waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance,distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp,timeall)
if rank==0:
if size > 1:
print "Synchronized"
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env())]
dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths]
if 'Rayonix' in command_line.options.address:
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size)
beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y]
detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size)
active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from xfel.detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.calib_dir is not None:
metro_path = command_line.options.calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
maxall = flex.double(maxall.astype(np.float64))
else:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
maxall = flex.double(maxall.astype(np.float64))
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d"%run.run())
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights, outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights >
self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((
peak_heights > self.m_threshold).__and__(
outvalue == 0)).count(True)
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
self.stats_logger.info("BRAGG %.3f %d" %
(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info(
"Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info(
"Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time,
number_of_accepted_peaks, distance, self.sifoil,
self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(
d,
integration_dirname=self.m_integration_dirname,
integration_basename=self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None and hasattr(info, 'spotfinder_results')
if self.m_progress_logging:
if self.m_db_version == 'v1':
if indexed:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best[
"identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host,
self.m_db_name,
self.m_db_user,
self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best[
"identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[
info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,
self.m_db_experiment_tag,
self.m_trial_id,
self.m_rungroup_id,
evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best[
"identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
elif self.m_db_version == 'v2':
key_low = 'cctbx.xfel.radial_average.two_theta_low'
key_high = 'cctbx.xfel.radial_average.two_theta_high'
tt_low = evt.get(key_low)
tt_high = evt.get(key_high)
from xfel.ui.db.dxtbx_db import log_frame
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
sfspots = evt.get('sfspots')
if sfspots is None:
if info is None or not isinstance(info, int):
n_spots = 0
else:
n_spots = info
else:
n_spots = sfspots
if indexed:
known_setting = info.horizons_phil.known_setting
indexed_setting = info.organizer.info[
'best_integration']['counter']
if known_setting is None or known_setting == indexed_setting:
from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
c = construct_reflection_table_and_experiment_list(
info.last_saved_best,
None,
pixel_size,
proceed_without_image=True)
c.assemble_experiments()
c.assemble_reflections()
log_frame(c.experiment_list, c.reflections,
self.db_params, evt.run(), n_spots,
self.timestamp, tt_low, tt_high)
else:
print(
"Not logging %s, wrong bravais setting (expecting %d, got %d)"
% (self.timestamp, known_setting,
indexed_setting))
else:
log_frame(None, None, self.db_params, evt.run(),
n_spots, self.timestamp, tt_low, tt_high)
if self.m_db_logging:
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get(
'ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get(
'ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get(
'ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
).parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time, n_spots,
distance, self.sifoil, self.wavelength, indexed,
mosaic_bloc_rotation, mosaic_block_size,
ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%" % detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(
horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%s" %
detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation, pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(
horizons_phil, imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead"
)
if (self.m_out_dirname is not None
or self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
None,
"--as_pickle",
"-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files"
).option(
None,
"--raw_data",
"-R",
action="store_true",
default=False,
dest="raw_data",
help=
"Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
).option(
None,
"--background_pickle",
"-B",
default=None,
dest="background_pickle",
help=""
).option(
None,
"--config",
"-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file"
).option(
None,
"--experiment",
"-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)"
).option(
None,
"--run",
"-r",
type="int",
default=None,
dest="run",
help="run number"
).option(
None,
"--address",
"-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)"
).option(
None,
"--detz_offset",
"-d",
type="float",
default=None,
dest="detz_offset",
help=
"offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
).option(
None,
"--outputdir",
"-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files"
).option(
None,
"--averagebase",
"-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help=
"Path to output average image without extension. String substitution allowed"
).option(
None,
"--stddevbase",
"-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help=
"Path to output standard deviation image without extension. String substitution allowed"
).option(
None,
"--maxbase",
"-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help=
"Path to output maximum projection image without extension. String substitution allowed"
).option(
None,
"--numevents",
"-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all"
).option(
None,
"--skipevents",
"-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--pickle-optical-metrology",
"-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help=
"If writing pickle files, use the optical metrology in the experiment's calib directory"
).option(
None,
"--bin_size",
"-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size"
).option(
None,
"--override_beam_x",
"-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate"
).option(
None,
"--override_beam_y",
"-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate"
).option(
None,
"--calib_dir",
"-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory"
).option(
None,
"--pickle_calib_dir",
"-P",
type="string",
default=None,
dest="pickle_calib_dir",
metavar="PATH",
help=
"pickle calibration directory specification. Replaces --calib_dir functionality."
).option(
None,
"--xtc_dir",
"-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory"
).option(
None,
"--use_ffb",
"-f",
action="store_true",
default=False,
dest="use_ffb",
help=
"Use the fast feedback filesystem at LCLS. Only for the active experiment!"
).option(
None,
"--gain_mask_value",
"-g",
type="float",
default=None,
dest="gain_mask_value",
help=
"Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
).option(
None,
"--min",
None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection"
).option(
None,
"--minpath",
None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help=
"Path to output minimum image without extension. String substitution allowed"
)).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxsize
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
command_line.options.experiment[0:3],
command_line.options.experiment)
if command_line.options.calib_dir is not None:
psana.setOption('psana.calib-dir', command_line.options.calib_dir)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)' % address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(
command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print("Skipping first %d events" % command_line.options.skipevents)
elif "Rayonix" in command_line.options.address:
print("Skipping first image in the Rayonix detector"
) # Shuttering issue
command_line.options.skipevents = 1
for i, evt in enumerate(run.events()):
if i % size != rank: continue
if i < command_line.options.skipevents: continue
if i >= maxevents: break
if i % 10 == 0: print('Rank', rank, 'processing event', i)
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1, src)
if data is None:
print("No data")
continue
data = data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det,
evt,
use_default=False,
dark=False,
common_mode=None,
apply_gain_mask=False,
per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det,
evt,
use_default=True)
else:
data = get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=True,
gain_mask_value=command_line.options.
gain_mask_value,
per_pixel_gain=False)
if data is None:
print("No data")
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(),
command_line.options.detz_offset)
if d is None:
print("No distance, using distance",
command_line.options.detz_offset)
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print("No wavelength")
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print("No timestamp, skipping shot")
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1] / 1000)
else:
timestamp = np.array([t[0] + (t[1] / 1000)])
if 'sum' in locals():
sum += data
else:
sum = np.array(data, copy=True)
if 'sumsq' in locals():
sumsq += data * data
else:
sumsq = data * data
if 'maximum' in locals():
maximum = np.maximum(maximum, data)
else:
maximum = np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum = np.minimum(minimum, data)
else:
minimum = np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print("Synchronizing rank", rank)
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent, totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum, sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq, sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum, maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum, minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength, waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance, distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp, timeall)
if rank == 0:
if size > 1:
print("Synchronized")
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [
cspad_tbx.pathsubst(command_line.options.averagepath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
ds.env())
]
if command_line.options.do_minimum_projection:
dest_paths.append(
cspad_tbx.pathsubst(command_line.options.minpath + extension,
evt, ds.env()))
dest_paths = [
os.path.join(command_line.options.outputdir, path)
for path in dest_paths
]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
command_line.options.bin_size)
beam_center = [
command_line.options.override_beam_x,
command_line.options.override_beam_y
]
active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(address=old_style_address,
data=flex.double(data),
distance=distance,
pixel_size=0.1,
timestamp=timestamp,
wavelength=wavelength)
print("Saving", path)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(
evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
old_style_address,
reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.pickle_calib_dir is not None:
metro_path = command_line.options.pickle_calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
address,
round_to_int=False)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_radial_average, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = 0.079346
saturated_value = 2**16 - 1
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
from xfel.command_line.radial_average import run
args = [
"file_path=XTC stream",
"xfel_target=%s"%self.m_xtal_target,
"verbose=False"
]
t = self.timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]
if self._dirname is not None:
dest_path = os.path.join(self._dirname, self._basename + s + ".txt")
args.append("output_file=%s"%dest_path)
self.logger.info("Calculating radial average for image %s"%s)
xvals, results = run(args, d)
evt.put(xvals, "cctbx.xfel.radial_average.xvals")
evt.put(results, "cctbx.xfel.radial_average.results")
def get_closest_idx(data, val):
from scitbx.array_family import flex
deltas = flex.abs(data - val)
return flex.first_index(deltas, flex.min(deltas))
if self._two_theta_low is not None:
i_low = results[get_closest_idx(xvals, self._two_theta_low)]
evt.put(i_low, "cctbx.xfel.radial_average.two_theta_low")
if self._two_theta_high is not None:
i_high = results[get_closest_idx(xvals, self._two_theta_high)]
evt.put(i_high, "cctbx.xfel.radial_average.two_theta_high")
