def run(args):
work_params = get_phil(args)
try:
validate_phil(work_params)
except Sorry as s:
raise s
except Exception as s:
raise s
C = fit_translation4(work_params)
print "done constructor"
#parse a phil file from the args
for item in args:
if os.path.isfile(item):
from xfel.phil_preferences import load_cxi_phil
C.optional_params = load_cxi_phil(item, [])
break
print
C.run_cycle_a()
C.run_cycle_b(0)
print
C.run_cycle_a()
C.run_cycle_b(1)
print
C.run_cycle_a()
def distl_filter(self,
address,
cspad_img,
distance,
timestamp,
wavelength):
self.hitfinder_d["DATA"] = cspad_img
self.hitfinder_d["DISTANCE"] = distance
self.hitfinder_d["TIMESTAMP"] = timestamp
self.hitfinder_d["WAVELENGTH"] = wavelength
self.hitfinder_d["DETECTOR_ADDRESS"] = address
args = ["indexing.data=dummy",
"distl.bins.verbose=False",
self.asic_filter,
]
detector_format_version = detector_format_function(
address, reverse_timestamp(timestamp)[0])
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = self.hitfinder_d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
imgdata = info.Files.images[0].linearintdata
active_data = self.get_active_data(info.Files.images[0],horizons_phil)
peak_heights = flex.int( [
imgdata[ spot.max_pxl_x(), spot.max_pxl_y() ]
for spot in info.S.images[info.frames[0]]["spots_total"]
])
outscale = 256
corrected = peak_heights.as_double() * self.correction
outvalue = outscale *(1.0-corrected)
outvalue.set_selected(outvalue<0.0,0.)
outvalue.set_selected(outvalue>=outscale,int(outscale)-1)
outvalue = flex.int(outvalue.as_numpy_array().astype(numpy.int32))
# essentially, select a peak if the peak's ADU value is > 2.5 * the 90-percentile pixel value
#work = display_spots.wrapper_of_callback(info)
#work.display_with_callback(horizons_phil.indexing.data)
return peak_heights,outvalue
def distl_filter(self,
address,
cspad_img,
distance,
timestamp,
wavelength):
self.hitfinder_d["DATA"] = cspad_img
self.hitfinder_d["DISTANCE"] = distance
self.hitfinder_d["TIMESTAMP"] = timestamp
self.hitfinder_d["WAVELENGTH"] = wavelength
self.hitfinder_d["DETECTOR_ADDRESS"] = address
args = ["indexing.data=dummy",
"distl.bins.verbose=False",
self.asic_filter,
]
detector_format_version = detector_format_function(
address, reverse_timestamp(timestamp)[0])
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = self.hitfinder_d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
imgdata = info.Files.images[0].linearintdata
active_data = self.get_active_data(info.Files.images[0],horizons_phil)
peak_heights = flex.int( [
imgdata[ spot.max_pxl_x(), spot.max_pxl_y() ]
for spot in info.S.images[info.frames[0]]["spots_total"]
])
outscale = 256
corrected = peak_heights.as_double() * self.correction
outvalue = outscale *(1.0-corrected)
outvalue.set_selected(outvalue<0.0,0.)
outvalue.set_selected(outvalue>=outscale,int(outscale)-1)
outvalue = flex.int(outvalue.as_numpy_array().astype(numpy.int32))
# essentially, select a peak if the peak's ADU value is > 2.5 * the 90-percentile pixel value
#work = display_spots.wrapper_of_callback(info)
#work.display_with_callback(horizons_phil.indexing.data)
return peak_heights,outvalue
def integrate_one_image(data, **kwargs):
from xfel.cxi.display_spots import run_one_index_core
from labelit.dptbx.error import NoAutoIndex
from libtbx.utils import Sorry
from spotfinder.exception import SpotfinderError
from labelit.exception import AutoIndexError
from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
basename = kwargs.get("integration_basename")
if (basename is None):
basename = ""
dirname = kwargs.get("integration_dirname")
if (dirname is None):
dirname = "integration"
if (not os.path.isdir(dirname)):
import errno
try:
os.makedirs(dirname)
except OSError as exc:
if exc.errno==errno.EEXIST: pass
path = os.path.join(dirname, basename \
+ data['TIMESTAMP'] \
+ ("_%05d.pickle" % data['SEQUENCE_NUMBER']))
args = ["indexing.data=dummy",
"beam_search_scope=0.5",
"lepage_max_delta = 3.0",
"spots_pickle = None",
"subgroups_pickle = None",
"refinements_pickle = None",
"rmsd_tolerance = 5.0",
"mosflm_rmsd_tolerance = 5.0",
"indexing.completeness_pickle=%s"%path,
"difflimit_sigma_cutoff=2.0",
#"indexing.open_wx_viewer=True"
]
detector_format_version = detector_format_function(
data['DETECTOR_ADDRESS'], reverse_timestamp(data['TIMESTAMP'])[0])
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(data["xtal_target"], args)
horizons_phil.indexing.data = data
print "XFEL processing: %s"%path
try:
return run_one_index_core(horizons_phil)
except NoAutoIndex,e:
print "NoAutoIndex", data['TIMESTAMP'], e
info = e.info
def run_one_index(path, *arguments, **kwargs):
assert arguments[0].find("target=") == 0
target = arguments[0].split("=")[1]
from xfel.phil_preferences import load_cxi_phil
if "--nodisplay" in arguments[1:]:
display = False
arguments = list(arguments)
arguments.remove("--nodisplay")
else:
display = True
args = [
"indexing.data=%s" % path,
"beam_search_scope=0.5",
"lepage_max_delta = 3.0",
"spots_pickle = None",
"subgroups_pickle = None",
"refinements_pickle = None",
"rmsd_tolerance = 5.0",
"mosflm_rmsd_tolerance = 5.0",
"difflimit_sigma_cutoff=2.0",
#"indexing.verbose_cv=True",
"indexing.open_wx_viewer=%s" % display
] + list(arguments[1:])
horizons_phil = load_cxi_phil(target, args)
info = run_one_index_core(horizons_phil)
info.Files = info.organizer.Files
info.phil_params = info.horizons_phil
# The spotfinder view within cxi.index is an anachronism; no useful purpose anymore
# therefore remove this option within cxi.index:
return
work = wrapper_of_callback(info)
if kwargs.get("display", False):
import wx
from rstbx.viewer import display
from rstbx.viewer.frame import XrayFrame
display.user_callback = work.user_callback
app = wx.App(0)
frame = XrayFrame(None, -1, "X-ray image display", size=(1200, 1080))
frame.SetSize((1024, 780))
frame.load_image(path)
frame.Show()
app.MainLoop()
def run_one_index(path, *arguments, **kwargs):
assert arguments[0].find("target=")==0
target = arguments[0].split("=")[1]
from xfel.phil_preferences import load_cxi_phil
if "--nodisplay" in arguments[1:]:
display = False
arguments = list(arguments)
arguments.remove("--nodisplay")
else:
display = True
args = ["indexing.data=%s"%path,
"beam_search_scope=0.5",
"lepage_max_delta = 3.0",
"spots_pickle = None",
"subgroups_pickle = None",
"refinements_pickle = None",
"rmsd_tolerance = 5.0",
"mosflm_rmsd_tolerance = 5.0",
"difflimit_sigma_cutoff=2.0",
#"indexing.verbose_cv=True",
"indexing.open_wx_viewer=%s"%display
] + list(arguments[1:])
horizons_phil = load_cxi_phil(target, args)
info = run_one_index_core(horizons_phil)
info.Files = info.organizer.Files
info.phil_params = info.horizons_phil
# The spotfinder view within cxi.index is an anachronism; no useful purpose anymore
# therefore remove this option within cxi.index:
return
work = wrapper_of_callback(info)
if kwargs.get("display",False):
import wx
from rstbx.viewer import display
from rstbx.viewer.frame import XrayFrame
display.user_callback = work.user_callback
app = wx.App(0)
frame = XrayFrame(None, -1, "X-ray image display", size=(1200,1080))
frame.SetSize((1024,780))
frame.load_image(path)
frame.Show()
app.MainLoop()
try:
validate_phil(work_params)
except Sorry, s:
raise s
except Exception, s:
raise s
C = fit_translation4(work_params)
print "done constructor"
# parse a phil file from the args
for item in args:
if os.path.isfile(item):
from xfel.phil_preferences import load_cxi_phil
C.optional_params = load_cxi_phil(item, [])
break
print
C.run_cycle_a()
C.run_cycle_b(0)
print
C.run_cycle_a()
C.run_cycle_b(1)
print
C.run_cycle_a()
if __name__ == "__main__":
import sys
called by easy_run.fully_buffered, protecting the calling program
from boost errors.
'''
import sys
from xfel.phil_preferences import load_cxi_phil
from xfel.cxi.display_spots import run_one_index_core
from libtbx import easy_pickle
if __name__ == "__main__":
# should be invoked like this: "iota.bulletproof tmppath target args"
tmppath = sys.argv[1]
target = sys.argv[2]
args = sys.argv[3:]
try:
# index the image
horizons_phil = load_cxi_phil(target, args)
info = run_one_index_core(horizons_phil)
# save specific results from the info object to be used by iota
int_final = info.last_saved_best
easy_pickle.dump(tmppath, int_final)
except Exception, e:
if hasattr(e, "classname"):
error_message = "{}: {}".format(e.classname, e[0].replace('\n',' ')[:50])
else:
error_message = "{}".format(str(e).replace('\n', ' ')[:50])
# save the error message to be picked up by iota
easy_pickle.dump(tmppath, error_message)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights, outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights >
self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((
peak_heights > self.m_threshold).__and__(
outvalue == 0)).count(True)
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
self.stats_logger.info("BRAGG %.3f %d" %
(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info(
"Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info(
"Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time,
number_of_accepted_peaks, distance, self.sifoil,
self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(
d,
integration_dirname=self.m_integration_dirname,
integration_basename=self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None and hasattr(info, 'spotfinder_results')
if self.m_progress_logging:
if self.m_db_version == 'v1':
if indexed:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best[
"identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host,
self.m_db_name,
self.m_db_user,
self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best[
"identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[
info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,
self.m_db_experiment_tag,
self.m_trial_id,
self.m_rungroup_id,
evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best[
"identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
elif self.m_db_version == 'v2':
key_low = 'cctbx.xfel.radial_average.two_theta_low'
key_high = 'cctbx.xfel.radial_average.two_theta_high'
tt_low = evt.get(key_low)
tt_high = evt.get(key_high)
from xfel.ui.db.dxtbx_db import log_frame
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
sfspots = evt.get('sfspots')
if sfspots is None:
if info is None or not isinstance(info, int):
n_spots = 0
else:
n_spots = info
else:
n_spots = sfspots
if indexed:
known_setting = info.horizons_phil.known_setting
indexed_setting = info.organizer.info[
'best_integration']['counter']
if known_setting is None or known_setting == indexed_setting:
from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
c = construct_reflection_table_and_experiment_list(
info.last_saved_best,
None,
pixel_size,
proceed_without_image=True)
c.assemble_experiments()
c.assemble_reflections()
log_frame(c.experiment_list, c.reflections,
self.db_params, evt.run(), n_spots,
self.timestamp, tt_low, tt_high)
else:
print(
"Not logging %s, wrong bravais setting (expecting %d, got %d)"
% (self.timestamp, known_setting,
indexed_setting))
else:
log_frame(None, None, self.db_params, evt.run(),
n_spots, self.timestamp, tt_low, tt_high)
if self.m_db_logging:
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get(
'ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get(
'ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get(
'ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
).parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time, n_spots,
distance, self.sifoil, self.wavelength, indexed,
mosaic_bloc_rotation, mosaic_block_size,
ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%" % detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(
horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%s" %
detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation, pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(
horizons_phil, imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead"
)
if (self.m_out_dirname is not None
or self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
work_params = get_phil(args)
try:
validate_phil(work_params)
except Sorry, s:
raise s
except Exception, s:
raise s
C = fit_translation4(work_params)
print "done constructor"
#parse a phil file from the args
for item in args:
if os.path.isfile(item):
from xfel.phil_preferences import load_cxi_phil
C.optional_params = load_cxi_phil(item,[])
break
print
C.run_cycle_a()
C.run_cycle_b(0)
print
C.run_cycle_a()
C.run_cycle_b(1)
print
C.run_cycle_a()
if (__name__ == "__main__"):
import sys
run(args=sys.argv[1:])
called by easy_run.fully_buffered, protecting the calling program
from boost errors.
'''
import sys
from xfel.phil_preferences import load_cxi_phil
from xfel.cxi.display_spots import run_one_index_core
from libtbx import easy_pickle
if __name__ == "__main__":
# should be invoked like this: "iota.bulletproof tmppath target args"
tmppath = sys.argv[1]
target = sys.argv[2]
args = sys.argv[3:]
try:
# index the image
horizons_phil = load_cxi_phil(target, args)
info = run_one_index_core(horizons_phil)
# save specific results from the info object to be used by iota
int_final = info.last_saved_best
easy_pickle.dump(tmppath, int_final)
except Exception, e:
if hasattr(e, "classname"):
error_message = "{}: {}".format(e.classname,
e[0].replace('\n', ' ')[:50])
else:
error_message = "{}".format(str(e).replace('\n', ' ')[:50])
# save the error message to be picked up by iota
easy_pickle.dump(tmppath, error_message)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights,outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info("Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(d,
integration_dirname = self.m_integration_dirname,
integration_basename = self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None
if indexed and self.m_progress_logging:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best["identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best["identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
if self.m_db_logging:
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead")
if (self.m_out_dirname is not None or
self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
