def event(self, evt, env):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
self.nshots += 1
if (evt.get("skip_event")):
return
# Get time as a (seconds, milliseconds) tuple.
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning("event(): no timestamp, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# Update laser status.
self.laser_1.set_status(cspad_tbx.env_laser_status(env, laser_id=1), t)
self.laser_4.set_status(cspad_tbx.env_laser_status(env, laser_id=4), t)
t1 = self.laser_1.ms_since_last_status_change(t)
t4 = self.laser_4.ms_since_last_status_change(t)
if (self.laser_4.status or
(t4 is not None and t4 < self._wait) or
(t1 is not None and t1 < self._wait)):
# If laser 4 is on or was switched off less than self._wait ms
# ago, the shot falls in the "other" category. If laser 1
# changed its state less than self._wait ms ago the shot falls
# in the "other" category.
if (self._filter != "other"):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.laser_1.status):
# If laser 1 is on the shot falls in the "light" category.
if (self._filter != "light"):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.laser_1.status):
# If laser 1 is off the shot falls in the "dark" category.
if (self._filter != "dark"):
evt.put(skip_event_flag(), "skip_event")
return
else:
# NOTREACHED
self.logger.error("Could not determine shot category")
raise RuntimeError("XXX")
self.naccepted += 1
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_radial_average, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = 0.079346
saturated_value = 2**16 - 1
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
from xfel.command_line.radial_average import run
args = [
"file_path=XTC stream",
"xfel_target=%s"%self.m_xtal_target,
"verbose=False"
]
t = self.timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]
if self._dirname is not None:
dest_path = os.path.join(self._dirname, self._basename + s + ".txt")
args.append("output_file=%s"%dest_path)
self.logger.info("Calculating radial average for image %s"%s)
xvals, results = run(args, d)
evt.put(xvals, "cctbx.xfel.radial_average.xvals")
evt.put(results, "cctbx.xfel.radial_average.results")
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_image_dict, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
if distance == 0:
distance = evt.get("marccd_distance")
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
evt.put(d, self.m_out_key)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
Once self.nshots shots are accumulated, this function turns into
a nop.
@param evt Event data object, a configure object
@param env Environment object
"""
self.nv=self.nv+1
super(mod_spectra, self).event(evt, env)
if (evt.get("skip_event")):
return
# self.nv=self.nv+1
self.n_shots += 1
self.logger.info("processing event Number: " + str(self.nv))
next_update = (self.nupdate - 1) - (self.nshots - 1) % self.nupdate
if (self.ncollate > 0 and next_update >= self.ncollate):
return
count=0
file=open("spectra.txt","w+")
if (self.nvalid == 0 or self.ncollate > 0 and self.nvalid >= self.ncollate):
self.img_sum = self.cspad_img
self.nvalid = 1
else:
self.img_sum += self.cspad_img
self.nvalid += 1
if next_update==0:
pixels=self.img_sum/self.nvalid
if self.angle==0: #if the signal is not rotated sum each raw and add it to oneD array
oneD=[]
pixels=pixels.matrix_transpose()
for r in flex.rows(pixels):
oneD.append(flex.sum(r))
if self.clean=="True":
oneD=self.Gauss(oneD)
else: #if the signal is rotated do projection and sum the signal based on the projection line
oneD=self.project(pixels)
if self.clean=="True":
oneD=self.Gauss(oneD)
(filter,location)=self.Filter(oneD, self.target, self.threshold)
if filter==2:
flag='2C'
elif filter==1:
if location==self.target[0]:
flag='E1'
if location==self.target[1]:
flag='E2'
else:
flag='bad'
if self.filter=='False':
evt.put(flag, "flag")
evt.put(oneD, "cctbx_spectra")
if self.filter=="True":
if self.mode==flag:
evt.put(flag, "flag")
evt.put(oneD, "cctbx_spectra")
else:
self.logger.warning("event(): Event Skipped")
evt.put(skip_event_flag(), "skip_event")
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
if (evt.get("skip_event")):
return
self.n_total += 1
ts = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
accept = self.filter.filter_event(evt, self.selected_filter)[0]
if not accept:
print "Skipping event", ts, ": didn't pass filter", self.selected_filter
evt.put(skip_event_flag(), "skip_event")
return
print "Accepting event", ts, ": passed filter", self.selected_filter
self.n_accepted += 1
def event(self, evt, evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
if (evt.get("skip_event")):
return
self.n_total += 1
ts = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
accept = self.filter.filter_event(evt, self.selected_filter)[0]
if not accept:
print "Skipping event", ts, ": didn't pass filter", self.selected_filter
evt.put(skip_event_flag(), "skip_event")
return
print "Accepting event", ts, ": passed filter", self.selected_filter
self.n_accepted += 1
def event(self, evt, env):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
self.nshots += 1
if (evt.get("skip_event")):
return
if (self.timestamps_list is not None):
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning(
"event(): no timestamp, shot skipped. Shot: %s" %
self.nshots)
evt.put(skip_event_flag(), "skip_event")
return
elif (self.negate and t in self.timestamps_list):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.negate and t not in self.timestamps_list):
evt.put(skip_event_flag(), "skip_event")
return
else:
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning(
"event(): no timestamp, shot skipped. Shot: %s" %
self.nshots)
evt.put(skip_event_flag(), "skip_event")
return
if (self.negate and self._tir(t, self.timestamps_interval)):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.negate
and not self._tir(t, self.timestamps_interval)):
evt.put(skip_event_flag(), "skip_event")
return
self.logger.info("event(): event accepted. Shot: %s, TS: %s" %
(self.nshots, cspad_tbx.evt_timestamp(t)))
self.naccepted += 1
def event(self, evt, env):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
self.nshots += 1
if (evt.get("skip_event")):
return
if (self.timestamps_list is not None):
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning("event(): no timestamp, shot skipped. Shot: %s"%self.nshots)
evt.put(skip_event_flag(), "skip_event")
return
elif (self.negate and t in self.timestamps_list):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.negate and t not in self.timestamps_list):
evt.put(skip_event_flag(), "skip_event")
return
else:
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning("event(): no timestamp, shot skipped. Shot: %s"%self.nshots)
evt.put(skip_event_flag(), "skip_event")
return
if (self.negate and self._tir(t, self.timestamps_interval)):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.negate and not self._tir(t, self.timestamps_interval)):
evt.put(skip_event_flag(), "skip_event")
return
self.logger.info("event(): event accepted. Shot: %s, TS: %s"%(self.nshots,cspad_tbx.evt_timestamp(t)))
self.naccepted += 1
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights,outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True)
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info("Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %(
env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance,
self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(d,
integration_dirname = self.m_integration_dirname,
integration_basename = self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None
if indexed and self.m_progress_logging:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best["identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best["identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
if self.m_db_logging:
sec,ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms/1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance,
self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation,
mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a,
cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead")
if (self.m_out_dirname is not None or
self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_radial_average, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# See r17537 of mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = 0.079346
saturated_value = 2**16 - 1
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
from xfel.command_line.radial_average import run
args = [
"file_path=XTC stream",
"xfel_target=%s" % self.m_xtal_target, "verbose=False"
]
t = self.timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
20:23]
if self._dirname is not None:
dest_path = os.path.join(self._dirname,
self._basename + s + ".txt")
args.append("output_file=%s" % dest_path)
self.logger.info("Calculating radial average for image %s" % s)
xvals, results = run(args, d)
evt.put(xvals, "cctbx.xfel.radial_average.xvals")
evt.put(results, "cctbx.xfel.radial_average.results")
def get_closest_idx(data, val):
from scitbx.array_family import flex
deltas = flex.abs(data - val)
return flex.first_index(deltas, flex.min(deltas))
if self._two_theta_low is not None:
i_low = results[get_closest_idx(xvals, self._two_theta_low)]
evt.put(i_low, "cctbx.xfel.radial_average.two_theta_low")
if self._two_theta_high is not None:
i_high = results[get_closest_idx(xvals, self._two_theta_high)]
evt.put(i_high, "cctbx.xfel.radial_average.two_theta_high")
def event(self, evt, evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(
cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata += 1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(), "skip_event")
if timestamp is None:
evt.put(skip_event_flag(), "skip_event")
self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots += 1
# get data as array and split into two half to find each peak
if one_D:
data = np.array(data.hproj().astype(np.float64))
if 'dark' in locals():
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0] // 2]
spectrum2 = data[data.shape[0] // 2:]
else:
data = np.array(data.data16().astype(np.float64))
if 'dark' in locals():
data = data - self.dark
data_split1 = data[:, :data.shape[1] // 2]
data_split2 = data[:, data.shape[1] // 2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data, 0) / data.shape[0]
spectrum1 = np.sum(data_split1, 0) / data_split1.shape[0]
spectrum2 = np.sum(data_split2, 0) / data_split2.shape[0]
peak_one = np.max(spectrum1)
peak_two = np.max(spectrum2)
peak_one_position = np.argmax(spectrum1)
peak_two_position = np.argmax(spectrum2) + len(spectrum2)
# define the limits of the regions between the two peaks
peak_one_lower_limit = self.peak_one_position_min - self.peak_one_width
peak_one_upper_limit = self.peak_one_position_max + self.peak_one_width
peak_two_lower_limit = self.peak_two_position_min - self.peak_two_width
peak_two_upper_limit = self.peak_two_position_max + self.peak_two_width
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in range(peak_one_lower_limit, peak_one_upper_limit):
weighted_peak_one_positions.append(spectrum[i] * i)
weighted_sum_peak_one = sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one // sum(
spectrum[peak_one_lower_limit:peak_one_upper_limit])
weighted_peak_two_positions = []
for i in range(peak_two_lower_limit, peak_two_upper_limit):
weighted_peak_two_positions.append(spectrum[i] * i)
weighted_sum_peak_two = sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two // sum(
spectrum[peak_two_lower_limit:peak_two_upper_limit])
# normalized integrated peaks
int_peak_one = np.sum(
spectrum[peak_one_lower_limit:self.peak_one_position_max]
) / len(spectrum[peak_one_lower_limit:self.peak_one_position_max])
int_peak_two = np.sum(
spectrum[peak_two_lower_limit:self.peak_two_position_max]
) / len(spectrum[peak_two_lower_limit:self.peak_two_position_max])
# normalized integrated regions between the peaks
int_left_region = np.sum(spectrum[0:peak_one_lower_limit]) / len(
spectrum[0:peak_one_lower_limit])
int_middle_region = np.sum(
spectrum[peak_one_upper_limit:peak_two_lower_limit]) / len(
spectrum[peak_one_upper_limit:peak_two_lower_limit])
int_right_region = np.sum(spectrum[peak_two_upper_limit:]) / len(
spectrum[:peak_two_upper_limit])
# now to do the filtering
if peak_one / peak_two < self.peak_ratio or peak_one / peak_two > 1 / self.peak_ratio:
print "event(): too low"
evt.put(skip_event_flag(), "skip_event")
return
if weighted_peak_two_center_position < self.peak_two_position_min or weighted_peak_two_center_position > self.peak_two_position_max:
print "event(): out of range high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if weighted_peak_one_center_position < self.peak_one_position_min or weighted_peak_one_center_position > self.peak_one_position_max:
print "event(): out of range low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_left_region / int_peak_one >
self.normalized_peak_to_noise_ratio):
print "event(): noisy left of low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_middle_region / int_peak_one >
self.normalized_peak_to_noise_ratio):
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_middle_region / int_peak_two >
self.normalized_peak_to_noise_ratio):
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_right_region / int_peak_two >
self.normalized_peak_to_noise_ratio):
print "event(): noisy right of high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
#iron edge at 738 pixels on FFE detetor
if one_D and (spectrum[self.iron_edge_position] >=
spectrum[weighted_peak_one_center_position]):
print "event(): peak at iron edge"
evt.put(skip_event_flag(), "skip_event")
return
if one_D and (spectrum[self.iron_edge_position] >=
spectrum[weighted_peak_two_center_position]):
print "event(): peak at iron edge"
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print "%d Two Color shots" % self.ntwo_color
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata +=1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(),"skip_event")
if timestamp is None:
evt.put(skip_event_flag(),"skip_event")
#self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots +=1
# get data as array and split into two half to find each peak
if one_D:
# filtering out outlier spikes in FEE data
data = np.array(data.hproj().astype(np.float64))
for i in xrange(len(data)):
if data[i]>1000000000:
data[i]=data[i]-(2**32)
if self.dark is not None:
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0]//2]
spectrum2 = data[data.shape[0]//2:]
else:
data = np.array(data.data16().astype(np.int32))
if self.dark is not None:
data = data - self.dark
data = np.double(data)
data_split1 = data[:,:data.shape[1]//2]
data_split2 = data[:,data.shape[1]//2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data,0)/data.shape[0]
spectrum1 = np.sum(data_split1,0)/data_split1.shape[0]
spectrum2 = np.sum(data_split2,0)/data_split2.shape[0]
if not one_D:
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in xrange(self.peak_one_range_min,self.peak_one_range_max):
weighted_peak_one_positions.append(spectrum[i]*i)
weighted_sum_peak_one = np.sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one/np.sum(spectrum[self.peak_one_range_min:self.peak_one_range_max])
weighted_peak_two_positions = []
for i in xrange(self.peak_two_range_min,self.peak_two_range_max):
weighted_peak_two_positions.append(spectrum[i]*i)
weighted_sum_peak_two = np.sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two/np.sum(spectrum[self.peak_two_range_min:self.peak_two_range_max])
# normalized integrated regions between the peaks
#int_left_region = np.sum(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_left_region = np.sum(spectrum[:weighted_peak_two_center_position/2])
#int_left_region_norm = np.sum(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_left_region_norm = np.sum(spectrum[:weighted_peak_two_center_position/2])/len(spectrum[:weighted_peak_two_center_position/2])
int_right_region = np.sum(spectrum[self.peak_two_range_max:])
int_right_region_norm = np.sum(spectrum[self.peak_two_range_max:])/len(spectrum[self.peak_two_range_max:])
# normalized integrated peaks
int_peak_one = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_one_norm = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])/len(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_two = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_peak_two_norm = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
if not one_D:
if int_peak_one_norm/int_peak_two_norm > self.peak_ratio:
print "event(): inflection peak too high"
evt.put(skip_event_flag(), "skip_event")
return
if int_left_region_norm > self.normalized_peak_to_noise_ratio*int_peak_two_norm:
print "event(): noisy left of low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if int_right_region_norm > self.normalized_peak_to_noise_ratio*int_peak_two_norm:
print "event(): noisy right of high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print "%d Remote shot" %self.ntwo_color
print "%s Remote timestamp" %timestamp
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata +=1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(),"skip_event")
if timestamp is None:
evt.put(skip_event_flag(),"skip_event")
#self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots +=1
# get data as array and split into two half to find each peak
if one_D:
# filtering out outlier spikes in FEE data
data = np.array(data.hproj().astype(np.float64))
for i in range(len(data)):
if data[i]>1000000000:
data[i]=data[i]-(2**32)
if self.dark is not None:
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0]//2]
spectrum2 = data[data.shape[0]//2:]
else:
data = np.array(data.data16().astype(np.int32))
if self.dark is not None:
data = data - self.dark
data = np.double(data)
data_split1 = data[:,:data.shape[1]//2]
data_split2 = data[:,data.shape[1]//2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data,0)/data.shape[0]
spectrum1 = np.sum(data_split1,0)/data_split1.shape[0]
spectrum2 = np.sum(data_split2,0)/data_split2.shape[0]
if not one_D:
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in range(self.peak_one_range_min,self.peak_one_range_max):
weighted_peak_one_positions.append(spectrum[i]*i)
weighted_sum_peak_one = np.sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one/np.sum(spectrum[self.peak_one_range_min:self.peak_one_range_max])
weighted_peak_two_positions = []
for i in range(self.peak_two_range_min,self.peak_two_range_max):
weighted_peak_two_positions.append(spectrum[i]*i)
weighted_sum_peak_two = np.sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two/np.sum(spectrum[self.peak_two_range_min:self.peak_two_range_max])
# normalized integrated regions between the peaks
#int_left_region = np.sum(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_left_region = np.sum(spectrum[:weighted_peak_two_center_position/2])
#int_left_region_norm = np.sum(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_left_region_norm = np.sum(spectrum[:weighted_peak_two_center_position/2])/len(spectrum[:weighted_peak_two_center_position/2])
int_right_region = np.sum(spectrum[self.peak_two_range_max:])
int_right_region_norm = np.sum(spectrum[self.peak_two_range_max:])/len(spectrum[self.peak_two_range_max:])
# normalized integrated peaks
int_peak_one = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_one_norm = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])/len(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_two = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_peak_two_norm = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
if not one_D:
if int_peak_one_norm/int_peak_two_norm > self.peak_ratio:
print("event(): inflection peak too high")
evt.put(skip_event_flag(), "skip_event")
return
if int_left_region_norm > self.normalized_peak_to_noise_ratio*int_peak_two_norm:
print("event(): noisy left of low energy peak")
evt.put(skip_event_flag(), "skip_event")
return
if int_right_region_norm > self.normalized_peak_to_noise_ratio*int_peak_two_norm:
print("event(): noisy right of high energy peak")
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print("%d Remote shot" %self.ntwo_color)
print("%s Remote timestamp" %timestamp)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX more?
Previously, common-mode correction was applied only after initial
threshold filtering. Since the common_mode class applies the
(lengthy) common-mode correction immediately after reading the
image from the stream, this optimisation is currently not
(elegantly) doable.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_hitfind, self).event(evt, env)
if (evt.get("skip_event")):
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
# ***** HITFINDING ***** XXX For hitfinding it may be interesting
# to look at the fraction of subzero pixels in the dark-corrected
# image.
if (self.m_threshold is not None):
# If a threshold value is given it can be applied in one of three ways:
# 1. Apply it over the whole image
if (self.m_roi is None and self.m_distl_min_peaks is None):
vmax = flex.max(self.cspad_img)
if (vmax < self.m_threshold):
if not self.m_negate_hits:
# Tell downstream modules to skip this event if the threshold was not met.
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 2. Apply threshold over a rectangular region of interest.
elif (self.m_roi is not None):
vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3],
self.m_roi[0]:self.m_roi[1]])
if (vmax < self.m_threshold):
if not self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
elif self.m_negate_hits:
evt.put(skip_event_flag(), "skip_event")
return
# 3. Determine the spotfinder spots within the central ASICS, and accept the
# image as a hit if there are m_distl_min_peaks exceeding m_threshold.
# As a further requirement, the peaks must exceed 2.5 * the 90-percentile
# pixel value of the central ASICS. This filter was added to avoid high-background
# false positives.
elif (self.m_distl_min_peaks is not None):
if device == 'marccd':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
elif device == 'Rayonix':
self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0]
self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1]
peak_heights, outvalue = self.distl_filter(
self.address,
self.cspad_img.iround(), # XXX correct?
distance,
self.timestamp,
self.wavelength)
if ('permissive' in self.m_distl_flags):
number_of_accepted_peaks = (peak_heights >
self.m_threshold).count(True)
else:
number_of_accepted_peaks = ((
peak_heights > self.m_threshold).__and__(
outvalue == 0)).count(True)
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
self.stats_logger.info("BRAGG %.3f %d" %
(evt_time, number_of_accepted_peaks))
skip_event = False
if number_of_accepted_peaks < self.m_distl_min_peaks:
self.logger.info(
"Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if not self.m_negate_hits:
skip_event = True
else:
self.logger.info(
"Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d"
% (env.subprocess(), self.nshots, self.timestamp,
number_of_accepted_peaks, self.m_threshold))
if self.m_negate_hits:
skip_event = True
if skip_event:
if self.m_db_logging:
# log misses to the database
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time,
number_of_accepted_peaks, distance, self.sifoil,
self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, self.m_db_tags))
evt.put(skip_event_flag(), "skip_event")
return
# the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this
# hit will not be logged in the db.
evt.put(number_of_accepted_peaks, 'sfspots')
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength,
xtal_target=self.m_xtal_target)
if (self.m_dispatch == "index"):
import sys
from xfel.cxi.integrate_image_api import integrate_one_image
info = integrate_one_image(
d,
integration_dirname=self.m_integration_dirname,
integration_basename=self.m_integration_basename)
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
indexed = info is not None and hasattr(info, 'spotfinder_results')
if self.m_progress_logging:
if self.m_db_version == 'v1':
if indexed:
# integration pickle dictionary is available here as info.last_saved_best
if info.last_saved_best[
"identified_isoform"] is not None:
#print info.last_saved_best.keys()
from cxi_xdr_xes.cftbx.cspad_ana import db
dbobj = db.dbconnect(self.m_db_host,
self.m_db_name,
self.m_db_user,
self.m_db_password)
cursor = dbobj.cursor()
if info.last_saved_best[
"identified_isoform"] in self.isoforms:
PM, indices, miller_id = self.isoforms[
info.last_saved_best["identified_isoform"]]
else:
from xfel.xpp.progress_support import progress_manager
PM = progress_manager(info.last_saved_best,
self.m_db_experiment_tag,
self.m_trial_id,
self.m_rungroup_id,
evt.run())
indices, miller_id = PM.get_HKL(cursor)
# cache these as they don't change for a given isoform
self.isoforms[info.last_saved_best[
"identified_isoform"]] = PM, indices, miller_id
if self.m_sql_buffer_size > 1:
self.queue_progress_entry(
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=False))
else:
PM.scale_frame_detail(self.timestamp,
cursor,
do_inserts=True)
dbobj.commit()
cursor.close()
dbobj.close()
elif self.m_db_version == 'v2':
key_low = 'cctbx.xfel.radial_average.two_theta_low'
key_high = 'cctbx.xfel.radial_average.two_theta_high'
tt_low = evt.get(key_low)
tt_high = evt.get(key_high)
from xfel.ui.db.dxtbx_db import log_frame
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
sfspots = evt.get('sfspots')
if sfspots is None:
if info is None or not isinstance(info, int):
n_spots = 0
else:
n_spots = info
else:
n_spots = sfspots
if indexed:
known_setting = info.horizons_phil.known_setting
indexed_setting = info.organizer.info[
'best_integration']['counter']
if known_setting is None or known_setting == indexed_setting:
from xfel.command_line.frame_unpickler import construct_reflection_table_and_experiment_list
c = construct_reflection_table_and_experiment_list(
info.last_saved_best,
None,
pixel_size,
proceed_without_image=True)
c.assemble_experiments()
c.assemble_reflections()
log_frame(c.experiment_list, c.reflections,
self.db_params, evt.run(), n_spots,
self.timestamp, tt_low, tt_high)
else:
print(
"Not logging %s, wrong bravais setting (expecting %d, got %d)"
% (self.timestamp, known_setting,
indexed_setting))
else:
log_frame(None, None, self.db_params, evt.run(),
n_spots, self.timestamp, tt_low, tt_high)
if self.m_db_logging:
sec, ms = cspad_tbx.evt_time(evt)
evt_time = sec + ms / 1000
sfspots = evt.get('sfspots')
if sfspots is None:
if indexed:
n_spots = len(info.spotfinder_results.images[
info.frames[0]]['spots_total'])
else:
n_spots = 0
else:
n_spots = sfspots
if indexed:
mosaic_bloc_rotation = info.last_saved_best.get(
'ML_half_mosaicity_deg', [0])[0]
mosaic_block_size = info.last_saved_best.get(
'ML_domain_size_ang', [0])[0]
ewald_proximal_volume = info.last_saved_best.get(
'ewald_proximal_volume', [0])[0]
obs = info.last_saved_best['observations'][0]
cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell(
).parameters()
pointgroup = info.last_saved_best['pointgroup']
resolution = obs.d_min()
else:
mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \
cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0
self.queue_entry(
(self.trial, evt.run(), "%.3f" % evt_time, n_spots,
distance, self.sifoil, self.wavelength, indexed,
mosaic_bloc_rotation, mosaic_block_size,
ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c,
cell_alpha, cell_beta, cell_gamma, resolution,
self.m_db_tags))
if (not indexed):
evt.put(skip_event_flag(), "skip_event")
return
elif (self.m_dispatch == "nop"):
pass
elif (self.m_dispatch == "view"): #interactive image viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%" % detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
display_spots.wrapper_of_callback().display(
horizons_phil.indexing.data)
elif (self.m_dispatch == "spots"): #interactive spotfinder viewer
args = ["indexing.data=dummy"]
detector_format_version = detector_format_function(
self.address, evt.GetTime())
if detector_format_version is not None:
args += [
"distl.detector_format_version=%s" %
detector_format_version
]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation, pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
horizons_phil.persist.show()
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(
horizons_phil, imagefile_arguments)
work = display_spots.wrapper_of_callback(info)
work.display_with_callback(horizons_phil.indexing.data)
elif (self.m_dispatch == "write_dict"):
self.logger.warning(
"event(): deprecated dispatch 'write_dict', use mod_dump instead"
)
if (self.m_out_dirname is not None
or self.m_out_basename is not None):
cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(average_mixin, self).event(evt, env)
if evt.get('skip_event'):
return
# Get the distance for the detectors that should have it, and set
# it to NaN for those that should not.
if self.detector == 'CxiDs1' or \
self.detector == 'CxiDs2' or \
self.detector == 'CxiDsd' or \
self.detector == 'XppGon':
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self._nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
else:
distance = float('nan')
if ("skew" in self.flags):
# Take out inactive pixels
if self.roi is not None:
pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
pixels = pixels.as_1d().select(dark_mask.as_1d())
else:
pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double()
stats = scitbx.math.basic_statistics(pixels.as_double())
#stats.show()
self.logger.info("skew: %.3f" %stats.skew)
self.logger.info("kurtosis: %.3f" %stats.kurtosis)
if 0:
from matplotlib import pyplot
hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double())
print hist_min, hist_max
n_slots = 100
n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
pyplot.show()
# XXX This skew threshold probably needs fine-tuning
skew_threshold = 0.35
if stats.skew < skew_threshold:
self._nfail += 1
self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold)
evt.put(skip_event_flag(), 'skip_event')
return
#self.cspad_img *= stats.skew
if ("inactive" in self.flags):
self.cspad_img.set_selected(self.dark_stddev <= 0, 0)
if ("noelastic" in self.flags):
ELASTIC_THRESHOLD = self.elastic_threshold
self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0)
if self.hot_threshold is not None:
HOT_THRESHOLD = self.hot_threshold
self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0)
if self.gain_map is not None and self.gain_threshold is not None:
# XXX comparing each pixel to a moving average would probably be better
# since the gain should vary approximately smoothly over different areas
# of the detector
GAIN_THRESHOLD = self.gain_threshold
#self.logger.debug(
#"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True))
self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0)
if ("nonoise" in self.flags):
NOISE_THRESHOLD = self.noise_threshold
self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0)
if ("sigma_scaling" in self.flags):
self.do_sigma_scaling()
if ("symnoise" in self.flags):
SYMNOISE_THRESHOLD = self.symnoise_threshold
self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) &
( self.cspad_img < SYMNOISE_THRESHOLD), 0)
if ("output" in self.flags):
try:
import cPickle as pickle
except ImportError:
import pickle
import os
if (not os.path.isdir(self.pickle_dirname)):
os.makedirs(self.pickle_dirname)
flexdata = flex.int(self.cspad_img.astype(numpy.int32))
d = cspad_tbx.dpack(
address=self.address,
data=flexdata,
timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
)
G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename,
"ab")
pickle.dump(d,G,pickle.HIGHEST_PROTOCOL)
G.close()
if self.photon_threshold is not None and self.two_photon_threshold is not None:
self.do_photon_counting()
if self.background_path is not None:
self.cspad_img -= self.background_img
# t and self._sum_time are a two-long arrays of seconds and
# milliseconds which hold time with respect to the base time.
t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt),
self._metadata['time_base'])]
if self._nmemb == 0:
# The peers metadata item is a bit field where a bit is set if
# the partial sum from the corresponding worker process is
# pending. If this is the first frame a worker process sees,
# set its corresponding bit in the bit field since it will
# contribute a partial sum.
if env.subprocess() >= 0:
self._lock.acquire()
if 'peers' in self._metadata.keys():
self._metadata['peers'] |= (1 << env.subprocess())
else:
self._metadata['peers'] = (1 << env.subprocess())
self._lock.release()
self._sum_distance = distance
self._sum_time = (t[0], t[1])
self._sum_wavelength = self.wavelength
if self._have_max:
self._max_img = self.cspad_img.deep_copy()
if self._have_mean:
self._sum_img = self.cspad_img.deep_copy()
if self._have_std:
self._ssq_img = flex.pow2(self.cspad_img)
else:
self._sum_distance += distance
self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1])
self._sum_wavelength += self.wavelength
if self._have_max:
sel = (self.cspad_img > self._max_img).as_1d()
self._max_img.as_1d().set_selected(
sel, self.cspad_img.as_1d().select(sel))
if self._have_mean:
self._sum_img += self.cspad_img
if self._have_std:
self._ssq_img += flex.pow2(self.cspad_img)
self._nmemb += 1
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).event(evt, env)
if (evt.get("skip_event")):
return
if not hasattr(self, 'active_areas') or self.active_areas is None or \
not hasattr(self, 'beam_center') or self.beam_center is None:
if self.address == 'XppGon-0|marccd-0':
# The mod_mar module needs to have been called before this one
# to set this up. The MAR does not have a configure object.
self.beam_center = evt.get("marccd_beam_center")
self.active_areas = evt.get("marccd_active_areas")
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
pass # bc and aa set in the beginjob function
elif self.address == 'XppGon-0|Cspad-0':
# Load the active areas as determined from the optical metrology
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(self.timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = \
cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)
if self.filter_laser_1_status is not None:
if (self.laser_1_status.status != self.filter_laser_1_status or
(self.laser_1_ms_since_change is not None and
self.laser_1_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
if self.filter_laser_4_status is not None:
if (self.laser_4_status.status != self.filter_laser_4_status or
(self.laser_4_ms_since_change is not None and
self.laser_4_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
# Early return if the full detector image is already stored in the
# event. Otherwise, get it from the stream as a double-precision
# floating-point flex array. XXX It is probably not safe to key
# the image on self.address, so we should come up with our own
# namespace. XXX Misnomer--could be CAMP, too
self.cspad_img = evt.get(self.address)
if self.cspad_img is not None:
return
if self.address == 'XppGon-0|Cspad-0':
# Kludge until cspad_tbx.image() can be rewritten to handle the
# XPP metrology.
self.cspad_img = cspad_tbx.image_xpp(
self.address, evt, env, self.active_areas)
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
from psana import Source, Camera
import numpy as np
address = cspad_tbx.old_address_to_new_address(self.address)
src=Source('DetInfo(%s)'%address)
self.cspad_img = evt.get(Camera.FrameV1,src)
if self.cspad_img is not None:
self.cspad_img = self.cspad_img.data16().astype(np.float64)
elif self.address=='CxiDg3-0|Opal1000-0':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='CxiEndstation-0|Opal1000-2':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='FeeHxSpectrometer-0|Opal1000-1':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='NoDetector-0|Cspad2x2-0':
import numpy as np
from pypdsdata import xtc
test=[]
self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,self.address).data()
self.cspad_img=np.reshape(self.cspad_img,(370, 388))
else:
try:
self.cspad_img = cspad_tbx.image(
self.address, cspad_tbx.getConfig(self.address, env),
evt, env, self.sections)
except Exception, e:
self.logger.error("Error reading image data: " + str(e))
evt.put(skip_event_flag(), "skip_event")
return
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).event(evt, env)
if (evt.get("skip_event")):
return
if not hasattr(self, 'active_areas') or self.active_areas is None or \
not hasattr(self, 'beam_center') or self.beam_center is None:
if self.address == 'XppGon-0|marccd-0':
# The mod_mar module needs to have been called before this one
# to set this up. The MAR does not have a configure object.
self.beam_center = evt.get("marccd_beam_center")
self.active_areas = evt.get("marccd_active_areas")
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
pass # bc and aa set in the beginjob function
elif self.address == 'XppGon-0|Cspad-0':
# Load the active areas as determined from the optical metrology
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
self.address,
reverse_timestamp(self.timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup][
'active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = \
cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)
if self.filter_laser_1_status is not None:
if (self.laser_1_status.status != self.filter_laser_1_status or
(self.laser_1_ms_since_change is not None and
self.laser_1_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
if self.filter_laser_4_status is not None:
if (self.laser_4_status.status != self.filter_laser_4_status or
(self.laser_4_ms_since_change is not None and
self.laser_4_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
# Early return if the full detector image is already stored in the
# event. Otherwise, get it from the stream as a double-precision
# floating-point flex array. XXX It is probably not safe to key
# the image on self.address, so we should come up with our own
# namespace. XXX Misnomer--could be CAMP, too
self.cspad_img = evt.get(self.address)
if self.cspad_img is not None:
return
if self.address == 'XppGon-0|Cspad-0':
# Kludge until cspad_tbx.image() can be rewritten to handle the
# XPP metrology.
self.cspad_img = cspad_tbx.image_xpp(self.address, evt, env,
self.active_areas)
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
from psana import Source, Camera
import numpy as np
address = cspad_tbx.old_address_to_new_address(self.address)
src = Source('DetInfo(%s)' % address)
self.cspad_img = evt.get(Camera.FrameV1, src)
if self.cspad_img is not None:
self.cspad_img = self.cspad_img.data16().astype(np.float64)
elif self.address == 'CxiDg3-0|Opal1000-0':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'CxiEndstation-0|Opal1000-2':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'FeeHxSpectrometer-0|Opal1000-1':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'NoDetector-0|Cspad2x2-0':
import numpy as np
from pypdsdata import xtc
test = []
self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,
self.address).data()
self.cspad_img = np.reshape(self.cspad_img, (370, 388))
else:
try:
self.cspad_img = cspad_tbx.image(
self.address, cspad_tbx.getConfig(self.address, env), evt,
env, self.sections)
except Exception as e:
self.logger.error("Error reading image data: " + str(e))
evt.put(skip_event_flag(), "skip_event")
return
if self.cspad_img is None:
if cspad_tbx.address_split(self.address)[2] != 'Andor':
self.nfail += 1
self.logger.warning("event(): no image, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
self.cspad_img = flex.double(self.cspad_img.astype(numpy.float64))
# If a dark image was provided, subtract it from the image. There
# is no point in doing common-mode correction unless the dark
# image was subtracted.
if (self.dark_img is not None):
self.cspad_img -= self.dark_img
if (self.common_mode_correction != "none"):
# Mask out inactive pixels prior to common mode correction.
# Pixels are marked as inactive either due to low ADU values
# or non-positive standard deviations in dark image. XXX Make
# the threshold tunable?
cspad_mask = self.dark_mask.deep_copy()
if self.roi is not None and self.common_mode_correction == "chebyshev":
roi_mask = cspad_mask[self.roi[2]:self.roi[3], :]
roi_mask = flex.bool(roi_mask.accessor(), False)
cspad_mask.matrix_paste_block_in_place(block=roi_mask,
i_row=self.roi[2],
i_column=0)
# Extract each active section from the assembled detector
# image and apply the common mode correction. XXX Make up a
# quadrant mask for the emission detector. Needs to be
# checked!
config = cspad_tbx.getConfig(self.address, env)
if len(self.sections) == 1:
q_mask = 1
else:
q_mask = config.quadMask()
for q in range(len(self.sections)):
if (not ((1 << q) & q_mask)):
continue
# XXX Make up section mask for the emission detector. Needs
# to be checked!
import _pdsdata
if len(self.sections) == 1 and type(config) in (
_pdsdata.cspad2x2.ConfigV1,
_pdsdata.cspad2x2.ConfigV2):
s_mask = config.roiMask()
else:
s_mask = config.roiMask(q)
for s in range(len(self.sections[q])):
# XXX DAQ misconfiguration? This mask appears not to work
# reliably for the Sc1 detector.
# if (not((1 << s) & s_mask)):
# continue
corners = self.sections[q][s].corners()
i_row = int(round(min(c[0] for c in corners)))
i_column = int(round(min(c[1] for c in corners)))
n_rows = int(round(max(c[0] for c in corners))) - i_row
n_columns = int(round(max(
c[1] for c in corners))) - i_column
section_img = self.cspad_img.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
section_mask = cspad_mask.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
section_stddev = self.dark_stddev.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
if section_mask.count(True) == 0: continue
if self.common_mode_correction == "chebyshev":
assert len(self.sections[q]) == 2
if s == 0:
section_imgs = [section_img]
section_masks = [section_mask]
i_rows = [i_row]
i_columns = [i_column]
continue
else:
section_imgs.append(section_img)
section_masks.append(section_mask)
i_rows.append(i_row)
i_columns.append(i_column)
chebyshev_corrected_imgs = self.chebyshev_common_mode(
section_imgs, section_masks)
for i in range(2):
section_imgs[i].as_1d().copy_selected(
section_masks[i].as_1d().iselection(),
chebyshev_corrected_imgs[i].as_1d())
self.cspad_img.matrix_paste_block_in_place(
block=section_imgs[i],
i_row=i_rows[i],
i_column=i_columns[i])
else:
common_mode = self.common_mode(
section_img, section_stddev, section_mask)
self.sum_common_mode += common_mode
self.sumsq_common_mode += common_mode**2
# Apply the common mode correction to the
# section, and paste it back into the image.
self.cspad_img.matrix_paste_block_in_place(
block=section_img - common_mode,
i_row=i_row,
i_column=i_column)
if self.gain_map is not None:
self.cspad_img *= self.gain_map
if (self.mask_img is not None):
sel = (self.mask_img == -2) | (self.mask_img
== cspad_tbx.cspad_mask_value)
self.cspad_img.set_selected(sel, cspad_tbx.cspad_mask_value)
if (self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0') and \
self.crop_rayonix:
# Crop the masked data so that the beam center is in the center of the image
self.cspad_img = self.cspad_img[self.rayonix_crop_slice[0],
self.rayonix_crop_slice[1]]
if self.cache_image:
# Store the image in the event.
evt.put(self.cspad_img, self.address)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX Since the viewer is now running in a parallel process, the
averaging here is now the bottleneck.
@param evt Event data object, a configure object
@param env Environment object
"""
from pyana.event import Event
self.n_shots += 1
super(mod_view, self).event(evt, env)
if evt.status() != Event.Normal or evt.get('skip_event'): # XXX transition
return
# Get the distance for the detectors that should have it, and set
# it to NaN for those that should not.
if self.detector == 'CxiDs1' or \
self.detector == 'CxiDsd' or \
self.detector == 'XppGon':
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
distance = float('nan')
if not self._proc.is_alive():
evt.setStatus(Event.Stop)
# Early return if the next update to the viewer is more than
# self.ncollate shots away. XXX Since the common_mode.event()
# function does quite a bit of processing, the savings are
# probably not so big.
next_update = (self.nupdate - 1) - (self.nshots - 1) % self.nupdate
if (self.ncollate > 0 and next_update >= self.ncollate):
return
if self.sigma_scaling:
self.do_sigma_scaling()
if self.photon_counting:
self.do_photon_counting()
# Trim the disabled section from the Sc1 detector image. XXX This
# is a bit of a kludge, really.
# if (self.address == "CxiSc1-0|Cspad2x2-0"):
# self.cspad_img = self.cspad_img[185:2 * 185, :]
# Update the sum of the valid images, starting a new collation if
# appropriate. This guarantees self.nvalid > 0.
if (self.nvalid == 0 or self.ncollate > 0 and self.nvalid >= self.ncollate):
self.img_sum = self.cspad_img
self.nvalid = 1
else:
self.img_sum += self.cspad_img
self.nvalid += 1
# Update the viewer to display the current average image, and
# start a new collation, if appropriate.
if (next_update == 0):
from time import localtime, strftime
time_str = strftime("%H:%M:%S", localtime(evt.getTime().seconds()))
title = "r%04d@%s: average of %d last images on %s" \
% (evt.run(), time_str, self.nvalid, self.address)
# See also mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
beam_center = self.beam_center
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
beam_center = tuple(t // 2 for t in self.img_sum.focus())
pixel_size = 0.079346
saturated_value = 2**16 - 1
# Wait for the viewer process to empty the queue before feeding
# it a new image, and ensure not to hang if the viewer process
# exits. Because of multithreading/multiprocessing semantics,
# self._queue.empty() is unreliable.
fmt = _Format(BEAM_CENTER=beam_center,
DATA=self.img_sum / self.nvalid,
DETECTOR_ADDRESS=self.address,
DISTANCE=distance,
PIXEL_SIZE=pixel_size,
SATURATED_VALUE=saturated_value,
TIME_TUPLE=cspad_tbx.evt_time(evt),
WAVELENGTH=self.wavelength)
while not self._queue.empty():
if not self._proc.is_alive():
evt.setStatus(Event.Stop)
return
while True:
try:
self._queue.put((fmt, title), timeout=1)
break
except Exception:
pass
if (self.ncollate > 0):
self.nvalid = 0
def event(self, evt, evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(
cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata += 1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(), "skip_event")
if timestamp is None:
evt.put(skip_event_flag(), "skip_event")
#self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots += 1
# get data as array and split into two half to find each peak
if one_D:
# filtering out outlier spikes in FEE data
data = np.array(data.hproj().astype(np.float64))
for i in range(len(data)):
if data[i] > 1000000000:
data[i] = data[i] - (2**32)
if self.dark is not None:
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0] // 2]
spectrum2 = data[data.shape[0] // 2:]
else:
data = np.array(data.data16().astype(np.int32))
if self.dark is not None:
data = data - self.dark
data_split1 = data[:, :data.shape[1] // 2]
data_split2 = data[:, data.shape[1] // 2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data, 0) / data.shape[0]
spectrum1 = np.sum(data_split1, 0) / data_split1.shape[0]
spectrum2 = np.sum(data_split2, 0) / data_split2.shape[0]
if not one_D:
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in range(self.peak_one_range_min,
self.peak_one_range_max):
weighted_peak_one_positions.append(spectrum[i] * i)
weighted_sum_peak_one = sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one // sum(
spectrum[self.peak_one_range_min:self.peak_one_range_max])
weighted_peak_two_positions = []
for i in range(self.peak_two_range_min,
self.peak_two_range_max):
weighted_peak_two_positions.append(spectrum[i] * i)
weighted_sum_peak_two = sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two // sum(
spectrum[self.peak_two_range_min:self.peak_two_range_max])
# normalized integrated regions between the peaks
int_left_region_norm = np.sum(spectrum[0:(
weighted_peak_one_center_position -
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)]) / len(spectrum[0:(
weighted_peak_one_center_position -
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)])
int_middle_region_norm = np.sum(
spectrum[(weighted_peak_one_center_position +
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2):
(weighted_peak_two_center_position -
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)]
) / len(spectrum[(weighted_peak_one_center_position +
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2):
(weighted_peak_two_center_position -
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)])
int_right_region_norm = np.sum(spectrum[(
weighted_peak_two_center_position +
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2):]) / len(spectrum[(
weighted_peak_two_center_position +
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2):])
# normalized integrated peaks
int_peak_one_norm = np.sum(
spectrum[(weighted_peak_one_center_position -
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2):
(weighted_peak_one_center_position +
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)]
) / len(spectrum[(weighted_peak_one_center_position -
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2):
(weighted_peak_one_center_position +
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)])
int_peak_two_norm = np.sum(
spectrum[(weighted_peak_two_center_position -
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2):
(weighted_peak_two_center_position +
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)]
) / len(spectrum[(weighted_peak_two_center_position -
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2):
(weighted_peak_two_center_position +
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)])
else:
# convert eV range of iron edge (7112 eV) to pixels:
metal_edge_max = (self.forbidden_range_eV / 2.) / 0.059 + 738
metal_edge_min = (-self.forbidden_range_eV / 2.) / 0.059 + 738
int_metal_region = np.sum(
spectrum[metal_edge_min:metal_edge_max])
#peak one region integrate:
int_peak_one = np.sum(spectrum[0:metal_edge_min])
int_peak_two = np.sum(spectrum[metal_edge_max:])
# now to do the filtering
if not one_D:
if min(int_peak_one_norm, int_peak_two_norm) / max(
int_peak_one_norm,
int_peak_two_norm) < self.peak_ratio:
print("event(): too low")
evt.put(skip_event_flag(), "skip_event")
return
if (np.argmax(spectrum2) + len(spectrum2)) > (
weighted_peak_two_center_position +
(len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) /
2)) or (np.argmax(spectrum2) + len(spectrum2)) < (
weighted_peak_two_center_position -
(len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)):
print("event(): out of range high energy peak")
evt.put(skip_event_flag(), "skip_event")
return
if np.argmax(spectrum1) > (
weighted_peak_one_center_position +
(len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) /
2)) or np.argmax(spectrum1) < (
weighted_peak_one_center_position -
(len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)):
print("event(): out of range low energy peak")
evt.put(skip_event_flag(), "skip_event")
return
if int_left_region_norm / int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print("event(): noisy left of low energy peak")
evt.put(skip_event_flag(), "skip_event")
return
if int_middle_region_norm / int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print("event(): noisy middle")
evt.put(skip_event_flag(), "skip_event")
return
if int_middle_region_norm / int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print("event(): noisy middle")
evt.put(skip_event_flag(), "skip_event")
return
if int_right_region_norm / int_peak_two_norm > self.normalized_peak_to_noise_ratio:
print("event(): noisy right of high energy peak")
evt.put(skip_event_flag(), "skip_event")
return
else:
# filter for cxih8015
#iron edge at 738 pixels on FFE detetor
if int_metal_region >= 0.10 * int_peak_two:
print("event(): high intensity at metal edge")
evt.put(skip_event_flag(), "skip_event")
return
if int_metal_region >= 0.10 * int_peak_one:
print("event(): high intensity at metal edge")
evt.put(skip_event_flag(), "skip_event")
return
if min(int_peak_one, int_peak_two) / max(
int_peak_one, int_peak_two) < self.peak_ratio:
print("event(): peak ratio too low")
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print("%d Two Color shots" % self.ntwo_color)
def event(self, evt, env):
if (evt.get("skip_event")):
return
self.nshots += 1
s = None
t = evt.getTime()
if (t is not None):
s = t.seconds() + (t.nanoseconds() / 1000000000)
else:
self.nfail += 1
self.logger.warning("event(): no timestamp, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if (not isinstance(s, float)):
raise RuntimeError("Wrong type for 's': %s" % type(s).__name__)
# XXX This hardcodes the address for the front detector!
det_z = cspad_tbx.env_detz('CxiDs1-0|Cspad-0', env)
if (det_z is None):
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
laser01 = cspad_tbx.env_laser_status(env, 1)
if laser01 is None:
self.nfail += 1
self.logger.warning("event(): no status for laser 1, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
laser04 = cspad_tbx.env_laser_status(env, 4)
if laser04 is None:
self.nfail += 1
self.logger.warning("event(): no status for laser 4, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
# Laser power for fourth laser. The control name was provided by
# Jan Kern. XXX Move to its own function in cspad_tbx?
laser04_power = None
if env is not None:
pv = env.epicsStore().value('CXI:LAS:MMN:02:ROT.RBV')
if pv is not None and len(pv.values) == 1:
laser04_power = pv.values[0]
if laser04_power is None:
self.nfail += 1
self.logger.warning("event(): no power for laser 4, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
si_foil = cspad_tbx.env_sifoil(env)
if (si_foil is None):
self.nfail += 1
self.logger.warning("event(): no Si-foil thickness, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if (not (isinstance(si_foil, float) or isinstance(si_foil, int))):
raise RuntimeError("Wrong type for 'si_foil': %s" %
type(si_foil).__name__)
wavelength = cspad_tbx.evt_wavelength(evt)
if (wavelength is None):
self.nfail += 1
self.logger.warning("event(): no wavelength, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# In order to keep all arrays the same length, only append once
# all values have been successfully obtained. XXX Still bugs: see
# June run 119.
self._t.append(s)
self._si_foil.append(si_foil)
self._wavelength.append(wavelength)
self._det_z.append(det_z)
self._laser01.append(laser01)
self._laser04.append(laser04)
self._laser04_power.append(laser04_power)
if (self.nshots % 120 == 0):
self.update_plot()
def event (self, evt, env) :
if (evt.get("skip_event")) :
return
self.nshots += 1
s = None
t = evt.getTime()
if (t is not None):
s = t.seconds() + (t.nanoseconds() / 1000000000)
else :
self.nfail += 1
self.logger.warning("event(): no timestamp, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if (not isinstance(s, float)) :
raise RuntimeError("Wrong type for 's': %s" % type(s).__name__)
# XXX This hardcodes the address for the front detector!
det_z = cspad_tbx.env_detz('CxiDs1-0|Cspad-0', env)
if (det_z is None):
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
laser01 = cspad_tbx.env_laser_status(env, 1)
if laser01 is None:
self.nfail += 1
self.logger.warning("event(): no status for laser 1, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
laser04 = cspad_tbx.env_laser_status(env, 4)
if laser04 is None:
self.nfail += 1
self.logger.warning("event(): no status for laser 4, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
# Laser power for fourth laser. The control name was provided by
# Jan Kern. XXX Move to its own function in cspad_tbx?
laser04_power = None
if env is not None:
pv = env.epicsStore().value('CXI:LAS:MMN:02:ROT.RBV')
if pv is not None and len(pv.values) == 1:
laser04_power = pv.values[0]
if laser04_power is None:
self.nfail += 1
self.logger.warning("event(): no power for laser 4, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
si_foil = cspad_tbx.env_sifoil(env)
if (si_foil is None):
self.nfail += 1
self.logger.warning("event(): no Si-foil thickness, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if (not (isinstance(si_foil, float) or isinstance(si_foil, int))) :
raise RuntimeError("Wrong type for 'si_foil': %s"% type(si_foil).__name__)
wavelength = cspad_tbx.evt_wavelength(evt)
if (wavelength is None):
self.nfail += 1
self.logger.warning("event(): no wavelength, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
# In order to keep all arrays the same length, only append once
# all values have been successfully obtained. XXX Still bugs: see
# June run 119.
self._t.append(s)
self._si_foil.append(si_foil)
self._wavelength.append(wavelength)
self._det_z.append(det_z)
self._laser01.append(laser01)
self._laser04.append(laser04)
self._laser04_power.append(laser04_power)
if (self.nshots % 120 == 0) :
self.update_plot()
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata +=1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(),"skip_event")
if timestamp is None:
evt.put(skip_event_flag(),"skip_event")
#self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots +=1
# get data as array and split into two half to find each peak
if one_D:
# filtering out outlier spikes in FEE data
data = np.array(data.hproj().astype(np.float64))
for i in xrange(len(data)):
if data[i]>1000000000:
data[i]=data[i]-(2**32)
if self.dark is not None:
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0]//2]
spectrum2 = data[data.shape[0]//2:]
else:
data = np.array(data.data16().astype(np.int32))
if self.dark is not None:
data = data - self.dark
data_split1 = data[:,:data.shape[1]//2]
data_split2 = data[:,data.shape[1]//2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data,0)/data.shape[0]
spectrum1 = np.sum(data_split1,0)/data_split1.shape[0]
spectrum2 = np.sum(data_split2,0)/data_split2.shape[0]
if not one_D:
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in xrange(self.peak_one_range_min,self.peak_one_range_max):
weighted_peak_one_positions.append(spectrum[i]*i)
weighted_sum_peak_one = sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one//sum(spectrum[self.peak_one_range_min:self.peak_one_range_max])
weighted_peak_two_positions = []
for i in xrange(self.peak_two_range_min,self.peak_two_range_max):
weighted_peak_two_positions.append(spectrum[i]*i)
weighted_sum_peak_two = sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two//sum(spectrum[self.peak_two_range_min:self.peak_two_range_max])
# normalized integrated regions between the peaks
int_left_region_norm = np.sum(spectrum[0:(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])/len(spectrum[0:(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_middle_region_norm = np.sum(spectrum[(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_right_region_norm = np.sum(spectrum[(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):])/len(spectrum[(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):])
# normalized integrated peaks
int_peak_one_norm = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])/len(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_two_norm = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
else:
# convert eV range of iron edge (7112 eV) to pixels:
metal_edge_max=(self.forbidden_range_eV/2.)/0.059+738
metal_edge_min=(-self.forbidden_range_eV/2.)/0.059+738
int_metal_region = np.sum(spectrum[metal_edge_min:metal_edge_max])
#peak one region integrate:
int_peak_one=np.sum(spectrum[0:metal_edge_min])
int_peak_two=np.sum(spectrum[metal_edge_max:])
# now to do the filtering
if not one_D:
if min(int_peak_one_norm,int_peak_two_norm)/max(int_peak_one_norm,int_peak_two_norm) < self.peak_ratio:
print "event(): too low"
evt.put(skip_event_flag(), "skip_event")
return
if (np.argmax(spectrum2)+len(spectrum2)) > (weighted_peak_two_center_position+(len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)) or (np.argmax(spectrum2)+len(spectrum2)) < (weighted_peak_two_center_position-(len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)):
print "event(): out of range high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if np.argmax(spectrum1) > (weighted_peak_one_center_position+(len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)) or np.argmax(spectrum1) < (weighted_peak_one_center_position-(len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)):
print "event(): out of range low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if int_left_region_norm/int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print "event(): noisy left of low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if int_middle_region_norm/int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if int_middle_region_norm/int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if int_right_region_norm/int_peak_two_norm > self.normalized_peak_to_noise_ratio:
print "event(): noisy right of high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
else:
# filter for cxih8015
#iron edge at 738 pixels on FFE detetor
if int_metal_region>=0.10*int_peak_two:
print "event(): high intensity at metal edge"
evt.put(skip_event_flag(), "skip_event")
return
if int_metal_region>=0.10*int_peak_one:
print "event(): high intensity at metal edge"
evt.put(skip_event_flag(), "skip_event")
return
if min(int_peak_one,int_peak_two)/max(int_peak_one,int_peak_two) < self.peak_ratio:
print "event(): peak ratio too low"
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print "%d Two Color shots" %self.ntwo_color
def event(self, evt, env):
from dxtbx.format.Registry import Registry
from os.path import exists
from time import sleep
# Nop if there is no image. For experiments configured to have
# exactly one event per calibration cycle, this should never
# happen.
if self._path is None:
evt.put(skip_event_flag(), "skip_event")
return
# Skip this event if the template isn't in the path
if self._template is not None and not True in [t in self._path for t in self._template.split(',')]:
evt.put(skip_event_flag(), "skip_event")
return
if "phi" in self._path:
evt.put(skip_event_flag(), "skip_event")
return
# Wait for the image to appear in the file system, probing for it
# at exponentially increasing delays.
t = 1
t_tot = 0
if not exists(self._path):
self._logger.info("Waiting for path %s"%self._path)
while not exists(self._path):
if t_tot > 1:
self._logger.info("Timeout waiting for path %s"%self._path)
evt.put(skip_event_flag(), "skip_event")
self._logger.info("Image not found: %s"%self._path)
return
sleep(t)
t_tot += t
t *= 2
# Find a matching Format object and instantiate it using the
# given path. If the Format object does not understand the image,
# try determining a new format. XXX Emits "Couldn't create a
# detector model for this image".
if self._fmt is None:
self._fmt = Registry.find(self._path)
if self._fmt is None:
evt.put(skip_event_flag(), "skip_event")
return
img = self._fmt(self._path)
if img is None:
self._fmt = Registry.find(self._path)
if self._fmt is None:
evt.put(skip_event_flag(), "skip_event")
return
img = self._fmt(self._path)
if img is None:
evt.put(skip_event_flag(), "skip_event")
return
self._logger.info(
"Reading %s using %s" % (self._path, self._fmt.__name__))
# Get the raw image data and convert to double precision floating
# point array. XXX Why will img.get_raw_data() not work, like it
# does in print_header?
db = img.get_detectorbase()
db.readHeader()
db.read()
data = db.get_raw_data().as_double()
# Get the pixel size and store it for common_mode.py
detector = img.get_detector()[0]
ps = detector.get_pixel_size()
assert ps[0] == ps[1]
pixel_size = ps[0]
evt.put(ps[0],"marccd_pixel_size")
evt.put(detector.get_trusted_range()[1],"marccd_saturated_value")
evt.put(detector.get_distance(),"marccd_distance")
# If the beam center isn't provided in the config file, get it from the
# image. It will probably be wrong.
if self._beam_x is None or self._beam_y is None:
self._beam_x, self._beam_y = detector.get_beam_centre_px(img.get_beam().get_s0())
self._beam_x = int(round(self._beam_x))
self._beam_y = int(round(self._beam_y))
# Crop the data so that the beam center is in the center of the image
maxy, maxx = data.focus()
minsize = min([self._beam_x,self._beam_y,maxx-self._beam_x,maxy-self._beam_y])
data = data[self._beam_y-minsize:self._beam_y+minsize,self._beam_x-minsize:self._beam_x+minsize]
evt.put((minsize,minsize),"marccd_beam_center")
evt.put(flex.int([0,0,minsize*2,minsize*2]),"marccd_active_areas")
# Store the image in the event.
evt.put(data, self._address)
# Store the .mmcd file name in the event
evt.put(self._mccd_name, "mccd_name")
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
self._logger.info("converted %s to pickle with timestamp %s" %(self._path, timestamp))
# This should not be necessary as the machine is configured with
# one event per calibration cycle.
self._path = None
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(average_mixin, self).event(evt, env)
if evt.get('skip_event'):
return
# Get the distance for the detectors that should have it, and set
# it to NaN for those that should not.
if self.detector == 'CxiDs1' or \
self.detector == 'CxiDs2' or \
self.detector == 'CxiDsd' or \
self.detector == 'XppGon':
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self._nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
else:
distance = float('nan')
if ("skew" in self.flags):
# Take out inactive pixels
if self.roi is not None:
pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
pixels = pixels.as_1d().select(dark_mask.as_1d())
else:
pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double()
stats = scitbx.math.basic_statistics(pixels.as_double())
#stats.show()
self.logger.info("skew: %.3f" %stats.skew)
self.logger.info("kurtosis: %.3f" %stats.kurtosis)
if 0:
from matplotlib import pyplot
hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double())
print hist_min, hist_max
n_slots = 100
n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
pyplot.show()
# XXX This skew threshold probably needs fine-tuning
skew_threshold = 0.35
if stats.skew < skew_threshold:
self._nfail += 1
self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold)
evt.put(skip_event_flag(), 'skip_event')
return
#self.cspad_img *= stats.skew
if ("inactive" in self.flags):
self.cspad_img.set_selected(self.dark_stddev <= 0, 0)
if ("noelastic" in self.flags):
ELASTIC_THRESHOLD = self.elastic_threshold
self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0)
if self.hot_threshold is not None:
HOT_THRESHOLD = self.hot_threshold
self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0)
if self.gain_map is not None and self.gain_threshold is not None:
# XXX comparing each pixel to a moving average would probably be better
# since the gain should vary approximately smoothly over different areas
# of the detector
GAIN_THRESHOLD = self.gain_threshold
#self.logger.debug(
#"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True))
self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0)
if ("nonoise" in self.flags):
NOISE_THRESHOLD = self.noise_threshold
self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0)
if ("sigma_scaling" in self.flags):
self.do_sigma_scaling()
if ("symnoise" in self.flags):
SYMNOISE_THRESHOLD = self.symnoise_threshold
self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) &
( self.cspad_img < SYMNOISE_THRESHOLD), 0)
if ("output" in self.flags):
import pickle,os
if (not os.path.isdir(self.pickle_dirname)):
os.makedirs(self.pickle_dirname)
flexdata = flex.int(self.cspad_img.astype(numpy.int32))
d = cspad_tbx.dpack(
address=self.address,
data=flexdata,
timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
)
G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename,
"ab")
pickle.dump(d,G,pickle.HIGHEST_PROTOCOL)
G.close()
if self.photon_threshold is not None and self.two_photon_threshold is not None:
self.do_photon_counting()
if self.background_path is not None:
self.cspad_img -= self.background_img
# t and self._sum_time are a two-long arrays of seconds and
# milliseconds which hold time with respect to the base time.
t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt),
self._metadata['time_base'])]
if self._nmemb == 0:
# The peers metadata item is a bit field where a bit is set if
# the partial sum from the corresponding worker process is
# pending. If this is the first frame a worker process sees,
# set its corresponding bit in the bit field since it will
# contribute a partial sum.
if env.subprocess() >= 0:
self._lock.acquire()
if 'peers' in self._metadata.keys():
self._metadata['peers'] |= (1 << env.subprocess())
else:
self._metadata['peers'] = (1 << env.subprocess())
self._lock.release()
self._sum_distance = distance
self._sum_time = (t[0], t[1])
self._sum_wavelength = self.wavelength
if self._have_max:
self._max_img = self.cspad_img.deep_copy()
if self._have_mean:
self._sum_img = self.cspad_img.deep_copy()
if self._have_std:
self._ssq_img = flex.pow2(self.cspad_img)
else:
self._sum_distance += distance
self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1])
self._sum_wavelength += self.wavelength
if self._have_max:
sel = (self.cspad_img > self._max_img).as_1d()
self._max_img.as_1d().set_selected(
sel, self.cspad_img.as_1d().select(sel))
if self._have_mean:
self._sum_img += self.cspad_img
if self._have_std:
self._ssq_img += flex.pow2(self.cspad_img)
self._nmemb += 1
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
XXX Since the viewer is now running in a parallel process, the
averaging here is now the bottleneck.
@param evt Event data object, a configure object
@param env Environment object
"""
from pyana.event import Event
self.n_shots += 1
super(mod_view, self).event(evt, env)
if evt.status() != Event.Normal or evt.get(
'skip_event'): # XXX transition
return
# Get the distance for the detectors that should have it, and set
# it to NaN for those that should not.
if self.detector == 'CxiDs1' or \
self.detector == 'CxiDsd' or \
self.detector == 'XppGon':
distance = cspad_tbx.env_distance(self.address, env,
self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
distance = float('nan')
if not self._proc.is_alive():
evt.setStatus(Event.Stop)
# Early return if the next update to the viewer is more than
# self.ncollate shots away. XXX Since the common_mode.event()
# function does quite a bit of processing, the savings are
# probably not so big.
next_update = (self.nupdate - 1) - (self.nshots - 1) % self.nupdate
if (self.ncollate > 0 and next_update >= self.ncollate):
return
if self.sigma_scaling:
self.do_sigma_scaling()
if self.photon_counting:
self.do_photon_counting()
# Trim the disabled section from the Sc1 detector image. XXX This
# is a bit of a kludge, really.
# if (self.address == "CxiSc1-0|Cspad2x2-0"):
# self.cspad_img = self.cspad_img[185:2 * 185, :]
# Update the sum of the valid images, starting a new collation if
# appropriate. This guarantees self.nvalid > 0.
if (self.nvalid == 0
or self.ncollate > 0 and self.nvalid >= self.ncollate):
self.img_sum = self.cspad_img
self.nvalid = 1
else:
self.img_sum += self.cspad_img
self.nvalid += 1
# Update the viewer to display the current average image, and
# start a new collation, if appropriate.
if (next_update == 0):
from time import localtime, strftime
time_str = strftime("%H:%M:%S", localtime(evt.getTime().seconds()))
title = "r%04d@%s: average of %d last images on %s" \
% (evt.run(), time_str, self.nvalid, self.address)
# See also mod_average.py.
device = cspad_tbx.address_split(self.address)[2]
if device == 'Cspad':
beam_center = self.beam_center
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
beam_center = tuple(t // 2 for t in self.img_sum.focus())
pixel_size = 0.079346
saturated_value = 2**16 - 1
# Wait for the viewer process to empty the queue before feeding
# it a new image, and ensure not to hang if the viewer process
# exits. Because of multithreading/multiprocessing semantics,
# self._queue.empty() is unreliable.
fmt = _Format(BEAM_CENTER=beam_center,
DATA=self.img_sum / self.nvalid,
DETECTOR_ADDRESS=self.address,
DISTANCE=distance,
PIXEL_SIZE=pixel_size,
SATURATED_VALUE=saturated_value,
TIME_TUPLE=cspad_tbx.evt_time(evt),
WAVELENGTH=self.wavelength)
while not self._queue.empty():
if not self._proc.is_alive():
evt.setStatus(Event.Stop)
return
while True:
try:
self._queue.put((fmt, title), timeout=1)
break
except Exception:
pass
if (self.ncollate > 0):
self.nvalid = 0
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata +=1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(),"skip_event")
if timestamp is None:
evt.put(skip_event_flag(),"skip_event")
self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots +=1
# get data as array and split into two half to find each peak
if one_D:
data = np.array(data.hproj().astype(np.float64))
if 'dark' in locals():
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0]//2]
spectrum2 = data[data.shape[0]//2:]
else:
data = np.array(data.data16().astype(np.float64))
if 'dark' in locals():
data = data - self.dark
data_split1 = data[:,:data.shape[1]//2]
data_split2 = data[:,data.shape[1]//2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data,0)/data.shape[0]
spectrum1 = np.sum(data_split1,0)/data_split1.shape[0]
spectrum2 = np.sum(data_split2,0)/data_split2.shape[0]
peak_one = np.max(spectrum1)
peak_two = np.max(spectrum2)
peak_one_position = np.argmax(spectrum1)
peak_two_position = np.argmax(spectrum2) + len(spectrum2)
# define the limits of the regions between the two peaks
peak_one_lower_limit = self.peak_one_position_min - self.peak_one_width
peak_one_upper_limit = self.peak_one_position_max + self.peak_one_width
peak_two_lower_limit = self.peak_two_position_min - self.peak_two_width
peak_two_upper_limit = self.peak_two_position_max + self.peak_two_width
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in xrange(peak_one_lower_limit,peak_one_upper_limit):
weighted_peak_one_positions.append(spectrum[i]*i)
weighted_sum_peak_one = sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one//sum(spectrum[peak_one_lower_limit:peak_one_upper_limit])
weighted_peak_two_positions = []
for i in xrange(peak_two_lower_limit,peak_two_upper_limit):
weighted_peak_two_positions.append(spectrum[i]*i)
weighted_sum_peak_two = sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two//sum(spectrum[peak_two_lower_limit:peak_two_upper_limit])
# normalized integrated peaks
int_peak_one = np.sum(spectrum[peak_one_lower_limit:self.peak_one_position_max])/len(spectrum[peak_one_lower_limit:self.peak_one_position_max])
int_peak_two = np.sum(spectrum[peak_two_lower_limit:self.peak_two_position_max])/len(spectrum[peak_two_lower_limit:self.peak_two_position_max])
# normalized integrated regions between the peaks
int_left_region = np.sum(spectrum[0:peak_one_lower_limit])/len(spectrum[0:peak_one_lower_limit])
int_middle_region = np.sum(spectrum[peak_one_upper_limit:peak_two_lower_limit])/len(spectrum[peak_one_upper_limit:peak_two_lower_limit])
int_right_region = np.sum(spectrum[peak_two_upper_limit:])/len(spectrum[:peak_two_upper_limit])
# now to do the filtering
if peak_one/peak_two < self.peak_ratio or peak_one/peak_two > 1/self.peak_ratio:
print "event(): too low"
evt.put(skip_event_flag(), "skip_event")
return
if weighted_peak_two_center_position < self.peak_two_position_min or weighted_peak_two_center_position > self.peak_two_position_max:
print "event(): out of range high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if weighted_peak_one_center_position < self.peak_one_position_min or weighted_peak_one_center_position > self.peak_one_position_max:
print "event(): out of range low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_left_region/int_peak_one > self.normalized_peak_to_noise_ratio):
print "event(): noisy left of low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_middle_region/int_peak_one > self.normalized_peak_to_noise_ratio):
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_middle_region/int_peak_two > self.normalized_peak_to_noise_ratio):
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_right_region/int_peak_two > self.normalized_peak_to_noise_ratio):
print "event(): noisy right of high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
#iron edge at 738 pixels on FFE detetor
if one_D and (spectrum[self.iron_edge_position]>=spectrum[weighted_peak_one_center_position]):
print "event(): peak at iron edge"
evt.put(skip_event_flag(), "skip_event")
return
if one_D and (spectrum[self.iron_edge_position]>=spectrum[weighted_peak_two_center_position]):
print "event(): peak at iron edge"
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print "%d Two Color shots" %self.ntwo_color
class common_mode_correction(mod_event_info):
"""Dark subtraction and alternate implementation of common mode
substituting for cspad_tbx.
Known problems: the algorithm relies on a substantial part of the
sensor having no signal, which is not the case if a water ring
crosses the sensor.
"""
def __init__(self,
address,
calib_dir=None,
common_mode_correction="none",
photon_threshold=None,
two_photon_threshold=None,
dark_path=None,
dark_stddev=None,
mask_path=None,
gain_map_path=None,
gain_map_level=None,
cache_image=True,
roi=None,
laser_1_status=None,
laser_4_status=None,
laser_wait_time=None,
override_beam_x=None,
override_beam_y=None,
bin_size=None,
crop_rayonix=False,
**kwds):
"""The common_mode_correction class constructor stores the
parameters passed from the pyana configuration file in instance
variables.
@param address Full data source address of the DAQ device
@param calib_dir Directory with calibration information
@param common_mode_correction The type of common mode correction to apply
@param dark_path Path to input average dark image
@param dark_stddev Path to input standard deviation dark
image, required if @p dark_path is given
@param mask_path Path to input mask. Pixels to mask out should be set to -2
@param gain_map_path Path to input gain map. Multiplied times the image.
@param gain_map_level If set, all the '1' pixels in the gain_map are set to this multiplier
and all the '0' pixels in the gain_map are set to '1'. If not set,
use the values in the gain_map directly
@param laser_1_status 0 or 1 to indicate that the laser should be off or on respectively
@param laser_4_status 0 or 1 to indicate that the laser should be off or on respectively
@param laser_wait_time Length of time in milliseconds to wait after a laser
change of status to begin accepting images again.
(rejection of images occurs immediately after status
change).
@param override_beam_x override value for x coordinate of beam center in pixels
@param override_beam_y override value for y coordinate of beam center in pixels
@param bin_size bin size for rayonix detector used to determin pixel size
@param crop_rayonix whether to crop rayonix images such that image center is the beam center
"""
# Cannot use the super().__init__() construct here, because
# common_mode_correction refers to the argument, and not the
# class.
mod_event_info.__init__(self, address=address, **kwds)
# The paths will be substituted in beginjob(), where evt and env
# are available.
self._dark_path = cspad_tbx.getOptString(dark_path)
self._dark_stddev_path = cspad_tbx.getOptString(dark_stddev)
self._gain_map_path = cspad_tbx.getOptString(gain_map_path)
self._mask_path = cspad_tbx.getOptString(mask_path)
self.gain_map_level = cspad_tbx.getOptFloat(gain_map_level)
self.common_mode_correction = cspad_tbx.getOptString(common_mode_correction)
self.photon_threshold = cspad_tbx.getOptFloat(photon_threshold)
self.two_photon_threshold = cspad_tbx.getOptFloat(two_photon_threshold)
self.cache_image = cspad_tbx.getOptBool(cache_image)
self.filter_laser_1_status = cspad_tbx.getOptInteger(laser_1_status)
self.filter_laser_4_status = cspad_tbx.getOptInteger(laser_4_status)
if self.filter_laser_1_status is not None:
self.filter_laser_1_status = bool(self.filter_laser_1_status)
if self.filter_laser_4_status is not None:
self.filter_laser_4_status = bool(self.filter_laser_4_status)
self.filter_laser_wait_time = cspad_tbx.getOptInteger(laser_wait_time)
self.override_beam_x = cspad_tbx.getOptFloat(override_beam_x)
self.override_beam_y = cspad_tbx.getOptFloat(override_beam_y)
self.bin_size = cspad_tbx.getOptInteger(bin_size)
self.crop_rayonix = cspad_tbx.getOptBool(crop_rayonix)
self.cspad_img = None # The current image - set by self.event()
self.sum_common_mode = 0
self.sumsq_common_mode = 0
self.roi = cspad_tbx.getOptROI(roi) # used to ignore the signal region in chebyshev fit
assert self.common_mode_correction in \
("gaussian", "mean", "median", "mode", "none", "chebyshev")
# Get and parse metrology.
self.sections = None
device = cspad_tbx.address_split(self.address)[2]
if device == 'Andor':
self.sections = [] # XXX FICTION
elif device == 'Cspad':
if self.address == 'XppGon-0|Cspad-0':
self.sections = [] # Not used for XPP
else:
self.sections = calib2sections(cspad_tbx.getOptString(calib_dir))
elif device == 'Cspad2x2':
# There is no metrology information for the Sc1 detector, so
# make it up. The sections are rotated by 90 degrees with
# respect to the "standing up" convention.
self.sections = [[Section(90, (185 / 2 + 0, (2 * 194 + 3) / 2)),
Section(90, (185 / 2 + 185, (2 * 194 + 3) / 2))]]
elif device == 'marccd':
self.sections = [] # XXX FICTION
elif device == 'pnCCD':
self.sections = [] # XXX FICTION
elif device == 'Rayonix':
self.sections = [] # XXX FICTION
elif device == 'Opal1000':
self.sections = [] # XXX FICTION
if self.sections is None:
raise RuntimeError("Failed to load metrology")
def beginjob(self, evt, env):
"""The beginjob() function does one-time initialisation from
event- or environment data. It is called at an XTC configure
transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).beginjob(evt, env)
# Load the dark image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing. If a dark image
# is provided, a standard deviation image is required, and all the
# ADU scales must match up.
#
# XXX Can we zap all the ADU_SCALE stuff?
#
# XXX Do we really need to store the substituted values in the
# instance here? At least self._dark_path is referenced later on.
#
# Note that this will load the dark, standard deviation and gain
# once (SEVERAL TIMES) for each process, but the gain is that we
# can do substitutions. But it is only done at the beginning of
# the job.
self.dark_img = None
if self._dark_path is not None:
self._dark_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_path, evt, env))
assert self._dark_stddev_path is not None
dark_dict = easy_pickle.load(self._dark_path)
#assert "ADU_SCALE" not in dark_dict # force use of recalculated dark
self.dark_img = dark_dict['DATA']
assert isinstance(self.dark_img, flex.double)
self._dark_stddev_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_stddev_path, evt, env))
self.dark_stddev = easy_pickle.load(self._dark_stddev_path)['DATA']
assert isinstance(self.dark_stddev, flex.double)
self.dark_mask = (self.dark_stddev > 0)
# Load the mask image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing.
self.gain_map = None
if self._gain_map_path is not None:
self._gain_map_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._gain_map_path, evt, env))
self.gain_map = easy_pickle.load(self._gain_map_path)['DATA']
if self.gain_map_level is not None:
sel = flex.bool([bool(f) for f in self.gain_map])
sel.reshape(flex.grid(self.gain_map.focus()))
self.gain_map = self.gain_map.set_selected(~sel, self.gain_map_level)
self.gain_map = self.gain_map.set_selected(sel, 1)
assert isinstance(self.gain_map, flex.double)
self.mask_img = None
if self._mask_path is not None:
self._mask_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._mask_path, evt, env))
self.mask_img = easy_pickle.load(self._mask_path)['DATA']
assert isinstance(self.mask_img, flex.double) \
or isinstance(self.mask_img, flex.int)
if self.address == 'XppGon-0|marccd-0':
#mod_mar.py will set these during its event function
self.active_areas = None
self.beam_center = None
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
assert self.override_beam_x is not None
assert self.override_beam_y is not None
from xfel.cxi.cspad_ana import rayonix_tbx
maxx, maxy = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
if self.crop_rayonix:
bx = int(round(self.override_beam_x))
by = int(round(self.override_beam_y))
minsize = min([bx,by,maxx-bx,maxy-by])
self.beam_center = minsize,minsize
self.active_areas = flex.int([0,0,2*minsize,2*minsize])
self.rayonix_crop_slice = slice(by-minsize,by+minsize), slice(bx-minsize,bx+minsize)
else:
self.beam_center = self.override_beam_x,self.override_beam_y
self.active_areas = flex.int([0,0,maxx,maxy])
elif self.address == 'XppGon-0|Cspad-0':
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = cspad_tbx.cbcaa(
cspad_tbx.getConfig(self.address, env), self.sections)
def common_mode(self, img, stddev, mask):
"""The common_mode() function returns the mode of image stored in
the array pointed to by @p img. @p mask must be such that the @p
stddev at the selected pixels is greater than zero.
@param img 2D integer array of the image
@param stddev 2D integer array of the standard deviation of each
pixel in @p img
@param mask 2D Boolean array, @c True if the pixel is to be
included, @c False otherwise
@return Mode of the image, as a real number
"""
# Flatten the image and take out inactive pixels XXX because we
# cannot take means and medians of 2D arrays?
img_1d = img.as_1d().select(mask.as_1d()).as_double()
assert img_1d.size() > 0
if (self.common_mode_correction == "mean"):
# The common mode is approximated by the mean of the pixels with
# signal-to-noise ratio less than a given threshold. XXX Breaks
# if the selection is empty!
THRESHOLD_SNR = 2
img_snr = img_1d / stddev.as_double().as_1d().select(mask.as_1d())
return (flex.mean(img_1d.select(img_snr < THRESHOLD_SNR)))
elif (self.common_mode_correction == "median"):
return (flex.median(img_1d))
# Identify the common-mode correction as the peak histogram of the
# histogram of pixel values (the "standard" common-mode correction, as
# previously implemented in this class).
hist_min = -40
hist_max = 40
n_slots = 100
hist = flex.histogram(img_1d, hist_min, hist_max, n_slots=n_slots)
slots = hist.slots()
i = flex.max_index(slots)
common_mode = list(hist.slot_infos())[i].center()
if (self.common_mode_correction == "mode"):
return (common_mode)
# Determine the common-mode correction from the peak of a single
# Gaussian function fitted to the histogram.
from scitbx.math.curve_fitting import single_gaussian_fit
x = hist.slot_centers()
y = slots.as_double()
fit = single_gaussian_fit(x, y)
scale, mu, sigma = fit.a, fit.b, fit.c
self.logger.debug("fitted gaussian: mu=%.3f, sigma=%.3f" %(mu, sigma))
mode = common_mode
common_mode = mu
if abs(mode-common_mode) > 1000: common_mode = mode # XXX
self.logger.debug("delta common mode corrections: %.3f" %(mode-common_mode))
if 0 and abs(mode-common_mode) > 0:
#if 0 and skew > 0.5:
# view histogram and fitted gaussian
from numpy import exp
from matplotlib import pyplot
x_all = x
n, bins, patches = pyplot.hist(section_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
y_all = scale * flex.exp(-flex.pow2(x_all-mu) / (2 * sigma**2))
scale = slots[flex.max_index(slots)]
y_all *= scale/flex.max(y_all)
pyplot.plot(x_all, y_all)
pyplot.show()
return (common_mode)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).event(evt, env)
if (evt.get("skip_event")):
return
if not hasattr(self, 'active_areas') or self.active_areas is None or \
not hasattr(self, 'beam_center') or self.beam_center is None:
if self.address == 'XppGon-0|marccd-0':
# The mod_mar module needs to have been called before this one
# to set this up. The MAR does not have a configure object.
self.beam_center = evt.get("marccd_beam_center")
self.active_areas = evt.get("marccd_active_areas")
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
pass # bc and aa set in the beginjob function
elif self.address == 'XppGon-0|Cspad-0':
# Load the active areas as determined from the optical metrology
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(self.timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = \
cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)
if self.filter_laser_1_status is not None:
if (self.laser_1_status.status != self.filter_laser_1_status or
(self.laser_1_ms_since_change is not None and
self.laser_1_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
if self.filter_laser_4_status is not None:
if (self.laser_4_status.status != self.filter_laser_4_status or
(self.laser_4_ms_since_change is not None and
self.laser_4_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
# Early return if the full detector image is already stored in the
# event. Otherwise, get it from the stream as a double-precision
# floating-point flex array. XXX It is probably not safe to key
# the image on self.address, so we should come up with our own
# namespace. XXX Misnomer--could be CAMP, too
self.cspad_img = evt.get(self.address)
if self.cspad_img is not None:
return
if self.address == 'XppGon-0|Cspad-0':
# Kludge until cspad_tbx.image() can be rewritten to handle the
# XPP metrology.
self.cspad_img = cspad_tbx.image_xpp(
self.address, evt, env, self.active_areas)
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
from psana import Source, Camera
import numpy as np
address = cspad_tbx.old_address_to_new_address(self.address)
src=Source('DetInfo(%s)'%address)
self.cspad_img = evt.get(Camera.FrameV1,src)
if self.cspad_img is not None:
self.cspad_img = self.cspad_img.data16().astype(np.float64)
elif self.address=='CxiDg3-0|Opal1000-0':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='CxiEndstation-0|Opal1000-2':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='FeeHxSpectrometer-0|Opal1000-1':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='NoDetector-0|Cspad2x2-0':
import numpy as np
from pypdsdata import xtc
test=[]
self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,self.address).data()
self.cspad_img=np.reshape(self.cspad_img,(370, 388))
else:
try:
self.cspad_img = cspad_tbx.image(
self.address, cspad_tbx.getConfig(self.address, env),
evt, env, self.sections)
except Exception, e:
self.logger.error("Error reading image data: " + str(e))
evt.put(skip_event_flag(), "skip_event")
return
if self.cspad_img is None:
if cspad_tbx.address_split(self.address)[2] != 'Andor':
self.nfail += 1
self.logger.warning("event(): no image, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
self.cspad_img = flex.double(self.cspad_img.astype(numpy.float64))
# If a dark image was provided, subtract it from the image. There
# is no point in doing common-mode correction unless the dark
# image was subtracted.
if (self.dark_img is not None):
self.cspad_img -= self.dark_img
if (self.common_mode_correction != "none"):
# Mask out inactive pixels prior to common mode correction.
# Pixels are marked as inactive either due to low ADU values
# or non-positive standard deviations in dark image. XXX Make
# the threshold tunable?
cspad_mask = self.dark_mask.deep_copy()
if self.roi is not None and self.common_mode_correction == "chebyshev":
roi_mask = cspad_mask[self.roi[2]:self.roi[3], :]
roi_mask = flex.bool(roi_mask.accessor(), False)
cspad_mask.matrix_paste_block_in_place(
block=roi_mask,
i_row=self.roi[2],
i_column=0)
# Extract each active section from the assembled detector
# image and apply the common mode correction. XXX Make up a
# quadrant mask for the emission detector. Needs to be
# checked!
config = cspad_tbx.getConfig(self.address, env)
if len(self.sections) == 1:
q_mask = 1
else:
q_mask = config.quadMask()
for q in xrange(len(self.sections)):
if (not((1 << q) & q_mask)):
continue
# XXX Make up section mask for the emission detector. Needs
# to be checked!
import _pdsdata
if len(self.sections) == 1 and type(config) in (
_pdsdata.cspad2x2.ConfigV1, _pdsdata.cspad2x2.ConfigV2):
s_mask = config.roiMask()
else:
s_mask = config.roiMask(q)
for s in xrange(len(self.sections[q])):
# XXX DAQ misconfiguration? This mask appears not to work
# reliably for the Sc1 detector.
# if (not((1 << s) & s_mask)):
# continue
corners = self.sections[q][s].corners()
i_row = int(round(min(c[0] for c in corners)))
i_column = int(round(min(c[1] for c in corners)))
n_rows = int(round(max(c[0] for c in corners))) - i_row
n_columns = int(round(max(c[1] for c in corners))) - i_column
section_img = self.cspad_img.matrix_copy_block(
i_row = i_row, i_column = i_column,
n_rows = n_rows, n_columns = n_columns)
section_mask = cspad_mask.matrix_copy_block(
i_row = i_row, i_column = i_column,
n_rows = n_rows, n_columns = n_columns)
section_stddev = self.dark_stddev.matrix_copy_block(
i_row = i_row, i_column = i_column,
n_rows = n_rows, n_columns = n_columns)
if section_mask.count(True) == 0: continue
if self.common_mode_correction == "chebyshev":
assert len(self.sections[q]) == 2
if s == 0:
section_imgs = [section_img]
section_masks = [section_mask]
i_rows = [i_row]
i_columns = [i_column]
continue
else:
section_imgs.append(section_img)
section_masks.append(section_mask)
i_rows.append(i_row)
i_columns.append(i_column)
chebyshev_corrected_imgs = self.chebyshev_common_mode(
section_imgs, section_masks)
for i in range(2):
section_imgs[i].as_1d().copy_selected(
section_masks[i].as_1d().iselection(),
chebyshev_corrected_imgs[i].as_1d())
self.cspad_img.matrix_paste_block_in_place(
block=section_imgs[i],
i_row=i_rows[i],
i_column=i_columns[i])
else:
common_mode = self.common_mode(
section_img, section_stddev, section_mask)
self.sum_common_mode += common_mode
self.sumsq_common_mode += common_mode**2
# Apply the common mode correction to the
# section, and paste it back into the image.
self.cspad_img.matrix_paste_block_in_place(
block = section_img - common_mode,
i_row = i_row,
i_column = i_column)
if self.gain_map is not None:
self.cspad_img *= self.gain_map
if (self.mask_img is not None):
sel = (self.mask_img == -2 )|(self.mask_img == cspad_tbx.cspad_mask_value)
self.cspad_img.set_selected(sel, cspad_tbx.cspad_mask_value)
if (self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0') and \
self.crop_rayonix:
# Crop the masked data so that the beam center is in the center of the image
self.cspad_img = self.cspad_img[self.rayonix_crop_slice[0], self.rayonix_crop_slice[1]]
if self.cache_image:
# Store the image in the event.
evt.put(self.cspad_img, self.address)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
# Increase the event counter, even if this event is to be skipped.
self.nshots += 1
if (evt.get("skip_event")):
return
sifoil = cspad_tbx.env_sifoil(env)
if self.check_beam_status and sifoil is None:
self.nfail += 1
self.logger.warning("event(): no Si-foil thickness, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if (self.sifoil is not None and self.sifoil != sifoil):
self.logger.warning(
"event(): Si-foil changed mid-run: % 8i -> % 8d" %
(self.sifoil, sifoil))
self.sifoil = sifoil
if self.verbose: self.logger.info("Si-foil thickness: %i" % sifoil)
self.evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
self.timestamp = cspad_tbx.evt_timestamp(
self.evt_time) # human readable format
if (self.timestamp is None):
self.nfail += 1
self.logger.warning("event(): no timestamp, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if self.verbose: self.logger.info(self.timestamp)
if self.override_energy is None:
self.wavelength = cspad_tbx.evt_wavelength(evt, self.delta_k)
if self.wavelength is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no wavelength, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.wavelength = 0
else:
self.wavelength = 12398.4187 / self.override_energy
if self.verbose: self.logger.info("Wavelength: %.4f" % self.wavelength)
self.pulse_length = cspad_tbx.evt_pulse_length(evt)
if self.pulse_length is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no pulse length, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.pulse_length = 0
if self.verbose:
self.logger.info("Pulse length: %s" % self.pulse_length)
self.beam_charge = cspad_tbx.evt_beam_charge(evt)
if self.beam_charge is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no beam charge, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.beam_charge = 0
if self.verbose: self.logger.info("Beam charge: %s" % self.beam_charge)
self.injector_xyz = cspad_tbx.env_injector_xyz(env)
#if self.injector_xyz is not None:
#self.logger.info("injector_z: %i" %self.injector_xyz[2].value)
self.laser_1_status.set_status(
cspad_tbx.env_laser_status(env, laser_id=1), self.evt_time)
self.laser_4_status.set_status(
cspad_tbx.env_laser_status(env, laser_id=4), self.evt_time)
self.laser_1_ms_since_change = self.laser_1_status.ms_since_last_status_change(
self.evt_time)
self.laser_4_ms_since_change = self.laser_4_status.ms_since_last_status_change(
self.evt_time)
if self.verbose:
if self.laser_1_ms_since_change is not None:
self.logger.info("ms since laser 1 status change: %i" %
self.laser_1_ms_since_change)
if self.laser_4_ms_since_change is not None:
self.logger.info("ms since laser 4 status change: %i" %
self.laser_4_ms_since_change)
if self.laser_1_status is not None and self.laser_1_status.status is not None:
self.logger.info("Laser 1 status: %i" %
int(self.laser_1_status.status))
if self.laser_4_status is not None and self.laser_4_status.status is not None:
self.logger.info("Laser 4 status: %i" %
int(self.laser_4_status.status))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
# Increase the event counter, even if this event is to be skipped.
self.nshots += 1
if (evt.get("skip_event")):
return
sifoil = cspad_tbx.env_sifoil(env)
if self.check_beam_status and sifoil is None:
self.nfail += 1
self.logger.warning("event(): no Si-foil thickness, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if (self.sifoil is not None and self.sifoil != sifoil):
self.logger.warning("event(): Si-foil changed mid-run: % 8i -> % 8d" %
(self.sifoil, sifoil))
self.sifoil = sifoil
if self.verbose: self.logger.info("Si-foil thickness: %i" %sifoil)
self.evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
self.timestamp = cspad_tbx.evt_timestamp(self.evt_time) # human readable format
if (self.timestamp is None):
self.nfail += 1
self.logger.warning("event(): no timestamp, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if self.verbose: self.logger.info(self.timestamp)
if self.override_energy is None:
self.wavelength = cspad_tbx.evt_wavelength(evt, self.delta_k)
if self.wavelength is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no wavelength, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.wavelength = 0
else:
self.wavelength = 12398.4187/self.override_energy
if self.verbose: self.logger.info("Wavelength: %.4f" %self.wavelength)
self.pulse_length = cspad_tbx.evt_pulse_length(evt)
if self.pulse_length is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no pulse length, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.pulse_length = 0
if self.verbose: self.logger.info("Pulse length: %s" %self.pulse_length)
self.beam_charge = cspad_tbx.evt_beam_charge(evt)
if self.beam_charge is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no beam charge, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.beam_charge = 0
if self.verbose: self.logger.info("Beam charge: %s" %self.beam_charge)
self.injector_xyz = cspad_tbx.env_injector_xyz(env)
#if self.injector_xyz is not None:
#self.logger.info("injector_z: %i" %self.injector_xyz[2].value)
self.laser_1_status.set_status(cspad_tbx.env_laser_status(env, laser_id=1), self.evt_time)
self.laser_4_status.set_status(cspad_tbx.env_laser_status(env, laser_id=4), self.evt_time)
self.laser_1_ms_since_change = self.laser_1_status.ms_since_last_status_change(self.evt_time)
self.laser_4_ms_since_change = self.laser_4_status.ms_since_last_status_change(self.evt_time)
if self.verbose:
if self.laser_1_ms_since_change is not None:
self.logger.info("ms since laser 1 status change: %i" %self.laser_1_ms_since_change)
if self.laser_4_ms_since_change is not None:
self.logger.info("ms since laser 4 status change: %i" %self.laser_4_ms_since_change)
if self.laser_1_status is not None and self.laser_1_status.status is not None:
self.logger.info("Laser 1 status: %i" %int(self.laser_1_status.status))
if self.laser_4_status is not None and self.laser_4_status.status is not None:
self.logger.info("Laser 4 status: %i" %int(self.laser_4_status.status))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(mod_image_dict, self).event(evt, env)
if (evt.get("skip_event")):
return
if self.cspad_img is None:
return
# This module only applies to detectors for which a distance is
# available.
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self.nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
device = cspad_tbx.address_split(self.address)[2]
self.logger.info("Subprocess %02d: process image #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
# See r17537 of mod_average.py.
if device == 'Cspad':
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'Rayonix':
pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size)
saturated_value = rayonix_tbx.rayonix_saturated_value
elif device == 'marccd':
pixel_size = evt.get("marccd_pixel_size")
saturated_value = evt.get("marccd_saturated_value")
if distance == 0:
distance = evt.get("marccd_distance")
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * self.beam_center[0],
beam_center_y=pixel_size * self.beam_center[1],
data=self.cspad_img.iround(), # XXX ouch!
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=self.timestamp,
wavelength=self.wavelength)
evt.put(d, self.m_out_key)
# Diagnostic message emitted only when all the processing is done.
if (env.subprocess() >= 0):
self.logger.info("Subprocess %02d: accepted #%05d @ %s" %
(env.subprocess(), self.nshots, self.timestamp))
else:
self.logger.info("Accepted #%05d @ %s" %
(self.nshots, self.timestamp))