def _filter_event(self, evt, flux_min, flux_max, peak_ratio_min,
peak_ratio_max):
all_data = evt.get(psana.Camera.FrameV1, self.src)
if all_data is None:
print "No data"
return False, None, None, None, None, None
print cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)),
data = np.array(all_data.data16().astype(np.int32))
if self.dark_pickle is None:
self.dark_pickle = np.zeros(data.shape)
if self.bg_roi is None:
dc_offset = None
dc_offset_mean = 0.0
else:
xmin, xmax, ymin, ymax = self.bg_roi
dc_offset = data[ymin:ymax, xmin:xmax] - self.dark_pickle[
ymin:ymax, xmin:xmax] #dc: direct current
dc_offset_mean = np.mean(dc_offset)
if self.roi is None:
xmin = 0
ymin = 0
ymax, xmax = data.shape
else:
xmin, xmax, ymin, ymax = self.roi
data_signal = data[ymin:ymax, xmin:xmax] - self.dark_pickle[ymin:ymax,
xmin:xmax]
data = data_signal - dc_offset_mean
# make a 1D trace
spectrum = np.sum(data, 0)
flux = np.sum(spectrum)
if self.peak_range is None:
print "Run", evt.run(), "flux:", flux
else:
peak = spectrum[self.peak_range[0] - xmin:self.peak_range[1] -
xmin]
peak_max = np.sum(peak)
print "Run", evt.run(), "peak max:", peak_max, "at", np.argmax(
peak) + xmin, "px", "flux:", flux, "m/f:", peak_max / flux
if flux_min is not None and flux < flux_min:
return False, None, None, None, None, None
if flux_max is not None and flux >= flux_max:
return False, None, None, None, None, None
if peak_ratio_min is not None and peak_max / flux < peak_ratio_min:
return False, None, None, None, None, None
if peak_ratio_max is not None and peak_max / flux >= peak_ratio_max:
return False, None, None, None, None, None
all_data = all_data.data16().astype(np.int32)
return True, data, spectrum, dc_offset, all_data - self.dark_pickle, all_data
def print_version(version):
"""
Given a detector format version, print its vital stats
"""
from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp
print "%14s "%version, "%17s"%_detector_format_version_dict[version]['address'],
if _detector_format_version_dict[version]['start_time'] is None:
print "None ",
else:
print evt_timestamp((_detector_format_version_dict[version]['start_time'], 0)),
if _detector_format_version_dict[version]['start_time'] is None:
print "None"
else:
print evt_timestamp((_detector_format_version_dict[version]['end_time'], 0))
def event(self, evt, env):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
Read the evr codes for this event and save the timestamp if it has an evr code we are
looking for.
@param evt Event data object, a configure object
@param env Environment object
"""
if (evt.get("skip_event")):
return
evr = psana.Detector('evr0')
codes = evr.eventCodes(evt)
timestamp = cspad_tbx.evt_timestamp(
cspad_tbx.evt_time(evt)) # human readable format
logging.info("mod_event_code, rank: %d, timestamp: %s, code_list: %s" %
(self.rank, timestamp, ",".join(["%d" % c
for c in codes])))
# Save this timestamp if it has an event_code w are looking for
for alist, code in zip(self.alist_names, self.event_codes):
if code in codes:
self.timestamps_d[alist].append(timestamp)
def print_version(version):
"""
Given a detector format version, print its vital stats
"""
from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp
print "%14s " % version, "%17s" % _detector_format_version_dict[version][
'address'],
if _detector_format_version_dict[version]['start_time'] is None:
print "None ",
else:
print evt_timestamp(
(_detector_format_version_dict[version]['start_time'], 0)),
if _detector_format_version_dict[version]['start_time'] is None:
print "None"
else:
print evt_timestamp(
(_detector_format_version_dict[version]['end_time'], 0))
def load_pars_from_file(self, path=None) :
""" Use the dxtbx object model to build a GeometryObject hierarchy
@param path Path to the CSPAD CBF file
"""
if path is not None : self.path = path
if self.pbits & 32 : print 'Load file: %s' % self.path
img = dxtbx.load(self.path)
cbf = img._cbf_handle
cbf.find_category("diffrn_source")
cbf.find_column("type")
self.dict_of_comments = {
"TITLE" : "Geometry parameters of CSPAD",
"DATE_TIME" : evt_timestamp(),
"AUTHOR" : getpass.getuser(),
"EXPERIMENT": cbf.get_value(),
"DETECTOR" : "CSPAD",
"CALIB_TYPE": "geometry",
"COMMENT:01": "Table contains the list of geometry parameters for alignment of 2x1 sensors, quads, CSPAD, etc",
"COMMENT:02": " translation and rotation pars of the object are defined w.r.t. parent object Cartesian frame",
"PARAM:01" : "PARENT - name and version of the parent object",
"PARAM:02" : "PARENT_IND - index of the parent object",
"PARAM:03" : "OBJECT - name and version of the object",
"PARAM:04" : "OBJECT_IND - index of the new object",
"PARAM:05" : "X0 - x-coordinate [um] of the object origin in the parent frame",
"PARAM:06" : "Y0 - y-coordinate [um] of the object origin in the parent frame",
"PARAM:07" : "Z0 - z-coordinate [um] of the object origin in the parent frame",
"PARAM:08" : "ROT_Z - object design rotation angle [deg] around Z axis of the parent frame",
"PARAM:09" : "ROT_Y - object design rotation angle [deg] around Y axis of the parent frame",
"PARAM:10" : "ROT_X - object design rotation angle [deg] around X axis of the parent frame",
"PARAM:11" : "TILT_Z - object tilt angle [deg] around Z axis of the parent frame",
"PARAM:12" : "TILT_Y - object tilt angle [deg] around Y axis of the parent frame",
"PARAM:13" : "TILT_X - object tilt angle [deg] around X axis of the parent frame"
}
self.list_of_geos = []
detector = img.get_detector()
hierarchy = detector.hierarchy()
for q, quad in enumerate(hierarchy):
for s, sensor in enumerate(quad):
self.list_of_geos.append(self._load_geo(q,"QUAD:V1",s,"SENS2X1:V1",sensor))
for q, quad in enumerate(hierarchy):
self.list_of_geos.append(self._load_geo(0,"CSPAD:V1",q,"QUAD:V1",quad))
# Add placeholder RAIL and IP vectors, including the XY component of the hierarchy's d0 vector
go = self._load_geo(0,'RAIL',0,'CSPAD:V1',hierarchy)
go.move_geo(0,0,-go.z0+1000000) # Placeholder
self.list_of_geos.append(go)
self.list_of_geos.append(self._null_geo(0,"IP",0,"RAIL"))
self._set_relations()
def get_psana_timestamp(self, index):
"""Get the cctbx.xfel style event timestamp given an index"""
evt = self._get_event(index)
time = evt.get(psana.EventId).time()
# fid = evt.get(psana.EventId).fiducials()
sec = time[0]
nsec = time[1]
return cspad_tbx.evt_timestamp((sec, nsec / 1e6))
def debug_write(self, string, state=None):
ts = cspad_tbx.evt_timestamp() # Now
debug_file_handle = open(self.debug_file_path, 'a')
if string == "":
debug_file_handle.write("\n")
else:
if state is None:
state = " "
debug_file_handle.write(self.debug_str % (ts, state, string))
debug_file_handle.close()
def debug_write(self, string, state = None):
ts = cspad_tbx.evt_timestamp() # Now
debug_file_handle = open(self.debug_file_path, 'a')
if string == "":
debug_file_handle.write("\n")
else:
if state is None:
state = " "
debug_file_handle.write(self.debug_str%(ts, state, string))
debug_file_handle.close()
def _filter_event(self, evt, flux_min, flux_max, peak_ratio_min, peak_ratio_max):
all_data = evt.get(psana.Camera.FrameV1, self.src)
if all_data is None:
print "No data"
return False, None, None, None, None, None
print cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)),
if self.bg_roi is None:
dc_offset = None
dc_offset_mean = 0.0
else:
xmin, xmax, ymin, ymax = self.bg_roi
dc_offset = np.array(all_data.data16().astype(np.int32))[ymin:ymax,xmin:xmax] - self.dark_pickle[ymin:ymax,xmin:xmax] #dc: direct current
dc_offset_mean = np.mean(dc_offset)
data = np.array(all_data.data16().astype(np.int32))
if self.roi is None:
xmin = 0; ymin = 0
ymax, xmax = data.shape
else:
xmin, xmax, ymin, ymax = self.roi
data_signal = data[ymin:ymax,xmin:xmax] - self.dark_pickle[ymin:ymax,xmin:xmax]
data = data_signal - dc_offset_mean
# make a 1D trace
spectrum = np.sum(data, 0)
flux = np.sum(spectrum)
if self.peak_range is None:
print "Run", evt.run(), "flux:", flux
else:
peak = spectrum[self.peak_range[0]-xmin:self.peak_range[1]-xmin]
peak_max = np.sum(peak)
print "Run", evt.run(), "peak max:", peak_max, "at", np.argmax(peak)+xmin, "px", "flux:", flux, "m/f:", peak_max/flux
if flux_min is not None and flux < flux_min: return False, None, None, None, None, None
if flux_max is not None and flux >= flux_max: return False, None, None, None, None, None
if peak_ratio_min is not None and peak_max/flux < peak_ratio_min: return False, None, None, None, None, None
if peak_ratio_max is not None and peak_max/flux >= peak_ratio_max: return False, None, None, None, None, None
all_data = all_data.data16().astype(np.int32)
return True, data, spectrum, dc_offset, all_data - self.dark_pickle, all_data
def run(args):
assert len(args) == 3
d1 = easy_pickle.load(args[0])
d2 = easy_pickle.load(args[1])
image_1 = d1["DATA"]
image_2 = d2["DATA"]
assert image_1.all() == image_2.all()
diff_image = image_1 - image_2
d = cspad_tbx.dpack(
data=diff_image,
timestamp=cspad_tbx.evt_timestamp(),
distance=1,
)
easy_pickle.dump(args[2], d)
def run(args):
assert len(args) == 3
d1 = easy_pickle.load(args[0])
d2 = easy_pickle.load(args[1])
image_1 = d1["DATA"]
image_2 = d2["DATA"]
assert image_1.all() == image_2.all()
diff_image = image_1 - image_2
d = cspad_tbx.dpack(
data=diff_image,
timestamp=cspad_tbx.evt_timestamp(),
distance=1,
)
easy_pickle.dump(args[2], d)
def event(self, evt, env):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
self.nshots += 1
if (evt.get("skip_event")):
return
if (self.timestamps_list is not None):
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning(
"event(): no timestamp, shot skipped. Shot: %s" %
self.nshots)
evt.put(skip_event_flag(), "skip_event")
return
elif (self.negate and t in self.timestamps_list):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.negate and t not in self.timestamps_list):
evt.put(skip_event_flag(), "skip_event")
return
else:
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning(
"event(): no timestamp, shot skipped. Shot: %s" %
self.nshots)
evt.put(skip_event_flag(), "skip_event")
return
if (self.negate and self._tir(t, self.timestamps_interval)):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.negate
and not self._tir(t, self.timestamps_interval)):
evt.put(skip_event_flag(), "skip_event")
return
self.logger.info("event(): event accepted. Shot: %s, TS: %s" %
(self.nshots, cspad_tbx.evt_timestamp(t)))
self.naccepted += 1
def process_event(self, run, timestamp):
"""
Process a single event from a run
@param run psana run object
@param timestamp psana timestamp object
"""
ts = cspad_tbx.evt_timestamp(
(timestamp.seconds(), timestamp.nanoseconds() / 1e6))
if self.params.debug.event_timestamp is not None and self.params.debug.event_timestamp != ts:
return
ts_path = os.path.join(self.params.output.output_dir,
"debug-" + ts + ".txt")
if self.params.debug.use_debug_files:
if not os.path.exists(ts_path):
print("Skipping event %s: no debug file found" % ts)
return
f = open(ts_path, "r")
if len(f.readlines()) > 1:
print("Skipping event %s: processed successfully previously" %
ts)
return
f.close()
print("Accepted", ts)
if self.params.debug.write_debug_files:
f = open(ts_path, "w")
f.write("%s about to process %s\n" % (socket.gethostname(), ts))
f.close()
# load the event. This will cause any modules to be run.
evt = run.event(timestamp)
if self.params.debug.write_debug_files:
f = open(ts_path, "a")
f.write("done\n")
f.close()
id = evt.get(EventId)
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
if (evt.get("skip_event")):
return
self.n_total += 1
ts = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
accept = self.filter.filter_event(evt, self.selected_filter)[0]
if not accept:
print "Skipping event", ts, ": didn't pass filter", self.selected_filter
evt.put(skip_event_flag(), "skip_event")
return
print "Accepting event", ts, ": passed filter", self.selected_filter
self.n_accepted += 1
def event(self, evt, evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
if (evt.get("skip_event")):
return
self.n_total += 1
ts = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
accept = self.filter.filter_event(evt, self.selected_filter)[0]
if not accept:
print "Skipping event", ts, ": didn't pass filter", self.selected_filter
evt.put(skip_event_flag(), "skip_event")
return
print "Accepting event", ts, ": passed filter", self.selected_filter
self.n_accepted += 1
def event(self, evt, env):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
self.nshots += 1
if (evt.get("skip_event")):
return
if (self.timestamps_list is not None):
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning("event(): no timestamp, shot skipped. Shot: %s"%self.nshots)
evt.put(skip_event_flag(), "skip_event")
return
elif (self.negate and t in self.timestamps_list):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.negate and t not in self.timestamps_list):
evt.put(skip_event_flag(), "skip_event")
return
else:
t = cspad_tbx.evt_time(evt)
if (t is None):
self.logger.warning("event(): no timestamp, shot skipped. Shot: %s"%self.nshots)
evt.put(skip_event_flag(), "skip_event")
return
if (self.negate and self._tir(t, self.timestamps_interval)):
evt.put(skip_event_flag(), "skip_event")
return
elif (not self.negate and not self._tir(t, self.timestamps_interval)):
evt.put(skip_event_flag(), "skip_event")
return
self.logger.info("event(): event accepted. Shot: %s, TS: %s"%(self.nshots,cspad_tbx.evt_timestamp(t)))
self.naccepted += 1
def process_event(self, run, timestamp):
"""
Process a single event from a run
@param run psana run object
@param timestamp psana timestamp object
"""
ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6))
if self.params.debug.event_timestamp is not None and self.params.debug.event_timestamp != ts:
return
ts_path = os.path.join(self.params.output.output_dir, "debug-" + ts + ".txt")
if self.params.debug.use_debug_files:
if not os.path.exists(ts_path):
print "Skipping event %s: no debug file found"%ts
return
f = open(ts_path, "r")
if len(f.readlines()) > 1:
print "Skipping event %s: processed successfully previously"%ts
return
f.close()
print "Accepted", ts
if self.params.debug.write_debug_files:
f = open(ts_path, "w")
f.write("%s about to process %s\n"%(socket.gethostname(), ts))
f.close()
# load the event. This will cause any modules to be run.
evt = run.event(timestamp)
if self.params.debug.write_debug_files:
f = open(ts_path, "a")
f.write("done\n")
f.close()
id = evt.get(EventId)
def endjob(self, obj1, obj2=None):
"""The endjob() function writes the mean and standard deviation images
to disk.
@param evt Event object (psana only)
@param env Environment object
"""
if obj2 is None:
env = obj1
else:
evt = obj1
env = obj2
stats = super(mod_average, self).endjob(env)
if stats is None:
return
device = cspad_tbx.address_split(self.address)[2]
if device == 'Andor':
beam_center = (0, 0) # XXX Fiction!
pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
saturated_value = 10000
elif device == 'Cspad' or device == 'Cspad2x2':
beam_center = self.beam_center
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
beam_center = tuple(t // 2 for t in d['mean_img'].focus())
pixel_size = 0.079346
saturated_value = 2**16 - 1
if stats['nmemb'] > 0:
if self.avg_dirname is not None or \
self.avg_basename is not None or \
self._mean_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['mean_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._mean_out is not None:
p = cspad_tbx.dwritef2(d, self._mean_out)
else:
p = cspad_tbx.dwritef(d, self.avg_dirname, self.avg_basename)
self.logger.info("Average written to %s" % p)
if self.stddev_dirname is not None or \
self.stddev_basename is not None or \
self._std_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['std_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._std_out is not None:
p = cspad_tbx.dwritef2(d, self._std_out)
else:
p = cspad_tbx.dwritef(d, self.stddev_dirname, self.stddev_basename)
self.logger.info("Standard deviation written to %s" % p)
if self.max_dirname is not None or \
self.max_basename is not None or \
self._max_out is not None:
d = cspad_tbx.dpack(
active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['max_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(stats['time']),
wavelength=stats['wavelength'])
if self._max_out is not None:
p = cspad_tbx.dwritef2(d, self._max_out)
else:
p = cspad_tbx.dwritef(d, self.max_dirname, self.max_basename)
self.logger.info("Max written to %s" % p)
if stats['nfail'] == 0:
self.logger.info("%d images processed" % stats['nmemb'])
else:
self.logger.warning(
"%d images processed, %d failed" % (stats['nmemb'], stats['nfail']))
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
The event() function does not log shots skipped due to
incompleteness in order to keep the output streams clean.
Instead, the number of skipped shots is reported by endjob().
@param evt Event data object, a configure object
@param env Environment object
"""
if (evt.get("skip_event")):
return
# XXX This hardcodes the address for the front detector!
detz = cspad_tbx.env_detz('CxiDs1-0|Cspad-0', env)
if (detz is None):
self.m_no_detz += 1
sifoil = cspad_tbx.env_sifoil(env)
if (sifoil is None):
self.m_no_sifoil += 1
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
if (timestamp is None):
self.m_no_timestamp += 1
wavelength = cspad_tbx.evt_wavelength(evt)
if (wavelength is None):
self.m_no_wavelength += 1
if (detz is None or
sifoil is None or
timestamp is None or
wavelength is None):
self.m_nfail += 1
return
if (detz != self.m_detz):
if (self.m_detz is None):
self.m_logger.info("%s: initial detz % 8.4f" %
(timestamp, detz))
else:
self.m_logger.info("%s: detz % 8.4f -> % 8.4f" %
(timestamp, self.m_detz, detz))
self.m_detz = detz
if (self.m_detz_max is None or detz > self.m_detz_max):
self.m_detz_max = detz
if (self.m_detz_min is None or detz < self.m_detz_min):
self.m_detz_min = detz
if (sifoil != self.m_sifoil):
if (self.m_sifoil is None):
self.m_logger.info("%s: initial Si-foil % 8d" %
(timestamp, sifoil))
else:
self.m_logger.info("%s: Si-foil % 8d -> % 8d" %
(timestamp, self.m_sifoil, sifoil))
self.m_sifoil = sifoil
if (self.m_sifoil_max is None or sifoil > self.m_sifoil_max):
self.m_sifoil_max = sifoil
if (self.m_sifoil_min is None or sifoil < self.m_sifoil_min):
self.m_sifoil_min = sifoil
# Accumulate the sum and the squared sum of the shifted the
# wavelength. The shift is taken as the first wavelength
# encountered. This may be more accurate than accumulating raw
# values [Chan et al. (1983) Am. Stat. 37, 242-247].
if (self.m_nmemb == 0):
self.m_wavelength_shift = wavelength
self.m_wavelength_sum = (wavelength - self.m_wavelength_shift)
self.m_wavelength_sumsq = (wavelength - self.m_wavelength_shift)**2
self.m_nmemb = 1
else:
self.m_wavelength_sum += (wavelength - self.m_wavelength_shift)
self.m_wavelength_sumsq += (wavelength - self.m_wavelength_shift)**2
self.m_nmemb += 1
def event(self, evt, env):
from dxtbx.format.Registry import Registry
from os.path import exists
from time import sleep
# Nop if there is no image. For experiments configured to have
# exactly one event per calibration cycle, this should never
# happen.
if self._path is None:
evt.put(skip_event_flag(), "skip_event")
return
# Skip this event if the template isn't in the path
if self._template is not None and not True in [t in self._path for t in self._template.split(',')]:
evt.put(skip_event_flag(), "skip_event")
return
if "phi" in self._path:
evt.put(skip_event_flag(), "skip_event")
return
# Wait for the image to appear in the file system, probing for it
# at exponentially increasing delays.
t = 1
t_tot = 0
if not exists(self._path):
self._logger.info("Waiting for path %s"%self._path)
while not exists(self._path):
if t_tot > 1:
self._logger.info("Timeout waiting for path %s"%self._path)
evt.put(skip_event_flag(), "skip_event")
self._logger.info("Image not found: %s"%self._path)
return
sleep(t)
t_tot += t
t *= 2
# Find a matching Format object and instantiate it using the
# given path. If the Format object does not understand the image,
# try determining a new format. XXX Emits "Couldn't create a
# detector model for this image".
if self._fmt is None:
self._fmt = Registry.find(self._path)
if self._fmt is None:
evt.put(skip_event_flag(), "skip_event")
return
img = self._fmt(self._path)
if img is None:
self._fmt = Registry.find(self._path)
if self._fmt is None:
evt.put(skip_event_flag(), "skip_event")
return
img = self._fmt(self._path)
if img is None:
evt.put(skip_event_flag(), "skip_event")
return
self._logger.info(
"Reading %s using %s" % (self._path, self._fmt.__name__))
# Get the raw image data and convert to double precision floating
# point array. XXX Why will img.get_raw_data() not work, like it
# does in print_header?
db = img.get_detectorbase()
db.readHeader()
db.read()
data = db.get_raw_data().as_double()
# Get the pixel size and store it for common_mode.py
detector = img.get_detector()[0]
ps = detector.get_pixel_size()
assert ps[0] == ps[1]
pixel_size = ps[0]
evt.put(ps[0],"marccd_pixel_size")
evt.put(detector.get_trusted_range()[1],"marccd_saturated_value")
evt.put(detector.get_distance(),"marccd_distance")
# If the beam center isn't provided in the config file, get it from the
# image. It will probably be wrong.
if self._beam_x is None or self._beam_y is None:
self._beam_x, self._beam_y = detector.get_beam_centre_px(img.get_beam().get_s0())
self._beam_x = int(round(self._beam_x))
self._beam_y = int(round(self._beam_y))
# Crop the data so that the beam center is in the center of the image
maxy, maxx = data.focus()
minsize = min([self._beam_x,self._beam_y,maxx-self._beam_x,maxy-self._beam_y])
data = data[self._beam_y-minsize:self._beam_y+minsize,self._beam_x-minsize:self._beam_x+minsize]
evt.put((minsize,minsize),"marccd_beam_center")
evt.put(flex.int([0,0,minsize*2,minsize*2]),"marccd_active_areas")
# Store the image in the event.
evt.put(data, self._address)
# Store the .mmcd file name in the event
evt.put(self._mccd_name, "mccd_name")
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
self._logger.info("converted %s to pickle with timestamp %s" %(self._path, timestamp))
# This should not be necessary as the machine is configured with
# one event per calibration cycle.
self._path = None
def process_event(self, run, timestamp):
"""
Process a single event from a run
@param run psana run object
@param timestamp psana timestamp object
"""
ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6))
if ts is None:
print "No timestamp, skipping shot"
return
if len(self.params_cache.debug.event_timestamp) > 0 and ts not in self.params_cache.debug.event_timestamp:
return
if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events:
if ts in self.known_events:
if self.known_events[ts] not in ["stop", "done", "fail"]:
if self.params_cache.debug.skip_bad_events:
print "Skipping event %s: possibly caused an unknown exception previously"%ts
return
elif self.params_cache.debug.skip_processed_events:
print "Skipping event %s: processed successfully previously"%ts
return
else:
if self.params_cache.debug.skip_unprocessed_events:
print "Skipping event %s: not processed previously"%ts
return
self.debug_start(ts)
evt = run.event(timestamp)
if evt.get("skip_event") or "skip_event" in [key.key() for key in evt.keys()]:
print "Skipping event",ts
self.debug_write("psana_skip", "skip")
return
print "Accepted", ts
self.params = copy.deepcopy(self.params_cache)
# the data needs to have already been processed and put into the event by psana
if self.params.format.file_format == 'cbf':
# get numpy array, 32x185x388
data = cspad_cbf_tbx.get_psana_corrected_data(self.psana_det, evt, use_default=False, dark=True,
common_mode=self.common_mode,
apply_gain_mask=self.params.format.cbf.gain_mask_value is not None,
gain_mask_value=self.params.format.cbf.gain_mask_value,
per_pixel_gain=False)
if data is None:
print "No data"
self.debug_write("no_data", "skip")
return
if self.params.format.cbf.override_distance is None:
distance = cspad_tbx.env_distance(self.params.input.address, run.env(), self.params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
self.debug_write("no_distance", "skip")
return
else:
distance = self.params.format.cbf.override_distance
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
self.debug_write("no_wavelength", "skip")
return
else:
wavelength = 12398.4187/self.params.format.cbf.override_energy
if self.params.format.file_format == 'pickle':
image_dict = evt.get(self.params.format.pickle.out_key)
data = image_dict['DATA']
timestamp = t = ts
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]
print "Processing shot", s
if self.params.format.file_format == 'cbf':
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address)
if self.params.input.reference_geometry is not None:
from dxtbx.model import Detector
# copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead.
cspad_img._detector_instance = Detector.from_dict(self.reference_detector.to_dict())
cspad_img.sync_detector_to_cbf()
elif self.params.format.file_format == 'pickle':
from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory
cspad_img = FormatPYunspecifiedStillInMemory(image_dict)
cspad_img.timestamp = s
if self.params.dispatch.dump_all:
self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s))
self.cache_ranges(cspad_img, self.params)
imgset = MemImageSet([cspad_img])
if self.params.dispatch.estimate_gain_only:
from dials.command_line.estimate_gain import estimate_gain
estimate_gain(imgset)
return
if not self.params.dispatch.find_spots:
self.debug_write("data_loaded", "done")
return
datablock = DataBlockFactory.from_imageset(imgset)[0]
# before calling DIALS for processing, set output paths according to the templates
if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template:
self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.indexed_filename_template%("idx-" + s))
if "%s" in self.refined_experiments_filename_template:
self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.refined_experiments_filename_template%("idx-" + s))
if "%s" in self.integrated_filename_template:
self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.integrated_filename_template%("idx-" + s))
if "%s" in self.reindexedstrong_filename_template:
self.params.output.reindexedstrong_filename = os.path.join(self.params.output.output_dir, self.reindexedstrong_filename_template%("idx-" + s))
# Load a dials mask from the trusted range and psana mask
from dials.util.masking import MaskGenerator
generator = MaskGenerator(self.params.border_mask)
mask = generator.generate(imgset)
if self.params.format.file_format == "cbf":
mask = tuple([a&b for a, b in zip(mask,self.dials_mask)])
if self.spotfinder_mask is None:
self.params.spotfinder.lookup.mask = mask
else:
self.params.spotfinder.lookup.mask = tuple([a&b for a, b in zip(mask,self.spotfinder_mask)])
if self.integration_mask is None:
self.params.integration.lookup.mask = mask
else:
self.params.integration.lookup.mask = tuple([a&b for a, b in zip(mask,self.integration_mask)])
self.debug_write("spotfind_start")
try:
observed = self.find_spots(datablock)
except Exception, e:
import traceback; traceback.print_exc()
print str(e), "event", timestamp
self.debug_write("spotfinding_exception", "fail")
return
def event(self, evt, evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(
cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata += 1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(), "skip_event")
if timestamp is None:
evt.put(skip_event_flag(), "skip_event")
self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots += 1
# get data as array and split into two half to find each peak
if one_D:
data = np.array(data.hproj().astype(np.float64))
if 'dark' in locals():
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0] // 2]
spectrum2 = data[data.shape[0] // 2:]
else:
data = np.array(data.data16().astype(np.float64))
if 'dark' in locals():
data = data - self.dark
data_split1 = data[:, :data.shape[1] // 2]
data_split2 = data[:, data.shape[1] // 2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data, 0) / data.shape[0]
spectrum1 = np.sum(data_split1, 0) / data_split1.shape[0]
spectrum2 = np.sum(data_split2, 0) / data_split2.shape[0]
peak_one = np.max(spectrum1)
peak_two = np.max(spectrum2)
peak_one_position = np.argmax(spectrum1)
peak_two_position = np.argmax(spectrum2) + len(spectrum2)
# define the limits of the regions between the two peaks
peak_one_lower_limit = self.peak_one_position_min - self.peak_one_width
peak_one_upper_limit = self.peak_one_position_max + self.peak_one_width
peak_two_lower_limit = self.peak_two_position_min - self.peak_two_width
peak_two_upper_limit = self.peak_two_position_max + self.peak_two_width
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in range(peak_one_lower_limit, peak_one_upper_limit):
weighted_peak_one_positions.append(spectrum[i] * i)
weighted_sum_peak_one = sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one // sum(
spectrum[peak_one_lower_limit:peak_one_upper_limit])
weighted_peak_two_positions = []
for i in range(peak_two_lower_limit, peak_two_upper_limit):
weighted_peak_two_positions.append(spectrum[i] * i)
weighted_sum_peak_two = sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two // sum(
spectrum[peak_two_lower_limit:peak_two_upper_limit])
# normalized integrated peaks
int_peak_one = np.sum(
spectrum[peak_one_lower_limit:self.peak_one_position_max]
) / len(spectrum[peak_one_lower_limit:self.peak_one_position_max])
int_peak_two = np.sum(
spectrum[peak_two_lower_limit:self.peak_two_position_max]
) / len(spectrum[peak_two_lower_limit:self.peak_two_position_max])
# normalized integrated regions between the peaks
int_left_region = np.sum(spectrum[0:peak_one_lower_limit]) / len(
spectrum[0:peak_one_lower_limit])
int_middle_region = np.sum(
spectrum[peak_one_upper_limit:peak_two_lower_limit]) / len(
spectrum[peak_one_upper_limit:peak_two_lower_limit])
int_right_region = np.sum(spectrum[peak_two_upper_limit:]) / len(
spectrum[:peak_two_upper_limit])
# now to do the filtering
if peak_one / peak_two < self.peak_ratio or peak_one / peak_two > 1 / self.peak_ratio:
print "event(): too low"
evt.put(skip_event_flag(), "skip_event")
return
if weighted_peak_two_center_position < self.peak_two_position_min or weighted_peak_two_center_position > self.peak_two_position_max:
print "event(): out of range high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if weighted_peak_one_center_position < self.peak_one_position_min or weighted_peak_one_center_position > self.peak_one_position_max:
print "event(): out of range low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_left_region / int_peak_one >
self.normalized_peak_to_noise_ratio):
print "event(): noisy left of low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_middle_region / int_peak_one >
self.normalized_peak_to_noise_ratio):
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_middle_region / int_peak_two >
self.normalized_peak_to_noise_ratio):
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_right_region / int_peak_two >
self.normalized_peak_to_noise_ratio):
print "event(): noisy right of high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
#iron edge at 738 pixels on FFE detetor
if one_D and (spectrum[self.iron_edge_position] >=
spectrum[weighted_peak_one_center_position]):
print "event(): peak at iron edge"
evt.put(skip_event_flag(), "skip_event")
return
if one_D and (spectrum[self.iron_edge_position] >=
spectrum[weighted_peak_two_center_position]):
print "event(): peak at iron edge"
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print "%d Two Color shots" % self.ntwo_color
def run(args):
command_line = (option_parser()
.option("-o", "--output_filename",
action="store",
type="string",
help="Filename for the output pickle file",
default="gain_map.pickle")
.option("-f", "--detector_format_version",
action="store",
type="string",
help="Detector format version to use for generating active areas and laying out tiles",
default=None)
.option("-m", "--optical_metrology_path",
action="store",
type="string",
help="Path to slac optical metrology file. If not set, use Run 4 metrology",
default=None)
.option("-d", "--distance",
action="store",
type="int",
help="Detector distance put into the gain pickle file. Not needed for processing.",
default="0")
.option("-w", "--wavelength",
action="store",
type="float",
help="Incident beam wavelength put into the gain pickle file. Not needed for processing.",
default="0")
).process(args=args)
output_filename = command_line.options.output_filename
detector_format_version = command_line.options.detector_format_version
if detector_format_version is None or 'XPP' not in detector_format_version:
beam_center_x = None
beam_center_y = None
else:
beam_center_x = 1765 // 2 * 0.11
beam_center_y = 1765 // 2 * 0.11
address, timestamp = address_and_timestamp_from_detector_format_version(detector_format_version)
# if no detector format version is provided, make sure to write no address to the image pickle
# but CsPadDetector (called later), needs an address, so give it a fake one
save_address = address is not None
if not save_address:
address = "CxiDs1-0|Cspad-0" # time stamp will still be None
timestamp = evt_timestamp((timestamp,0))
args = command_line.args
assert len(args) == 1
if args[0].endswith('.npy'):
data = numpy.load(args[0])
det, active_areas = convert_2x2(data, detector_format_version, address)
elif args[0].endswith('.txt') or args[0].endswith('.gain'):
raw_data = numpy.loadtxt(args[0])
assert raw_data.shape in [(5920, 388), (11840, 194)]
det, active_areas = convert_detector(raw_data, detector_format_version, address, command_line.options.optical_metrology_path)
img_diff = det
img_sel = (img_diff > 0).as_1d()
gain_map = flex.double(img_diff.accessor(), 0)
gain_map.as_1d().set_selected(img_sel.iselection(), 1/img_diff.as_1d().select(img_sel))
gain_map /= flex.mean(gain_map.as_1d().select(img_sel))
if not save_address:
address = None
d = cspad_tbx.dpack(data=gain_map, address=address, active_areas=active_areas, timestamp=timestamp,
distance=command_line.options.distance,wavelength=command_line.options.wavelength,
beam_center_x = beam_center_x, beam_center_y = beam_center_y)
easy_pickle.dump(output_filename, d)
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata +=1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(),"skip_event")
if timestamp is None:
evt.put(skip_event_flag(),"skip_event")
#self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots +=1
# get data as array and split into two half to find each peak
if one_D:
# filtering out outlier spikes in FEE data
data = np.array(data.hproj().astype(np.float64))
for i in range(len(data)):
if data[i]>1000000000:
data[i]=data[i]-(2**32)
if self.dark is not None:
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0]//2]
spectrum2 = data[data.shape[0]//2:]
else:
data = np.array(data.data16().astype(np.int32))
if self.dark is not None:
data = data - self.dark
data = np.double(data)
data_split1 = data[:,:data.shape[1]//2]
data_split2 = data[:,data.shape[1]//2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data,0)/data.shape[0]
spectrum1 = np.sum(data_split1,0)/data_split1.shape[0]
spectrum2 = np.sum(data_split2,0)/data_split2.shape[0]
if not one_D:
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in range(self.peak_one_range_min,self.peak_one_range_max):
weighted_peak_one_positions.append(spectrum[i]*i)
weighted_sum_peak_one = np.sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one/np.sum(spectrum[self.peak_one_range_min:self.peak_one_range_max])
weighted_peak_two_positions = []
for i in range(self.peak_two_range_min,self.peak_two_range_max):
weighted_peak_two_positions.append(spectrum[i]*i)
weighted_sum_peak_two = np.sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two/np.sum(spectrum[self.peak_two_range_min:self.peak_two_range_max])
# normalized integrated regions between the peaks
#int_left_region = np.sum(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_left_region = np.sum(spectrum[:weighted_peak_two_center_position/2])
#int_left_region_norm = np.sum(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_left_region_norm = np.sum(spectrum[:weighted_peak_two_center_position/2])/len(spectrum[:weighted_peak_two_center_position/2])
int_right_region = np.sum(spectrum[self.peak_two_range_max:])
int_right_region_norm = np.sum(spectrum[self.peak_two_range_max:])/len(spectrum[self.peak_two_range_max:])
# normalized integrated peaks
int_peak_one = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_one_norm = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])/len(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_two = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_peak_two_norm = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
if not one_D:
if int_peak_one_norm/int_peak_two_norm > self.peak_ratio:
print("event(): inflection peak too high")
evt.put(skip_event_flag(), "skip_event")
return
if int_left_region_norm > self.normalized_peak_to_noise_ratio*int_peak_two_norm:
print("event(): noisy left of low energy peak")
evt.put(skip_event_flag(), "skip_event")
return
if int_right_region_norm > self.normalized_peak_to_noise_ratio*int_peak_two_norm:
print("event(): noisy right of high energy peak")
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print("%d Remote shot" %self.ntwo_color)
print("%s Remote timestamp" %timestamp)
if annulus:
if params.resolution is None:
print "Generating annular gain mask using %s metrology between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms" % \
(str(params.detector_format_version), params.annulus_inner, params.annulus_outer, params.distance, params.wavelength)
else:
print "Generating annular gain mask using %s metrology between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms. Also, pixels higher than %f angstroms will be set to low gain." % \
(str(params.detector_format_version), params.annulus_inner, params.annulus_outer, params.distance, params.wavelength, params.resolution)
elif params.resolution is not None:
print "Generating circular gain mask using %s metrology at %s angstroms, assuming a distance %s mm and wavelength %s angstroms" % \
(str(params.detector_format_version), params.resolution, params.distance, params.wavelength)
from xfel.cxi.cspad_ana.cspad_tbx import dpack, evt_timestamp, cbcaa, pixel_size, CsPadDetector
from iotbx.detectors.cspad_detector_formats import address_and_timestamp_from_detector_format_version
from convert_gain_map import fake_env, fake_config, fake_evt, fake_cspad_ElementV2
address, timestamp = address_and_timestamp_from_detector_format_version(params.detector_format_version)
timestamp = evt_timestamp((timestamp,0))
raw_data = numpy.zeros((11840,194))
if params.detector_format_version is not None and "XPP" in params.detector_format_version:
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
active_areas = xpp_active_areas[params.detector_format_version]['active_areas']
data = flex.int(flex.grid((1765,1765)))
beam_center = (1765 // 2, 1765 // 2)
else:
if params.optical_metrology_path is None:
calib_dir = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = calib2sections(calib_dir)
else:
sections = calib2sections(params.optical_metrology_path)
asic_start = 0
data3d = []
def run(argv=None):
import libtbx.option_parser
if (argv is None):
argv = sys.argv
command_line = (libtbx.option_parser.option_parser(
usage="%s [-v] [-p poly_mask] [-c circle_mask] [-a avg_max] [-s stddev_max] [-m maxproj_min] [-x mask_pix_val] [-o output] avg_path stddev_path max_path" % libtbx.env.dispatcher_name)
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--poly_mask", "-p",
type="string",
default=None,
dest="poly_mask",
help="Polygon to mask out. Comma-seperated string of xy pairs.")
.option(None, "--circle_mask", "-c",
type="string",
default=None,
dest="circle_mask",
help="Circle to mask out. Comma-seperated string of x, y, and radius.")
.option(None, "--avg_max", "-a",
type="float",
default=2000.0,
dest="avg_max",
help="Maximum ADU that pixels in the average image are allowed to have before masked out")
.option(None, "--stddev_max", "-s",
type="float",
default=10.0,
dest="stddev_max",
help="Maximum ADU that pixels in the standard deviation image are allowed to have before masked out")
.option(None, "--maxproj_min", "-m",
type="float",
default=300.0,
dest="maxproj_min",
help="Minimum ADU that pixels in the maximum projection image are allowed to have before masked out")
.option(None, "--mask_pix_val", "-x",
type="int",
default=-2,
dest="mask_pix_val",
help="Value for masked out pixels")
.option(None, "--detector_format_version", "-d",
type="string",
default=None,
dest="detector_format_version",
help="detector format version string")
.option(None, "--output", "-o",
type="string",
default="mask_.pickle",
dest="destpath",
help="output file path, should be *.pickle")
).process(args=argv[1:])
# Must have exactly three remaining arguments.
paths = command_line.args
if (len(paths) != 3):
command_line.parser.print_usage(file=sys.stderr)
return
if command_line.options.detector_format_version is None:
address = timestamp = None
else:
from xfel.cxi.cspad_ana.cspad_tbx import evt_timestamp
from xfel.detector_formats import address_and_timestamp_from_detector_format_version
address, timestamp = address_and_timestamp_from_detector_format_version(command_line.options.detector_format_version)
timestamp = evt_timestamp((timestamp,0))
poly_mask = None
if not command_line.options.poly_mask == None:
poly_mask = []
poly_mask_tmp = command_line.options.poly_mask.split(",")
if len(poly_mask_tmp) % 2 != 0:
command_line.parser.print_usage(file=sys.stderr)
return
odd = True
for item in poly_mask_tmp:
try:
if odd:
poly_mask.append(int(item))
else:
poly_mask[-1] = (poly_mask[-1],int(item))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
odd = not odd
circle_mask = None
if command_line.options.circle_mask is not None:
circle_mask_tmp = command_line.options.circle_mask.split(",")
if len(circle_mask_tmp) != 3:
command_line.parser.print_usage(file=sys.stderr)
return
try:
circle_mask = (int(circle_mask_tmp[0]),int(circle_mask_tmp[1]),int(circle_mask_tmp[2]))
except ValueError:
command_line.parser.print_usage(file=sys.stderr)
return
avg_path = paths[0]
stddev_path = paths[1]
max_path = paths[2]
# load the three images
format_class = Registry.find(avg_path)
avg_f = format_class(avg_path)
avg_i = avg_f.get_detectorbase()
avg_d = avg_i.get_raw_data()
stddev_f = format_class(stddev_path)
stddev_i = stddev_f.get_detectorbase()
stddev_d = stddev_i.get_raw_data()
max_f = format_class(max_path)
max_i = max_f.get_detectorbase()
max_d = max_i.get_raw_data()
# first find all the pixels in the average that are less than zero or greater
# than a cutoff and set them to the masking value
avg_d.set_selected((avg_d <= 0) | (avg_d > command_line.options.avg_max), command_line.options.mask_pix_val)
# set all the rest of the pixels to zero. They will be accepted
avg_d.set_selected(avg_d != command_line.options.mask_pix_val, 0)
# mask out the overly noisy or flat pixels
avg_d.set_selected(stddev_d <= 0, command_line.options.mask_pix_val)
avg_d.set_selected(stddev_d >= command_line.options.stddev_max, command_line.options.mask_pix_val)
# these are the non-bonded pixels
avg_d.set_selected(max_d < command_line.options.maxproj_min, command_line.options.mask_pix_val)
# calculate the beam center
panel = avg_f.get_detector()[0]
bcx, bcy = panel.get_beam_centre(avg_f.get_beam().get_s0())
if poly_mask is not None or circle_mask is not None:
minx = miny = 0
maxx = avg_d.focus()[0]
maxy = avg_d.focus()[1]
if poly_mask is not None:
minx = min([x[0] for x in poly_mask])
miny = min([y[1] for y in poly_mask])
maxx = max([x[0] for x in poly_mask])
maxy = max([y[1] for y in poly_mask])
if circle_mask is not None:
circle_x, circle_y, radius = circle_mask
if circle_x - radius < minx: minx = circle_x - radius
if circle_y - radius < miny: miny = circle_y - radius
if circle_x + radius > maxx: maxx = circle_x + radius
if circle_y + radius > maxy: maxy = circle_y + radius
sel = avg_d == command_line.options.mask_pix_val
for j in xrange(miny, maxy):
for i in xrange(minx, maxx):
idx = j * avg_d.focus()[0] + i
if not sel[idx]:
if poly_mask is not None and point_in_polygon((i,j),poly_mask):
sel[idx] = True
elif circle_mask is not None and point_inside_circle(i,j,circle_x,circle_y,radius):
sel[idx] = True
avg_d.set_selected(sel,command_line.options.mask_pix_val)
# have to re-layout the data to match how it was stored originally
shifted_int_data_old = avg_d
shifted_int_data_new = shifted_int_data_old.__class__(
flex.grid(shifted_int_data_old.focus()))
shifted_int_data_new += command_line.options.mask_pix_val
phil = avg_i.horizons_phil_cache
manager = avg_i.get_tile_manager(phil)
for i,shift in enumerate(manager.effective_translations()):
shift_slow = shift[0]
shift_fast = shift[1]
ur_slow = phil.distl.detector_tiling[4 * i + 0] + shift_slow
ur_fast = phil.distl.detector_tiling[4 * i + 1] + shift_fast
ll_slow = phil.distl.detector_tiling[4 * i + 2] + shift_slow
ll_fast = phil.distl.detector_tiling[4 * i + 3] + shift_fast
#print "Shifting tile at (%d, %d) by (%d, %d)" % (ur_slow-shift_slow, ur_fast-shift_fast, -shift_slow, -shift_fast)
shifted_int_data_new.matrix_paste_block_in_place(
block = shifted_int_data_old.matrix_copy_block(
i_row=ur_slow,i_column=ur_fast,
n_rows=ll_slow-ur_slow, n_columns=ll_fast-ur_fast),
i_row = ur_slow - shift_slow,
i_column = ur_fast - shift_fast
)
d = dpack(
active_areas=avg_i.parameters['ACTIVE_AREAS'],
address=address,
beam_center_x=bcx,
beam_center_y=bcy,
data=shifted_int_data_new,
distance=avg_i.distance,
timestamp=timestamp,
wavelength=avg_i.wavelength,
xtal_target=None,
pixel_size=avg_i.pixel_size,
saturated_value=avg_i.saturation)
dwritef2(d, command_line.options.destpath)
#the minimum number of pixels to mask out cooresponding to the interstitial regions for the CS-PAD
min_count = 818265 # (1765 * 1765) - (194 * 185 * 64)
masked_out = len(avg_d.as_1d().select((avg_d == command_line.options.mask_pix_val).as_1d()))
assert masked_out >= min_count
print "Masked out %d pixels out of %d (%.2f%%)"% \
(masked_out-min_count,len(avg_d)-min_count,(masked_out-min_count)*100/(len(avg_d)-min_count))
def event(self, evt, evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(
cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata += 1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(), "skip_event")
if timestamp is None:
evt.put(skip_event_flag(), "skip_event")
#self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots += 1
# get data as array and split into two half to find each peak
if one_D:
# filtering out outlier spikes in FEE data
data = np.array(data.hproj().astype(np.float64))
for i in range(len(data)):
if data[i] > 1000000000:
data[i] = data[i] - (2**32)
if self.dark is not None:
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0] // 2]
spectrum2 = data[data.shape[0] // 2:]
else:
data = np.array(data.data16().astype(np.int32))
if self.dark is not None:
data = data - self.dark
data_split1 = data[:, :data.shape[1] // 2]
data_split2 = data[:, data.shape[1] // 2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data, 0) / data.shape[0]
spectrum1 = np.sum(data_split1, 0) / data_split1.shape[0]
spectrum2 = np.sum(data_split2, 0) / data_split2.shape[0]
if not one_D:
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in range(self.peak_one_range_min,
self.peak_one_range_max):
weighted_peak_one_positions.append(spectrum[i] * i)
weighted_sum_peak_one = sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one // sum(
spectrum[self.peak_one_range_min:self.peak_one_range_max])
weighted_peak_two_positions = []
for i in range(self.peak_two_range_min,
self.peak_two_range_max):
weighted_peak_two_positions.append(spectrum[i] * i)
weighted_sum_peak_two = sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two // sum(
spectrum[self.peak_two_range_min:self.peak_two_range_max])
# normalized integrated regions between the peaks
int_left_region_norm = np.sum(spectrum[0:(
weighted_peak_one_center_position -
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)]) / len(spectrum[0:(
weighted_peak_one_center_position -
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)])
int_middle_region_norm = np.sum(
spectrum[(weighted_peak_one_center_position +
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2):
(weighted_peak_two_center_position -
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)]
) / len(spectrum[(weighted_peak_one_center_position +
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2):
(weighted_peak_two_center_position -
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)])
int_right_region_norm = np.sum(spectrum[(
weighted_peak_two_center_position +
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2):]) / len(spectrum[(
weighted_peak_two_center_position +
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2):])
# normalized integrated peaks
int_peak_one_norm = np.sum(
spectrum[(weighted_peak_one_center_position -
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2):
(weighted_peak_one_center_position +
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)]
) / len(spectrum[(weighted_peak_one_center_position -
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2):
(weighted_peak_one_center_position +
len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)])
int_peak_two_norm = np.sum(
spectrum[(weighted_peak_two_center_position -
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2):
(weighted_peak_two_center_position +
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)]
) / len(spectrum[(weighted_peak_two_center_position -
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2):
(weighted_peak_two_center_position +
len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)])
else:
# convert eV range of iron edge (7112 eV) to pixels:
metal_edge_max = (self.forbidden_range_eV / 2.) / 0.059 + 738
metal_edge_min = (-self.forbidden_range_eV / 2.) / 0.059 + 738
int_metal_region = np.sum(
spectrum[metal_edge_min:metal_edge_max])
#peak one region integrate:
int_peak_one = np.sum(spectrum[0:metal_edge_min])
int_peak_two = np.sum(spectrum[metal_edge_max:])
# now to do the filtering
if not one_D:
if min(int_peak_one_norm, int_peak_two_norm) / max(
int_peak_one_norm,
int_peak_two_norm) < self.peak_ratio:
print("event(): too low")
evt.put(skip_event_flag(), "skip_event")
return
if (np.argmax(spectrum2) + len(spectrum2)) > (
weighted_peak_two_center_position +
(len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) /
2)) or (np.argmax(spectrum2) + len(spectrum2)) < (
weighted_peak_two_center_position -
(len(spectrum[self.peak_two_range_min:self.
peak_two_range_max]) / 2)):
print("event(): out of range high energy peak")
evt.put(skip_event_flag(), "skip_event")
return
if np.argmax(spectrum1) > (
weighted_peak_one_center_position +
(len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) /
2)) or np.argmax(spectrum1) < (
weighted_peak_one_center_position -
(len(spectrum[self.peak_one_range_min:self.
peak_one_range_max]) / 2)):
print("event(): out of range low energy peak")
evt.put(skip_event_flag(), "skip_event")
return
if int_left_region_norm / int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print("event(): noisy left of low energy peak")
evt.put(skip_event_flag(), "skip_event")
return
if int_middle_region_norm / int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print("event(): noisy middle")
evt.put(skip_event_flag(), "skip_event")
return
if int_middle_region_norm / int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print("event(): noisy middle")
evt.put(skip_event_flag(), "skip_event")
return
if int_right_region_norm / int_peak_two_norm > self.normalized_peak_to_noise_ratio:
print("event(): noisy right of high energy peak")
evt.put(skip_event_flag(), "skip_event")
return
else:
# filter for cxih8015
#iron edge at 738 pixels on FFE detetor
if int_metal_region >= 0.10 * int_peak_two:
print("event(): high intensity at metal edge")
evt.put(skip_event_flag(), "skip_event")
return
if int_metal_region >= 0.10 * int_peak_one:
print("event(): high intensity at metal edge")
evt.put(skip_event_flag(), "skip_event")
return
if min(int_peak_one, int_peak_two) / max(
int_peak_one, int_peak_two) < self.peak_ratio:
print("event(): peak ratio too low")
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print("%d Two Color shots" % self.ntwo_color)
det = PyDetector(psana.Source('rayonix'),ds.env())
images=[]
for ievt,evt in enumerate(ds.events()):
if det.raw(evt) is not None:
image = det.raw(evt)
image = flex.int(image.astype(np.intc))
lambda_wavelength = 12398./(ds.env().epicsStore().value("SIOC:SYS0:ML00:AO627"))
pixelSize = (170./3840.)*2. #standard 2x2 binning
saturation=65535 #maybe thus the trustable value should be lower?
#eventTime=evt.get(psana.EventId).time()[0]+evt.get(psana.EventId).time()[1]*1e-9
# Determine timestamp using cspad_tbx
evt_time = cspad_tbx.evt_time(evt)
evt_timestamp = cspad_tbx.evt_timestamp(evt_time)
#this assumes that we will get the correct numbers for one position
#then we will use the robot position to get relative numbers assuming
#the beam centr stays about the same
robX=ds.env().epicsStore().value("robot_x")
robY=ds.env().epicsStore().value("robot_y")
robZ=ds.env().epicsStore().value("robot_z")
#beamX=602.015+(ds.env().epicsStore().value("robot_x"))
#beamY=51.682+(ds.env().epicsStore().value("robot_y"))
#beamDist=-563.083+60.+ (ds.env().epicsStore().value("robot_z"))
beamX = 87.96
beamY = 85.57
beamDist = 150
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(
usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name)
.option(None, "--as_pickle", "-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files")
.option(None, "--config", "-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file")
.option(None, "--experiment", "-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)")
.option(None, "--run", "-r",
type="int",
default=None,
dest="run",
help="run number")
.option(None, "--address", "-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)")
.option(None, "--detz_offset", "-d",
type="float",
default=None,
dest="detz_offset",
help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)")
.option(None, "--outputdir", "-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files")
.option(None, "--averagebase", "-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help="Path to output average image without extension. String substitution allowed")
.option(None, "--stddevbase", "-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help="Path to output standard deviation image without extension. String substitution allowed")
.option(None, "--maxbase", "-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help="Path to output maximum projection image without extension. String substitution allowed")
.option(None, "--numevents", "-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all")
.option(None, "--skipevents", "-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--pickle-optical-metrology", "-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help="If writing pickle files, use the optical metrology in the experiment's calib directory")
.option(None, "--bin_size", "-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size")
.option(None, "--override_beam_x", "-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate")
.option(None, "--override_beam_y", "-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate")
.option(None, "--calib_dir", "-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory")
.option(None, "--xtc_dir", "-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory")
.option(None, "--use_ffb", "-f",
action="store_true",
default=False,
dest="use_ffb",
help="Use the fast feedback filesystem at LCLS. Only for the active experiment!")
).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxint
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s"%command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)'%address)
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
nevent = np.array([0.])
for run in ds.runs():
runnumber = run.run()
# list of all events
if command_line.options.skipevents > 0:
print "Skipping first %d events"%command_line.options.skipevents
times = run.times()[command_line.options.skipevents:]
nevents = min(len(times),maxevents)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0]
for i in xrange(len(mytimes)):
if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes)
evt = run.event(mytimes[i])
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address:
data = evt.get(Camera.FrameV1,src)
if data is None:
print "No data"
continue
data=data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations
if data is None:
print "No data"
continue
d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset)
if d is None:
print "No distance, skipping shot"
continue
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print "No wavelength, skipping shot"
continue
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print "No timestamp, skipping shot"
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1]/1000)
else:
timestamp = np.array([t[0] + (t[1]/1000)])
if 'sum' in locals():
sum+=data
else:
sum=np.array(data, copy=True)
if 'sumsq' in locals():
sumsq+=data*data
else:
sumsq=data*data
if 'maximum' in locals():
maximum=np.maximum(maximum,data)
else:
maximum=np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print "Synchronizing rank", rank
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent,totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum,sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq,sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum,maxall, op=MPI.MAX)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength,waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance,distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp,timeall)
if rank==0:
if size > 1:
print "Synchronized"
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env())]
dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths]
if 'Rayonix' in command_line.options.address:
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size)
beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y]
detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size)
active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from xfel.detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.calib_dir is not None:
metro_path = command_line.options.calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
maxall = flex.double(maxall.astype(np.float64))
else:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
maxall = flex.double(maxall.astype(np.float64))
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d"%run.run())
for data, path in zip([mean, stddev, maxall], dest_paths):
print "Saving", path
cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except IOError:
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
if abs(beam_x - beam_y) <= 0.1 or self.params.image_conversion.square_mode == "None":
img_type = 'converted'
else:
img_type = 'unconverted'
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec,msec))
if self.params.image_conversion.beamstop != 0 or\
self.params.image_conversion.beam_center.x != 0 or\
self.params.image_conversion.beam_center.y != 0 or\
self.params.image_conversion.rename_pickle_prefix != 'Auto' or\
self.params.image_conversion.rename_pickle_prefix != None:
img_type = 'unconverted'
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp
)
#print "data: ", type(raw_data)
#print "pixel size: ", type(pixel_size)
#print 'wavelength: ', type(wavelength)
#print "beamX: ", type(beam_x)
#print "saturation: ", type(overload)
#print "timestamp: ", type(timestamp)
#for i in dir(raw_data): print i
#exit()
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
img_type = 'unconverted'
if osc_start != osc_range:
data['OSC_START'] = osc_start
data['OSC_RANGE'] = osc_range
data['TIME'] = scan.get_exposure_times()[0]
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
try:
from dials.algorithms.image.threshold import KabschDebug
raw_data = [raw_data]
gain_value = 1
kernel_size=(10,10)
gain_map = [flex.double(raw_data[i].accessor(), gain_value)
for i in range(len(loaded_img.get_detector()))]
mask = loaded_img.get_mask()
min_local = 0
# dummy values, shouldn't affect results
nsigma_b = 6
nsigma_s = 3
global_threshold = 0
kabsch_debug_list = []
for i_panel in range(len(loaded_img.get_detector())):
kabsch_debug_list.append(
KabschDebug(
raw_data[i_panel].as_double(), mask[i_panel], gain_map[i_panel],
kernel_size, nsigma_b, nsigma_s, global_threshold, min_local))
dispersion = flex.double()
for kabsch in kabsch_debug_list:
dispersion.extend(kabsch.coefficient_of_variation().as_1d())
sorted_dispersion = flex.sorted(dispersion)
from libtbx.math_utils import nearest_integer as nint
q1 = sorted_dispersion[nint(len(sorted_dispersion)/4)]
q2 = sorted_dispersion[nint(len(sorted_dispersion)/2)]
q3 = sorted_dispersion[nint(len(sorted_dispersion)*3/4)]
iqr = q3-q1
inlier_sel = (sorted_dispersion > (q1 - 1.5*iqr)) & (sorted_dispersion < (q3 + 1.5*iqr))
sorted_dispersion = sorted_dispersion.select(inlier_sel)
self.gain = sorted_dispersion[nint(len(sorted_dispersion)/2)]
except IndexError:
self.gain = 1.0
else:
self.gain = 1.0
else:
data = None
return data, img_type
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata +=1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(),"skip_event")
if timestamp is None:
evt.put(skip_event_flag(),"skip_event")
self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots +=1
# get data as array and split into two half to find each peak
if one_D:
data = np.array(data.hproj().astype(np.float64))
if 'dark' in locals():
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0]//2]
spectrum2 = data[data.shape[0]//2:]
else:
data = np.array(data.data16().astype(np.float64))
if 'dark' in locals():
data = data - self.dark
data_split1 = data[:,:data.shape[1]//2]
data_split2 = data[:,data.shape[1]//2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data,0)/data.shape[0]
spectrum1 = np.sum(data_split1,0)/data_split1.shape[0]
spectrum2 = np.sum(data_split2,0)/data_split2.shape[0]
peak_one = np.max(spectrum1)
peak_two = np.max(spectrum2)
peak_one_position = np.argmax(spectrum1)
peak_two_position = np.argmax(spectrum2) + len(spectrum2)
# define the limits of the regions between the two peaks
peak_one_lower_limit = self.peak_one_position_min - self.peak_one_width
peak_one_upper_limit = self.peak_one_position_max + self.peak_one_width
peak_two_lower_limit = self.peak_two_position_min - self.peak_two_width
peak_two_upper_limit = self.peak_two_position_max + self.peak_two_width
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in xrange(peak_one_lower_limit,peak_one_upper_limit):
weighted_peak_one_positions.append(spectrum[i]*i)
weighted_sum_peak_one = sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one//sum(spectrum[peak_one_lower_limit:peak_one_upper_limit])
weighted_peak_two_positions = []
for i in xrange(peak_two_lower_limit,peak_two_upper_limit):
weighted_peak_two_positions.append(spectrum[i]*i)
weighted_sum_peak_two = sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two//sum(spectrum[peak_two_lower_limit:peak_two_upper_limit])
# normalized integrated peaks
int_peak_one = np.sum(spectrum[peak_one_lower_limit:self.peak_one_position_max])/len(spectrum[peak_one_lower_limit:self.peak_one_position_max])
int_peak_two = np.sum(spectrum[peak_two_lower_limit:self.peak_two_position_max])/len(spectrum[peak_two_lower_limit:self.peak_two_position_max])
# normalized integrated regions between the peaks
int_left_region = np.sum(spectrum[0:peak_one_lower_limit])/len(spectrum[0:peak_one_lower_limit])
int_middle_region = np.sum(spectrum[peak_one_upper_limit:peak_two_lower_limit])/len(spectrum[peak_one_upper_limit:peak_two_lower_limit])
int_right_region = np.sum(spectrum[peak_two_upper_limit:])/len(spectrum[:peak_two_upper_limit])
# now to do the filtering
if peak_one/peak_two < self.peak_ratio or peak_one/peak_two > 1/self.peak_ratio:
print "event(): too low"
evt.put(skip_event_flag(), "skip_event")
return
if weighted_peak_two_center_position < self.peak_two_position_min or weighted_peak_two_center_position > self.peak_two_position_max:
print "event(): out of range high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if weighted_peak_one_center_position < self.peak_one_position_min or weighted_peak_one_center_position > self.peak_one_position_max:
print "event(): out of range low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_left_region/int_peak_one > self.normalized_peak_to_noise_ratio):
print "event(): noisy left of low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_middle_region/int_peak_one > self.normalized_peak_to_noise_ratio):
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_middle_region/int_peak_two > self.normalized_peak_to_noise_ratio):
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if not one_D and (int_right_region/int_peak_two > self.normalized_peak_to_noise_ratio):
print "event(): noisy right of high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
#iron edge at 738 pixels on FFE detetor
if one_D and (spectrum[self.iron_edge_position]>=spectrum[weighted_peak_one_center_position]):
print "event(): peak at iron edge"
evt.put(skip_event_flag(), "skip_event")
return
if one_D and (spectrum[self.iron_edge_position]>=spectrum[weighted_peak_two_center_position]):
print "event(): peak at iron edge"
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print "%d Two Color shots" %self.ntwo_color
def load_pars_from_file(self, path=None):
""" Use the dxtbx object model to build a GeometryObject hierarchy
@param path Path to the CSPAD CBF file
"""
if path is not None: self.path = path
if self.pbits & 32: print('Load file: %s' % self.path)
img = dxtbx.load(self.path)
cbf = img._cbf_handle
cbf.find_category(b"diffrn_source")
cbf.find_column(b"type")
self.dict_of_comments = {
"TITLE":
"Geometry parameters of CSPAD",
"DATE_TIME":
evt_timestamp(),
"AUTHOR":
getpass.getuser(),
"EXPERIMENT":
cbf.get_value(),
"DETECTOR":
"CSPAD",
"CALIB_TYPE":
"geometry",
"COMMENT:01":
"Table contains the list of geometry parameters for alignment of 2x1 sensors, quads, CSPAD, etc",
"COMMENT:02":
" translation and rotation pars of the object are defined w.r.t. parent object Cartesian frame",
"PARAM:01":
"PARENT - name and version of the parent object",
"PARAM:02":
"PARENT_IND - index of the parent object",
"PARAM:03":
"OBJECT - name and version of the object",
"PARAM:04":
"OBJECT_IND - index of the new object",
"PARAM:05":
"X0 - x-coordinate [um] of the object origin in the parent frame",
"PARAM:06":
"Y0 - y-coordinate [um] of the object origin in the parent frame",
"PARAM:07":
"Z0 - z-coordinate [um] of the object origin in the parent frame",
"PARAM:08":
"ROT_Z - object design rotation angle [deg] around Z axis of the parent frame",
"PARAM:09":
"ROT_Y - object design rotation angle [deg] around Y axis of the parent frame",
"PARAM:10":
"ROT_X - object design rotation angle [deg] around X axis of the parent frame",
"PARAM:11":
"TILT_Z - object tilt angle [deg] around Z axis of the parent frame",
"PARAM:12":
"TILT_Y - object tilt angle [deg] around Y axis of the parent frame",
"PARAM:13":
"TILT_X - object tilt angle [deg] around X axis of the parent frame"
}
self.list_of_geos = []
detector = img.get_detector()
hierarchy = detector.hierarchy()
for q, quad in enumerate(hierarchy):
for s, sensor in enumerate(quad):
self.list_of_geos.append(
self._load_geo(q, "QUAD:V1", s, "SENS2X1:V1", sensor))
for q, quad in enumerate(hierarchy):
self.list_of_geos.append(
self._load_geo(0, "CSPAD:V1", q, "QUAD:V1", quad))
# Add placeholder RAIL and IP vectors, including the XY component of the hierarchy's d0 vector
go = self._load_geo(0, 'RAIL', 0, 'CSPAD:V1', hierarchy)
go.move_geo(0, 0, -go.z0 + 1000000) # Placeholder
self.list_of_geos.append(go)
self.list_of_geos.append(self._null_geo(0, "IP", 0, "RAIL"))
self._set_relations()
self.valid = True
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
None,
"--as_pickle",
"-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files"
).option(
None,
"--raw_data",
"-R",
action="store_true",
default=False,
dest="raw_data",
help=
"Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
).option(
None,
"--background_pickle",
"-B",
default=None,
dest="background_pickle",
help=""
).option(
None,
"--config",
"-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file"
).option(
None,
"--experiment",
"-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)"
).option(
None,
"--run",
"-r",
type="int",
default=None,
dest="run",
help="run number"
).option(
None,
"--address",
"-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)"
).option(
None,
"--detz_offset",
"-d",
type="float",
default=None,
dest="detz_offset",
help=
"offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
).option(
None,
"--outputdir",
"-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files"
).option(
None,
"--averagebase",
"-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help=
"Path to output average image without extension. String substitution allowed"
).option(
None,
"--stddevbase",
"-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help=
"Path to output standard deviation image without extension. String substitution allowed"
).option(
None,
"--maxbase",
"-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help=
"Path to output maximum projection image without extension. String substitution allowed"
).option(
None,
"--numevents",
"-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all"
).option(
None,
"--skipevents",
"-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--pickle-optical-metrology",
"-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help=
"If writing pickle files, use the optical metrology in the experiment's calib directory"
).option(
None,
"--bin_size",
"-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size"
).option(
None,
"--override_beam_x",
"-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate"
).option(
None,
"--override_beam_y",
"-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate"
).option(
None,
"--calib_dir",
"-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory"
).option(
None,
"--pickle_calib_dir",
"-P",
type="string",
default=None,
dest="pickle_calib_dir",
metavar="PATH",
help=
"pickle calibration directory specification. Replaces --calib_dir functionality."
).option(
None,
"--xtc_dir",
"-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory"
).option(
None,
"--use_ffb",
"-f",
action="store_true",
default=False,
dest="use_ffb",
help=
"Use the fast feedback filesystem at LCLS. Only for the active experiment!"
).option(
None,
"--gain_mask_value",
"-g",
type="float",
default=None,
dest="gain_mask_value",
help=
"Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
).option(
None,
"--min",
None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection"
).option(
None,
"--minpath",
None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help=
"Path to output minimum image without extension. String substitution allowed"
)).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxsize
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
command_line.options.experiment[0:3],
command_line.options.experiment)
if command_line.options.calib_dir is not None:
psana.setOption('psana.calib-dir', command_line.options.calib_dir)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)' % address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(
command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print("Skipping first %d events" % command_line.options.skipevents)
elif "Rayonix" in command_line.options.address:
print("Skipping first image in the Rayonix detector"
) # Shuttering issue
command_line.options.skipevents = 1
for i, evt in enumerate(run.events()):
if i % size != rank: continue
if i < command_line.options.skipevents: continue
if i >= maxevents: break
if i % 10 == 0: print('Rank', rank, 'processing event', i)
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1, src)
if data is None:
print("No data")
continue
data = data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det,
evt,
use_default=False,
dark=False,
common_mode=None,
apply_gain_mask=False,
per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det,
evt,
use_default=True)
else:
data = get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=True,
gain_mask_value=command_line.options.
gain_mask_value,
per_pixel_gain=False)
if data is None:
print("No data")
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(),
command_line.options.detz_offset)
if d is None:
print("No distance, using distance",
command_line.options.detz_offset)
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print("No wavelength")
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print("No timestamp, skipping shot")
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1] / 1000)
else:
timestamp = np.array([t[0] + (t[1] / 1000)])
if 'sum' in locals():
sum += data
else:
sum = np.array(data, copy=True)
if 'sumsq' in locals():
sumsq += data * data
else:
sumsq = data * data
if 'maximum' in locals():
maximum = np.maximum(maximum, data)
else:
maximum = np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum = np.minimum(minimum, data)
else:
minimum = np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print("Synchronizing rank", rank)
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent, totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum, sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq, sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum, maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum, minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength, waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance, distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp, timeall)
if rank == 0:
if size > 1:
print("Synchronized")
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [
cspad_tbx.pathsubst(command_line.options.averagepath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
ds.env())
]
if command_line.options.do_minimum_projection:
dest_paths.append(
cspad_tbx.pathsubst(command_line.options.minpath + extension,
evt, ds.env()))
dest_paths = [
os.path.join(command_line.options.outputdir, path)
for path in dest_paths
]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
command_line.options.bin_size)
beam_center = [
command_line.options.override_beam_x,
command_line.options.override_beam_y
]
active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(address=old_style_address,
data=flex.double(data),
distance=distance,
pixel_size=0.1,
timestamp=timestamp,
wavelength=wavelength)
print("Saving", path)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(
evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
old_style_address,
reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.pickle_calib_dir is not None:
metro_path = command_line.options.pickle_calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
address,
round_to_int=False)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata +=1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(),"skip_event")
if timestamp is None:
evt.put(skip_event_flag(),"skip_event")
#self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots +=1
# get data as array and split into two half to find each peak
if one_D:
# filtering out outlier spikes in FEE data
data = np.array(data.hproj().astype(np.float64))
for i in xrange(len(data)):
if data[i]>1000000000:
data[i]=data[i]-(2**32)
if self.dark is not None:
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0]//2]
spectrum2 = data[data.shape[0]//2:]
else:
data = np.array(data.data16().astype(np.int32))
if self.dark is not None:
data = data - self.dark
data = np.double(data)
data_split1 = data[:,:data.shape[1]//2]
data_split2 = data[:,data.shape[1]//2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data,0)/data.shape[0]
spectrum1 = np.sum(data_split1,0)/data_split1.shape[0]
spectrum2 = np.sum(data_split2,0)/data_split2.shape[0]
if not one_D:
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in xrange(self.peak_one_range_min,self.peak_one_range_max):
weighted_peak_one_positions.append(spectrum[i]*i)
weighted_sum_peak_one = np.sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one/np.sum(spectrum[self.peak_one_range_min:self.peak_one_range_max])
weighted_peak_two_positions = []
for i in xrange(self.peak_two_range_min,self.peak_two_range_max):
weighted_peak_two_positions.append(spectrum[i]*i)
weighted_sum_peak_two = np.sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two/np.sum(spectrum[self.peak_two_range_min:self.peak_two_range_max])
# normalized integrated regions between the peaks
#int_left_region = np.sum(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_left_region = np.sum(spectrum[:weighted_peak_two_center_position/2])
#int_left_region_norm = np.sum(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2:(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_left_region_norm = np.sum(spectrum[:weighted_peak_two_center_position/2])/len(spectrum[:weighted_peak_two_center_position/2])
int_right_region = np.sum(spectrum[self.peak_two_range_max:])
int_right_region_norm = np.sum(spectrum[self.peak_two_range_max:])/len(spectrum[self.peak_two_range_max:])
# normalized integrated peaks
int_peak_one = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_one_norm = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])/len(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_two = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_peak_two_norm = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
if not one_D:
if int_peak_one_norm/int_peak_two_norm > self.peak_ratio:
print "event(): inflection peak too high"
evt.put(skip_event_flag(), "skip_event")
return
if int_left_region_norm > self.normalized_peak_to_noise_ratio*int_peak_two_norm:
print "event(): noisy left of low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if int_right_region_norm > self.normalized_peak_to_noise_ratio*int_peak_two_norm:
print "event(): noisy right of high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print "%d Remote shot" %self.ntwo_color
print "%s Remote timestamp" %timestamp
def run(self):
""" Process all images assigned to this thread """
params, options = self.parser.parse_args(show_diff_phil=True)
if params.input.experiment is None or \
params.input.run_num is None or \
params.input.address is None:
raise Usage(self.usage)
if params.format.file_format == "cbf":
if params.format.cbf.detz_offset is None:
raise Usage(self.usage)
elif params.format.file_format == "pickle":
if params.format.pickle.cfg is None:
raise Usage(self.usage)
else:
raise Usage(self.usage)
if not os.path.exists(params.output.output_dir):
raise Sorry("Output path not found:" + params.output.output_dir)
# Save the paramters
self.params = params
self.options = options
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank(
) # each process in MPI has a unique id, 0-indexed
size = comm.Get_size() # size: number of processes running in this job
# set up psana
if params.format.file_format == "pickle":
psana.setConfigFile(params.format.pickle.cfg)
dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment,
params.input.run_num)
ds = psana.DataSource(dataset_name)
if params.format.file_format == "cbf":
src = psana.Source('DetInfo(%s)' % params.input.address)
psana_det = psana.Detector(params.input.address, ds.env())
# set this to sys.maxint to analyze all events
if params.dispatch.max_events is None:
max_events = sys.maxint
else:
max_events = params.dispatch.max_events
for run in ds.runs():
if params.format.file_format == "cbf":
# load a header only cspad cbf from the slac metrology
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, params.input.address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
# list of all events
times = run.times()
nevents = min(len(times), max_events)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [
times[i] for i in xrange(nevents) if (i + rank) % size == 0
]
for i in xrange(len(mytimes)):
evt = run.event(mytimes[i])
id = evt.get(psana.EventId)
print "Event #", i, " has id:", id
timestamp = cspad_tbx.evt_timestamp(
cspad_tbx.evt_time(evt)) # human readable format
if timestamp is None:
print "No timestamp, skipping shot"
continue
t = timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[
17:19] + t[20:23]
print "Processing shot", s
if params.format.file_format == "pickle":
if evt.get("skip_event"):
print "Skipping event", id
continue
# the data needs to have already been processed and put into the event by psana
data = evt.get(params.format.pickle.out_key)
if data is None:
print "No data"
continue
# set output paths according to the templates
path = os.path.join(params.output.output_dir,
"shot-" + s + ".pickle")
print "Saving", path
easy_pickle.dump(path, data)
elif params.format.file_format == "cbf":
# get numpy array, 32x185x388
data = cspad_cbf_tbx.get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=params.format.cbf.gain_mask_value
is not None,
gain_mask_value=params.format.cbf.gain_mask_value,
per_pixel_gain=False)
distance = cspad_tbx.env_distance(
params.input.address, run.env(),
params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
continue
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
continue
else:
wavelength = 12398.4187 / self.params.format.cbf.override_energy
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx, data, distance, wavelength, timestamp,
params.input.address)
path = os.path.join(params.output.output_dir,
"shot-" + s + ".cbf")
print "Saving", path
# write the file
import pycbf
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
run.end()
ds.end()
if params.resolution is None:
print "Generating annular gain mask using %s metrology between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms" % \
(str(params.detector_format_version), params.annulus_inner, params.annulus_outer, params.distance, params.wavelength)
else:
print "Generating annular gain mask using %s metrology between %f and %f angstroms, assuming a distance %s mm and wavelength %s angstroms. Also, pixels higher than %f angstroms will be set to low gain." % \
(str(params.detector_format_version), params.annulus_inner, params.annulus_outer, params.distance, params.wavelength, params.resolution)
elif params.resolution is not None:
print "Generating circular gain mask using %s metrology at %s angstroms, assuming a distance %s mm and wavelength %s angstroms" % \
(str(params.detector_format_version), params.resolution, params.distance, params.wavelength)
from xfel.cxi.cspad_ana.cspad_tbx import dpack, evt_timestamp, cbcaa, pixel_size, CsPadDetector
from iotbx.detectors.cspad_detector_formats import address_and_timestamp_from_detector_format_version
from xfel.command_line.convert_gain_map import fake_env, fake_config, fake_evt, fake_cspad_ElementV2
address, timestamp = address_and_timestamp_from_detector_format_version(
params.detector_format_version)
timestamp = evt_timestamp((timestamp, 0))
raw_data = numpy.zeros((11840, 194))
if params.detector_format_version is not None and "XPP" in params.detector_format_version:
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
active_areas = xpp_active_areas[
params.detector_format_version]['active_areas']
data = flex.int(flex.grid((1765, 1765)))
beam_center = (1765 // 2, 1765 // 2)
else:
if params.optical_metrology_path is None:
calib_dir = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = calib2sections(calib_dir)
else:
sections = calib2sections(params.optical_metrology_path)
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
# Increase the event counter, even if this event is to be skipped.
self.nshots += 1
if (evt.get("skip_event")):
return
sifoil = cspad_tbx.env_sifoil(env)
if self.check_beam_status and sifoil is None:
self.nfail += 1
self.logger.warning("event(): no Si-foil thickness, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if (self.sifoil is not None and self.sifoil != sifoil):
self.logger.warning("event(): Si-foil changed mid-run: % 8i -> % 8d" %
(self.sifoil, sifoil))
self.sifoil = sifoil
if self.verbose: self.logger.info("Si-foil thickness: %i" %sifoil)
self.evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
self.timestamp = cspad_tbx.evt_timestamp(self.evt_time) # human readable format
if (self.timestamp is None):
self.nfail += 1
self.logger.warning("event(): no timestamp, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if self.verbose: self.logger.info(self.timestamp)
if self.override_energy is None:
self.wavelength = cspad_tbx.evt_wavelength(evt, self.delta_k)
if self.wavelength is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no wavelength, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.wavelength = 0
else:
self.wavelength = 12398.4187/self.override_energy
if self.verbose: self.logger.info("Wavelength: %.4f" %self.wavelength)
self.pulse_length = cspad_tbx.evt_pulse_length(evt)
if self.pulse_length is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no pulse length, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.pulse_length = 0
if self.verbose: self.logger.info("Pulse length: %s" %self.pulse_length)
self.beam_charge = cspad_tbx.evt_beam_charge(evt)
if self.beam_charge is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no beam charge, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.beam_charge = 0
if self.verbose: self.logger.info("Beam charge: %s" %self.beam_charge)
self.injector_xyz = cspad_tbx.env_injector_xyz(env)
#if self.injector_xyz is not None:
#self.logger.info("injector_z: %i" %self.injector_xyz[2].value)
self.laser_1_status.set_status(cspad_tbx.env_laser_status(env, laser_id=1), self.evt_time)
self.laser_4_status.set_status(cspad_tbx.env_laser_status(env, laser_id=4), self.evt_time)
self.laser_1_ms_since_change = self.laser_1_status.ms_since_last_status_change(self.evt_time)
self.laser_4_ms_since_change = self.laser_4_status.ms_since_last_status_change(self.evt_time)
if self.verbose:
if self.laser_1_ms_since_change is not None:
self.logger.info("ms since laser 1 status change: %i" %self.laser_1_ms_since_change)
if self.laser_4_ms_since_change is not None:
self.logger.info("ms since laser 4 status change: %i" %self.laser_4_ms_since_change)
if self.laser_1_status is not None and self.laser_1_status.status is not None:
self.logger.info("Laser 1 status: %i" %int(self.laser_1_status.status))
if self.laser_4_status is not None and self.laser_4_status.status is not None:
self.logger.info("Laser 4 status: %i" %int(self.laser_4_status.status))
def run(argv=None):
if argv is None:
argv = sys.argv[1:]
command_line = (libtbx.option_parser.option_parser(
usage="%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files" % libtbx.env.dispatcher_name)
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--crop", "-c",
action="store_true",
default=False,
dest="crop",
help="Crop the image such that the beam center is in the middle")
.option(None, "--skip_converted", "-s",
action="store_true",
default=False,
dest="skip_converted",
help="Skip converting if an image already exist that matches the destination file name")
.option(None, "--wavelength", "-w",
type="float",
default=None,
dest="wavelength",
help="Override the image's wavelength (angstroms)")
.option(None, "--distance", "-d",
type="float",
default=None,
dest="distance",
help="Override the detector distance (mm)")
.option(None, "--pixel_size", "-p",
type="float",
default=None,
dest="pixel_size",
help="Override the detector pixel size (mm)")
.option(None, "--beam_x", "-x",
type="float",
default=None,
dest="beam_center_x",
help="Override the beam x position (pixels)")
.option(None, "--beam_y", "-y",
type="float",
default=None,
dest="beam_center_y",
help="Override the beam y position (pixels)")
.option(None, "--overload", "-o",
type="float",
default=None,
dest="overload",
help="Override the detector overload value (ADU)")
).process(args=argv)
paths = command_line.args
if len(paths) <= 0:
raise Usage("No files specified")
for imgpath in paths:
if command_line.options.verbose:
print "Reading %s"%(imgpath)
try:
img = dxtbx.load(imgpath)
except IOError:
img = None
pass
if img is None:
import numpy as np
try:
raw_data = np.loadtxt(imgpath)
from scitbx.array_family import flex
raw_data = flex.double(raw_data.astype(np.double))
except ValueError:
raise Usage("Couldn't load %s, no supported readers"%imgpath)
detector = None
beam = None
scan = None
is_multi_image = False
else:
try:
raw_data = img.get_raw_data()
is_multi_image = False
except TypeError:
raw_data = img.get_raw_data(0)
is_multi_image = True
detector = img.get_detector()
beam = img.get_beam()
scan = img.get_scan()
if detector is None:
if command_line.options.distance is None:
raise Usage("Can't get distance from image. Override with -d")
if command_line.options.pixel_size is None:
raise Usage("Can't get pixel size from image. Override with -p")
if command_line.options.overload is None:
raise Usage("Can't get overload value from image. Override with -o")
distance = command_line.options.distance
pixel_size = command_line.options.pixel_size
overload = command_line.options.overload
else:
detector = detector[0]
if command_line.options.distance is None:
distance = detector.get_distance()
else:
distance = command_line.options.distance
if command_line.options.pixel_size is None:
pixel_size = detector.get_pixel_size()[0]
else:
pixel_size = command_line.options.pixel_size
if command_line.options.overload is None:
overload = detector.get_trusted_range()[1]
else:
overload = command_line.options.overload
if beam is None:
if command_line.options.wavelength is None:
raise Usage("Can't get wavelength from image. Override with -w")
wavelength = command_line.options.wavelength
else:
if command_line.options.wavelength is None:
wavelength = beam.get_wavelength()
else:
wavelength = command_line.options.wavelength
if beam is None and detector is None:
if command_line.options.beam_center_x is None:
print "Can't get beam x position from image. Using image center. Override with -x"
beam_x = raw_data.focus()[0] * pixel_size
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
print "Can't get beam y position from image. Using image center. Override with -y"
beam_y = raw_data.focus()[1] * pixel_size
else:
beam_y = command_line.options.beam_center_y * pixel_size
else:
if command_line.options.beam_center_x is None:
beam_x = detector.get_beam_centre(beam.get_s0())[0]
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
beam_y = detector.get_beam_centre(beam.get_s0())[1]
else:
beam_y = command_line.options.beam_center_y * pixel_size
if scan is None:
timestamp = None
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec,msec))
if is_multi_image:
for i in xrange(img.get_num_images()):
save_image(command_line, imgpath, scan, img.get_raw_data(i), distance, pixel_size, wavelength, beam_x, beam_y, overload, timestamp, image_number = i)
else:
save_image(command_line, imgpath, scan, raw_data, distance, pixel_size, wavelength, beam_x, beam_y, overload, timestamp)
times = times[params.skip_events:]
nevents = min(len(times),max_events)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0]
for i, t in enumerate(mytimes):
evt = run.event(t)
accepted, data, spectrum, dc_offset, all_data, all_data_raw = spf.filter_event(evt, filter_name)
if not accepted:
continue
print cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)), "Publishing data for event", i
#header = "Event %d, m/f: %7.7f, f: %d"%(i, peak_max/flux, flux)
header = "Event %d"%(i)
if rank == 0:
fee = Image(header, "FEE", data) # make a 2D plot
publish.send("FEE", fee) # send to the display
spectrumplot = XYPlot(header, 'summed 1D trace', range(data.shape[1]), spectrum) # make a 1D plot
publish.send("SPECTRUM", spectrumplot) # send to the display
fee = Image(header, "DC_OFFSET", dc_offset) # make a 2D plot
publish.send("DC_OFFSET", fee) # send to the display
fee = Image(header, "ALL_FEE", all_data) # make a 2D plot
publish.send("ALL_FEE", fee) # send to the display
import psana
from xfel.cxi.cspad_ana import cspad_tbx
from scitbx.array_family import flex # implicit import
dataset_name = "exp=mfxls4916:run=17:idx"
ds = psana.DataSource(dataset_name)
#address = "MfxEndstation-0|Rayonix-0"
#src = psana.Source('DetInfo(%s)'%address)
from matplotlib import pyplot as plt
plt.ion()
for run in ds.runs():
times = run.times()
for i, t in enumerate(times):
evt = run.event(t)
ts = cspad_tbx.evt_timestamp((t.seconds(), t.nanoseconds() / 1e6))
print(ts, end=' ')
for key in evt:
if key.alias() in ['Rayonix', 'Jungfrau1M']:
print(key.alias(), end=' ')
if 'FEE-SPEC0' in str(key):
print(key.src(), end=' ')
d = evt.get(key.type(), key.src())
plt.cla()
plt.plot(range(len(d.hproj())), d.hproj(), '-')
plt.draw()
plt.pause(0.1)
print()
def run(self):
""" Process all images assigned to this thread """
params, options = self.parser.parse_args(show_diff_phil=True)
if params.input.experiment is None or \
params.input.run_num is None or \
params.input.address is None:
raise Usage(self.usage)
if params.format.file_format == "cbf":
if params.format.cbf.detz_offset is None:
raise Usage(self.usage)
elif params.format.file_format == "pickle":
if params.input.cfg is None:
raise Usage(self.usage)
else:
raise Usage(self.usage)
if not os.path.exists(params.output.output_dir):
raise Sorry("Output path not found:" + params.output.output_dir)
#Environment variable redirect for CBFLib temporary CBF_TMP_XYZ file output
if params.format.file_format == "cbf":
if params.output.tmp_output_dir is None:
tmp_dir = os.path.join(params.output.output_dir, '.tmp')
else:
tmp_dir = os.path.join(params.output.tmp_output_dir, '.tmp')
if not os.path.exists(tmp_dir):
with show_mail_on_error():
try:
os.makedirs(tmp_dir)
# Can fail if running multiprocessed - that's OK if the folder was created
except OSError as e: # In Python 2, a FileExistsError is just an OSError
if e.errno != errno.EEXIST: # If this OSError is not a FileExistsError
raise
os.environ['CBF_TMP_DIR'] = tmp_dir
# Save the paramters
self.params = params
self.options = options
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank(
) # each process in MPI has a unique id, 0-indexed
size = comm.Get_size() # size: number of processes running in this job
# set up psana
if params.input.cfg is not None:
psana.setConfigFile(params.input.cfg)
if params.input.calib_dir is not None:
psana.setOption('psana.calib-dir', params.input.calib_dir)
dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment,
params.input.run_num)
if params.input.xtc_dir is not None:
dataset_name = "exp=%s:run=%s:idx:dir=%s" % (
params.input.experiment, params.input.run_num,
params.input.xtc_dir)
ds = psana.DataSource(dataset_name)
if params.format.file_format == "cbf":
src = psana.Source('DetInfo(%s)' % params.input.address)
psana_det = psana.Detector(params.input.address, ds.env())
# set this to sys.maxint to analyze all events
if params.dispatch.max_events is None:
max_events = sys.maxsize
else:
max_events = params.dispatch.max_events
for run in ds.runs():
if params.format.file_format == "cbf":
if params.format.cbf.mode == "cspad":
# load a header only cspad cbf from the slac metrology
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, params.input.address)
if base_dxtbx is None:
raise Sorry(
"Couldn't load calibration file for run %d" %
run.run())
elif params.format.cbf.mode == "rayonix":
# load a header only rayonix cbf from the input parameters
detector_size = rayonix_tbx.get_rayonix_detector_dimensions(
ds.env())
base_dxtbx = rayonix_tbx.get_dxtbx_from_params(
params.format.cbf.rayonix, detector_size)
# list of all events
times = run.times()
if params.dispatch.selected_events:
times = [
t for t in times
if cspad_tbx.evt_timestamp((t.seconds(), t.nanoseconds() /
1e6)) in params.input.timestamp
]
nevents = min(len(times), max_events)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [
times[i] for i in range(nevents) if (i + rank) % size == 0
]
for i in range(len(mytimes)):
evt = run.event(mytimes[i])
id = evt.get(psana.EventId)
print("Event #", i, " has id:", id)
timestamp = cspad_tbx.evt_timestamp(
cspad_tbx.evt_time(evt)) # human readable format
if timestamp is None:
print("No timestamp, skipping shot")
continue
if evt.get("skip_event") or "skip_event" in [
key.key() for key in evt.keys()
]:
print("Skipping event", timestamp)
continue
t = timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[
17:19] + t[20:23]
print("Processing shot", s)
if params.format.file_format == "pickle":
if evt.get("skip_event"):
print("Skipping event", id)
continue
# the data needs to have already been processed and put into the event by psana
data = evt.get(params.format.pickle.out_key)
if data is None:
print("No data")
continue
# set output paths according to the templates
path = os.path.join(params.output.output_dir,
"shot-" + s + ".pickle")
print("Saving", path)
easy_pickle.dump(path, data)
elif params.format.file_format == "cbf":
if params.format.cbf.mode == "cspad":
# get numpy array, 32x185x388
data = cspad_cbf_tbx.get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=params.format.cbf.cspad.
gain_mask_value is not None,
gain_mask_value=params.format.cbf.cspad.
gain_mask_value,
per_pixel_gain=False)
distance = cspad_tbx.env_distance(
params.input.address, run.env(),
params.format.cbf.detz_offset)
elif params.format.cbf.mode == "rayonix":
data = rayonix_tbx.get_data_from_psana_event(
evt, params.input.address)
distance = params.format.cbf.detz_offset
if distance is None:
print("No distance, skipping shot")
continue
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print("No wavelength, skipping shot")
continue
else:
wavelength = 12398.4187 / self.params.format.cbf.override_energy
# stitch together the header, data and metadata into the final dxtbx format object
if params.format.cbf.mode == "cspad":
image = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
params.input.address,
round_to_int=False)
elif params.format.cbf.mode == "rayonix":
image = rayonix_tbx.format_object_from_data(
base_dxtbx, data, distance, wavelength, timestamp,
params.input.address)
path = os.path.join(params.output.output_dir,
"shot-" + s + ".cbf")
print("Saving", path)
# write the file
import pycbf
image._cbf_handle.write_widefile(path.encode(), pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
run.end()
ds.end()
def run(args):
user_phil = []
for arg in args:
if os.path.isfile(arg):
try:
user_phil.append(parse(file_name=arg))
except Exception as e:
print(str(e))
raise Sorry("Couldn't parse phil file %s" % arg)
else:
try:
user_phil.append(parse(arg))
except Exception as e:
print(str(e))
raise Sorry("Couldn't parse argument %s" % arg)
params = phil_scope.fetch(sources=user_phil).extract()
# cxid9114, source fee: FeeHxSpectrometer.0:Opal1000.1, downstream: CxiDg3.0:Opal1000.0
# cxig3614, source fee: FeeHxSpectrometer.0:OrcaFl40.0
src = psana.Source(params.spectra_filter.detector_address)
dataset_name = "exp=%s:run=%s:idx" % (params.experiment, params.runs)
print("Dataset string:", dataset_name)
ds = psana.DataSource(dataset_name)
spf = spectra_filter(params)
if params.selected_filter == None:
filter_name = params.spectra_filter.filter[0].name
else:
filter_name = params.selected_filter
rank = 0
size = 1
max_events = sys.maxsize
for run in ds.runs():
print("starting run", run.run())
# list of all events
times = run.times()
if params.skip_events is not None:
times = times[params.skip_events:]
nevents = min(len(times), max_events)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in range(nevents) if (i + rank) % size == 0]
for i, t in enumerate(mytimes):
evt = run.event(t)
accepted, data, spectrum, dc_offset, all_data, all_data_raw = spf.filter_event(
evt, filter_name)
if not accepted:
continue
print(cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)),
"Publishing data for event", i)
#header = "Event %d, m/f: %7.7f, f: %d"%(i, peak_max/flux, flux)
header = "Event %d" % (i)
if rank == 0:
fee = Image(header, "FEE", data) # make a 2D plot
publish.send("FEE", fee) # send to the display
spectrumplot = XYPlot(header, 'summed 1D trace',
list(range(data.shape[1])),
spectrum) # make a 1D plot
publish.send("SPECTRUM", spectrumplot) # send to the display
fee = Image(header, "DC_OFFSET", dc_offset) # make a 2D plot
publish.send("DC_OFFSET", fee) # send to the display
fee = Image(header, "ALL_FEE", all_data) # make a 2D plot
publish.send("ALL_FEE", fee) # send to the display
fee = Image(header, "ALL_FEE_RAW",
all_data_raw) # make a 2D plot
publish.send("ALL_FEE_RAW", fee) # send to the display
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
# Increase the event counter, even if this event is to be skipped.
self.nshots += 1
if (evt.get("skip_event")):
return
sifoil = cspad_tbx.env_sifoil(env)
if self.check_beam_status and sifoil is None:
self.nfail += 1
self.logger.warning("event(): no Si-foil thickness, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if (self.sifoil is not None and self.sifoil != sifoil):
self.logger.warning(
"event(): Si-foil changed mid-run: % 8i -> % 8d" %
(self.sifoil, sifoil))
self.sifoil = sifoil
if self.verbose: self.logger.info("Si-foil thickness: %i" % sifoil)
self.evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
self.timestamp = cspad_tbx.evt_timestamp(
self.evt_time) # human readable format
if (self.timestamp is None):
self.nfail += 1
self.logger.warning("event(): no timestamp, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
if self.verbose: self.logger.info(self.timestamp)
if self.override_energy is None:
self.wavelength = cspad_tbx.evt_wavelength(evt, self.delta_k)
if self.wavelength is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no wavelength, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.wavelength = 0
else:
self.wavelength = 12398.4187 / self.override_energy
if self.verbose: self.logger.info("Wavelength: %.4f" % self.wavelength)
self.pulse_length = cspad_tbx.evt_pulse_length(evt)
if self.pulse_length is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no pulse length, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.pulse_length = 0
if self.verbose:
self.logger.info("Pulse length: %s" % self.pulse_length)
self.beam_charge = cspad_tbx.evt_beam_charge(evt)
if self.beam_charge is None:
if self.check_beam_status:
self.nfail += 1
self.logger.warning("event(): no beam charge, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
else:
self.beam_charge = 0
if self.verbose: self.logger.info("Beam charge: %s" % self.beam_charge)
self.injector_xyz = cspad_tbx.env_injector_xyz(env)
#if self.injector_xyz is not None:
#self.logger.info("injector_z: %i" %self.injector_xyz[2].value)
self.laser_1_status.set_status(
cspad_tbx.env_laser_status(env, laser_id=1), self.evt_time)
self.laser_4_status.set_status(
cspad_tbx.env_laser_status(env, laser_id=4), self.evt_time)
self.laser_1_ms_since_change = self.laser_1_status.ms_since_last_status_change(
self.evt_time)
self.laser_4_ms_since_change = self.laser_4_status.ms_since_last_status_change(
self.evt_time)
if self.verbose:
if self.laser_1_ms_since_change is not None:
self.logger.info("ms since laser 1 status change: %i" %
self.laser_1_ms_since_change)
if self.laser_4_ms_since_change is not None:
self.logger.info("ms since laser 4 status change: %i" %
self.laser_4_ms_since_change)
if self.laser_1_status is not None and self.laser_1_status.status is not None:
self.logger.info("Laser 1 status: %i" %
int(self.laser_1_status.status))
if self.laser_4_status is not None and self.laser_4_status.status is not None:
self.logger.info("Laser 4 status: %i" %
int(self.laser_4_status.status))
def run(argv=None):
"""Compute mean, standard deviation, and maximum projection images
from a set of images given on the command line.
@param argv Command line argument list
@return @c 0 on successful termination, @c 1 on error, and @c 2
for command line syntax errors
"""
import libtbx.load_env
from libtbx import easy_pickle, option_parser
from scitbx.array_family import flex
from xfel.cxi.cspad_ana import cspad_tbx
if argv is None:
argv = sys.argv
command_line = (option_parser.option_parser(
usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
"image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
.option(None, "--average-path", "-a",
type="string",
default=None,
dest="avg_path",
metavar="PATH",
help="Write average image to PATH")
.option(None, "--maximum-path", "-m",
type="string",
default=None,
dest="max_path",
metavar="PATH",
help="Write maximum projection image to PATH")
.option(None, "--stddev-path", "-s",
type="string",
default=None,
dest="stddev_path",
metavar="PATH",
help="Write standard deviation image to PATH")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
).process(args=argv[1:])
# Note that it is not an error to omit the output paths, because
# certain statistics could still be printed, e.g. with the verbose
# option.
paths = command_line.args
if len(paths) == 0:
command_line.parser.print_usage(file=sys.stderr)
return 2
# Loop over all images and accumulate statistics.
nfail = 0
nmemb = 0
if len(paths) == 1:
# test if the iamge is a multi-image
from dxtbx.format.Registry import Registry
from dxtbx.format.FormatMultiImage import FormatMultiImage
format_class = Registry.find(paths[0])
if not issubclass(format_class, FormatMultiImage):
from libtbx.utils import Usage
raise Usage("Supply more than one image")
print "Loading image..."
i = format_class(paths[0])
print "Loaded"
def read_single_image(n):
if command_line.options.verbose:
sys.stdout.write("Processing %s: %d...\n" % (paths[0], n))
beam = i.get_beam(n)
assert len(i.get_detector(n)) == 1
detector = i.get_detector(n)[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = i.get_raw_data(n).as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
scan = i.get_scan(n)
if scan is None:
time_tuple = (0, 0)
else:
time_tuple = (scan.get_epochs()[0], 0)
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas
iterable = xrange(i.get_num_images())
else:
def read_single_image(path):
if command_line.options.verbose:
sys.stdout.write("Processing %s...\n" % path)
from dxtbx.format.Registry import Registry
format_class = Registry.find(path)
i = format_class(path)
beam = i.get_beam()
assert len(i.get_detector()) == 1
detector = i.get_detector()[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = i.get_raw_data().as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
scan = i.get_scan()
if scan is None:
time_tuple = (0, 0)
else:
time_tuple = (scan.get_epochs()[0], 0)
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
return beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas
iterable = paths
for item in iterable:
try:
#XXX This code assumes a monolithic detector!
beam_center, detector_address, distance, img, pixel_size, saturated_value, size, time_tuple, wavelength, active_areas = \
read_single_image(item)
except Exception:
nfail += 1
continue
# See also event() in xfel.cxi.cspad_ana.average_tbx. Record the
# base time as the timestamp of the first image.
#
# The sum-of-squares image is accumulated using long integers, as
# this delays the point where overflow occurs. But really, this
# is just a band-aid...
if nmemb == 0:
max_img = img.deep_copy()
sum_distance = distance
sum_img = img.deep_copy()
ssq_img = flex.pow2(img)
sum_wavelength = wavelength
sum_time = (0, 0)
time_base = time_tuple
else:
sel = (img > max_img).as_1d()
max_img.set_selected(sel, img.select(sel))
sum_distance += distance
sum_img += img
ssq_img += flex.pow2(img)
sum_wavelength += wavelength
sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]),
sum_time[1] + (time_tuple[1] - time_base[1]))
nmemb += 1
# Early exit if no statistics were accumulated.
if command_line.options.verbose:
sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
if nmemb == 0:
return 0
# Calculate averages for measures where other statistics do not make
# sense. Note that avg_img is required for stddev_img.
avg_img = sum_img.as_double() / nmemb
avg_distance = sum_distance / nmemb
avg_timestamp = cspad_tbx.evt_timestamp(
(time_base[0] + int(round(sum_time[0] / nmemb)),
time_base[1] + int(round(sum_time[1] / nmemb))))
avg_wavelength = sum_wavelength / nmemb
# Output the average image, maximum projection image, and standard
# deviation image, if requested.
if command_line.options.avg_path is not None:
avg_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=avg_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.avg_path, d)
if command_line.options.max_path is not None:
max_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=max_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.max_path, d)
if command_line.options.stddev_path is not None:
stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img
# Accumulating floating-point numbers introduces errors, which may
# cause negative variances. Since a two-pass approach is
# unacceptable, the standard deviation is clamped at zero.
stddev_img.set_selected(stddev_img < 0, 0)
if nmemb == 1:
stddev_img = flex.sqrt(stddev_img)
else:
stddev_img = flex.sqrt(stddev_img / (nmemb - 1))
stddev_img.resize(flex.grid(size[1], size[0]))
d = cspad_tbx.dpack(active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=stddev_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.stddev_path, d)
return 0
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(average_mixin, self).event(evt, env)
if evt.get('skip_event'):
return
# Get the distance for the detectors that should have it, and set
# it to NaN for those that should not.
if self.detector == 'CxiDs1' or \
self.detector == 'CxiDs2' or \
self.detector == 'CxiDsd' or \
self.detector == 'XppGon':
distance = cspad_tbx.env_distance(self.address, env, self._detz_offset)
if distance is None:
self._nfail += 1
self.logger.warning("event(): no distance, shot skipped")
evt.put(skip_event_flag(), 'skip_event')
return
else:
distance = float('nan')
if ("skew" in self.flags):
# Take out inactive pixels
if self.roi is not None:
pixels = self.cspad_img[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
dark_mask = self.dark_mask[self.roi[2]:self.roi[3], self.roi[0]:self.roi[1]]
pixels = pixels.as_1d().select(dark_mask.as_1d())
else:
pixels = self.cspad_img.as_1d().select(self.dark_mask.as_1d()).as_double()
stats = scitbx.math.basic_statistics(pixels.as_double())
#stats.show()
self.logger.info("skew: %.3f" %stats.skew)
self.logger.info("kurtosis: %.3f" %stats.kurtosis)
if 0:
from matplotlib import pyplot
hist_min, hist_max = flex.min(flex_cspad_img.as_double()), flex.max(flex_cspad_img.as_double())
print hist_min, hist_max
n_slots = 100
n, bins, patches = pyplot.hist(flex_cspad_img.as_1d().as_numpy_array(), bins=n_slots, range=(hist_min, hist_max))
pyplot.show()
# XXX This skew threshold probably needs fine-tuning
skew_threshold = 0.35
if stats.skew < skew_threshold:
self._nfail += 1
self.logger.warning("event(): skew < %f, shot skipped" % skew_threshold)
evt.put(skip_event_flag(), 'skip_event')
return
#self.cspad_img *= stats.skew
if ("inactive" in self.flags):
self.cspad_img.set_selected(self.dark_stddev <= 0, 0)
if ("noelastic" in self.flags):
ELASTIC_THRESHOLD = self.elastic_threshold
self.cspad_img.set_selected(self.cspad_img > ELASTIC_THRESHOLD, 0)
if self.hot_threshold is not None:
HOT_THRESHOLD = self.hot_threshold
self.cspad_img.set_selected(self.dark_img > HOT_THRESHOLD, 0)
if self.gain_map is not None and self.gain_threshold is not None:
# XXX comparing each pixel to a moving average would probably be better
# since the gain should vary approximately smoothly over different areas
# of the detector
GAIN_THRESHOLD = self.gain_threshold
#self.logger.debug(
#"rejecting: %i" %(self.gain_map > GAIN_THRESHOLD).count(True))
self.cspad_img.set_selected(self.gain_map > GAIN_THRESHOLD, 0)
if ("nonoise" in self.flags):
NOISE_THRESHOLD = self.noise_threshold
self.cspad_img.set_selected(self.cspad_img < NOISE_THRESHOLD, 0)
if ("sigma_scaling" in self.flags):
self.do_sigma_scaling()
if ("symnoise" in self.flags):
SYMNOISE_THRESHOLD = self.symnoise_threshold
self.cspad_img.set_selected((-SYMNOISE_THRESHOLD < self.cspad_img) &
( self.cspad_img < SYMNOISE_THRESHOLD), 0)
if ("output" in self.flags):
import pickle,os
if (not os.path.isdir(self.pickle_dirname)):
os.makedirs(self.pickle_dirname)
flexdata = flex.int(self.cspad_img.astype(numpy.int32))
d = cspad_tbx.dpack(
address=self.address,
data=flexdata,
timestamp=cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt))
)
G = open(os.path.join(".",self.pickle_dirname)+"/"+self.pickle_basename,
"ab")
pickle.dump(d,G,pickle.HIGHEST_PROTOCOL)
G.close()
if self.photon_threshold is not None and self.two_photon_threshold is not None:
self.do_photon_counting()
if self.background_path is not None:
self.cspad_img -= self.background_img
# t and self._sum_time are a two-long arrays of seconds and
# milliseconds which hold time with respect to the base time.
t = [t1 - t2 for (t1, t2) in zip(cspad_tbx.evt_time(evt),
self._metadata['time_base'])]
if self._nmemb == 0:
# The peers metadata item is a bit field where a bit is set if
# the partial sum from the corresponding worker process is
# pending. If this is the first frame a worker process sees,
# set its corresponding bit in the bit field since it will
# contribute a partial sum.
if env.subprocess() >= 0:
self._lock.acquire()
if 'peers' in self._metadata.keys():
self._metadata['peers'] |= (1 << env.subprocess())
else:
self._metadata['peers'] = (1 << env.subprocess())
self._lock.release()
self._sum_distance = distance
self._sum_time = (t[0], t[1])
self._sum_wavelength = self.wavelength
if self._have_max:
self._max_img = self.cspad_img.deep_copy()
if self._have_mean:
self._sum_img = self.cspad_img.deep_copy()
if self._have_std:
self._ssq_img = flex.pow2(self.cspad_img)
else:
self._sum_distance += distance
self._sum_time = (self._sum_time[0] + t[0], self._sum_time[1] + t[1])
self._sum_wavelength += self.wavelength
if self._have_max:
sel = (self.cspad_img > self._max_img).as_1d()
self._max_img.as_1d().set_selected(
sel, self.cspad_img.as_1d().select(sel))
if self._have_mean:
self._sum_img += self.cspad_img
if self._have_std:
self._ssq_img += flex.pow2(self.cspad_img)
self._nmemb += 1
class SingleImage(object):
def __init__(self, img, init, verbose=True, imported_grid=None):
""" Constructor for the SingleImage object using a raw image file or pickle
"""
# Initialize parameters
self.params = init.params
self.args = init.args
self.user_id = init.user_id
self.raw_img = img[2]
self.conv_img = img[2]
self.img_index = img[0]
self.center_int = 0
self.status = None
self.fail = None
self.final = None
self.log_info = []
self.gs_results = []
self.main_log = init.logfile
self.verbose = verbose
self.hmed = self.params.cctbx.grid_search.height_median
self.amed = self.params.cctbx.grid_search.area_median
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.viz_base = init.viz_base
self.log_base = init.log_base
self.tmp_base = init.tmp_base
self.abort_file = os.path.join(self.int_base, '.abort.tmp')
self.obj_path = None
self.obj_file = None
self.fin_path = None
self.fin_file = None
self.viz_path = None
# ============================== SELECTION-ONLY FUNCTIONS ============================== #
def import_int_file(self, init):
""" Replaces path settings in imported image object with new settings
NEED TO RE-DO LATER """
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Generate paths to output files
self.params = init.params
self.main_log = init.logfile
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.log_base = init.log_base
self.viz_base = init.viz_base
self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
self.obj_base)
self.obj_file = os.path.abspath(
os.path.join(
self.obj_path,
os.path.basename(self.conv_img).split('.')[0] + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base,
self.fin_base)
self.log_path = misc.make_image_path(self.conv_img, self.input_base,
self.log_base)
self.fin_file = os.path.abspath(
os.path.join(
self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.pickle"))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.viz_path = misc.make_image_path(self.conv_img, self.input_base,
self.viz_base)
self.viz_file = os.path.join(
self.viz_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.png")
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Grid search / integration log file
self.int_log = os.path.join(
self.log_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
# Reset status to 'grid search' to pick up at selection (if no fail)
if self.fail == None:
self.status = 'bypass grid search'
return self
def determine_gs_result_file(self):
""" For 'selection-only' cctbx.xfel runs, determine where the image objects are """
if self.params.cctbx.selection.select_only.grid_search_path != None:
obj_path = os.path.abspath(
self.params.cctbx.selection.select_only.grid_search_path)
else:
run_number = int(os.path.basename(self.int_base)) - 1
obj_path = "{}/integration/{:03d}/image_objects"\
"".format(os.path.abspath(os.curdir), run_number)
gs_result_file = os.path.join(obj_path,
os.path.basename(self.obj_file))
return gs_result_file
# =============================== IMAGE IMPORT FUNCTIONS =============================== #
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except Exception, e:
print 'IOTA IMPORT ERROR:', e
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
img_type = 'pickle'
else:
img_type = 'raw'
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
else:
data = None
img_type = 'not imported'
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
from dxtbx.datablock import DataBlockFactory
from dials.command_line.estimate_gain import estimate_gain
with misc.Capturing() as junk_output:
try:
datablock = DataBlockFactory.from_filenames([self.raw_img
])[0]
imageset = datablock.extract_imagesets()[0]
self.gain = estimate_gain(imageset)
except Exception, e:
self.gain = 1.0
def run(self):
""" Process all images assigned to this thread """
params, options = self.parser.parse_args(show_diff_phil=True)
if params.input.experiment is None or params.input.run_num is None or params.input.address is None:
raise Usage(self.usage)
if params.format.file_format == "cbf":
if params.format.cbf.detz_offset is None:
raise Usage(self.usage)
elif params.format.file_format == "pickle":
if params.format.pickle.cfg is None:
raise Usage(self.usage)
else:
raise Usage(self.usage)
if not os.path.exists(params.output.output_dir):
raise Sorry("Output path not found:" + params.output.output_dir)
# Save the paramters
self.params = params
self.options = options
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
size = comm.Get_size() # size: number of processes running in this job
# set up psana
if params.format.file_format == "pickle":
psana.setConfigFile(params.format.pickle.cfg)
dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment, params.input.run_num)
ds = psana.DataSource(dataset_name)
if params.format.file_format == "cbf":
src = psana.Source("DetInfo(%s)" % params.input.address)
psana_det = psana.Detector(params.input.address, ds.env())
# set this to sys.maxint to analyze all events
if params.dispatch.max_events is None:
max_events = sys.maxint
else:
max_events = params.dispatch.max_events
for run in ds.runs():
if params.format.file_format == "cbf":
# load a header only cspad cbf from the slac metrology
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, params.input.address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" % run.run())
if params.format.cbf.gain_mask_value is not None:
gain_mask = psana_det.gain_mask(gain=params.format.cbf.gain_mask_value)
# list of all events
times = run.times()
nevents = min(len(times), max_events)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in xrange(nevents) if (i + rank) % size == 0]
for i in xrange(len(mytimes)):
evt = run.event(mytimes[i])
id = evt.get(psana.EventId)
print "Event #", i, " has id:", id
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if timestamp is None:
print "No timestamp, skipping shot"
continue
t = timestamp
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]
print "Processing shot", s
if params.format.file_format == "pickle":
if evt.get("skip_event"):
print "Skipping event", id
continue
# the data needs to have already been processed and put into the event by psana
data = evt.get(params.format.pickle.out_key)
if data is None:
print "No data"
continue
# set output paths according to the templates
path = os.path.join(params.output.output_dir, "shot-" + s + ".pickle")
print "Saving", path
easy_pickle.dump(path, data)
elif params.format.file_format == "cbf":
# get numpy array, 32x185x388
data = psana_det.calib(evt) # applies psana's complex run-dependent calibrations
if params.format.cbf.gain_mask_value is not None:
# apply gain mask
data *= gain_mask
distance = cspad_tbx.env_distance(params.input.address, run.env(), params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
continue
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
continue
else:
wavelength = 12398.4187 / self.params.format.cbf.override_energy
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx, data, distance, wavelength, timestamp, params.input.address
)
path = os.path.join(params.output.output_dir, "shot-" + s + ".cbf")
print "Saving", path
# write the file
import pycbf
cspad_img._cbf_handle.write_widefile(
path, pycbf.CBF, pycbf.MIME_HEADERS | pycbf.MSG_DIGEST | pycbf.PAD_4K, 0
)
run.end()
ds.end()
def run(argv=None):
"""Compute mean, standard deviation, and maximum projection images
from a set of images given on the command line.
@param argv Command line argument list
@return @c 0 on successful termination, @c 1 on error, and @c 2
for command line syntax errors
"""
import libtbx.load_env
from libtbx import easy_pickle, option_parser
from scitbx.array_family import flex
from xfel.cxi.cspad_ana import cspad_tbx
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
if argv is None:
argv = sys.argv
command_line = (option_parser.option_parser(
usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
"image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
.option(None, "--average-path", "-a",
type="string",
default=None,
dest="avg_path",
metavar="PATH",
help="Write average image to PATH")
.option(None, "--maximum-path", "-m",
type="string",
default=None,
dest="max_path",
metavar="PATH",
help="Write maximum projection image to PATH")
.option(None, "--stddev-path", "-s",
type="string",
default=None,
dest="stddev_path",
metavar="PATH",
help="Write standard deviation image to PATH")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
).process(args=argv[1:])
# Note that it is not an error to omit the output paths, because
# certain statistics could still be printed, e.g. with the verbose
# option.
paths = command_line.args
if len(paths) == 0:
command_line.parser.print_usage(file=sys.stderr)
return 2
# Loop over all images and accumulate statistics.
nfail = 0
nmemb = 0
for path in paths:
if command_line.options.verbose:
sys.stdout.write("Processing %s...\n" % path)
try:
# Promote the image to double-precision floating point type.
# All real-valued flex arrays have the as_double() function.
d = easy_pickle.load(path)
distance = d['DISTANCE']
img = d['DATA'].as_1d().as_double()
wavelength = d['WAVELENGTH']
time_tuple = reverse_timestamp(d['TIMESTAMP'])
# Warn if the header items across the set of images do not match
# up. Note that discrepancies regarding the image size are
# fatal.
if 'active_areas' in locals():
if (active_areas != d['ACTIVE_AREAS']).count(True) != 0:
sys.stderr.write("Active areas do not match\n")
else:
active_areas = d['ACTIVE_AREAS']
if 'beam_center' in locals():
if beam_center != (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']):
sys.stderr.write("Beam centers do not match\n")
else:
beam_center = (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y'])
if 'detector_address' in locals():
if detector_address != d['DETECTOR_ADDRESS']:
sys.stderr.write("Detector addresses do not match\n")
else:
detector_address = d['DETECTOR_ADDRESS']
if 'saturated_value' in locals():
if saturated_value != d['SATURATED_VALUE']:
sys.stderr.write("Saturated values do not match\n")
else:
saturated_value = d['SATURATED_VALUE']
if 'size' in locals():
if size != (d['SIZE1'], d['SIZE2']):
sys.stderr.write("Image sizes do not match\n")
return 1
else:
size = (d['SIZE1'], d['SIZE2'])
if size != d['DATA'].focus():
sys.stderr.write("Image size does not match pixel array\n")
return 1
if 'pixel_size' in locals():
if pixel_size != d['PIXEL_SIZE']:
sys.stderr.write("Pixel sizes do not match\n")
return 1
else:
if 'PIXEL_SIZE' in d:
pixel_size = d['PIXEL_SIZE']
else:
pixel_size = None
except Exception:
try:
# Fall back on reading the image with dxtbx, and shoehorn the
# extracted information into what would have been found in a
# pickle file. XXX This code assumes a monolithic detector!
from dxtbx.format.Registry import Registry
format_class = Registry.find(path)
i = format_class(path)
beam = i.get_beam()
assert len(i.get_detector()) == 1
detector = i.get_detector()[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = i.get_raw_data().as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
time_tuple = (i.get_scan().get_epochs()[0], 0)
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
except Exception:
nfail += 1
continue
# See also event() in xfel.cxi.cspad_ana.average_tbx. Record the
# base time as the timestamp of the first image.
#
# The sum-of-squares image is accumulated using long integers, as
# this delays the point where overflow occurs. But really, this
# is just a band-aid...
if nmemb == 0:
max_img = img.deep_copy()
sum_distance = distance
sum_img = img.deep_copy()
ssq_img = flex.pow2(img)
sum_wavelength = wavelength
sum_time = (0, 0)
time_base = time_tuple
else:
sel = (img > max_img).as_1d()
max_img.set_selected(sel, img.select(sel))
sum_distance += distance
sum_img += img
ssq_img += flex.pow2(img)
sum_wavelength += wavelength
sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]),
sum_time[1] + (time_tuple[1] - time_base[1]))
nmemb += 1
# Early exit if no statistics were accumulated.
if command_line.options.verbose:
sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
if nmemb == 0:
return 0
# Calculate averages for measures where other statistics do not make
# sense. Note that avg_img is required for stddev_img.
avg_img = sum_img.as_double() / nmemb
avg_distance = sum_distance / nmemb
avg_timestamp = cspad_tbx.evt_timestamp(
(time_base[0] + int(round(sum_time[0] / nmemb)),
time_base[1] + int(round(sum_time[1] / nmemb))))
avg_wavelength = sum_wavelength / nmemb
# Output the average image, maximum projection image, and standard
# deviation image, if requested.
if command_line.options.avg_path is not None:
avg_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=avg_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.avg_path, d)
if command_line.options.max_path is not None:
max_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=max_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.max_path, d)
if command_line.options.stddev_path is not None:
stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img
# Accumulating floating-point numbers introduces errors, which may
# cause negative variances. Since a two-pass approach is
# unacceptable, the standard deviation is clamped at zero.
stddev_img.set_selected(stddev_img < 0, 0)
if nmemb == 1:
stddev_img = flex.sqrt(stddev_img)
else:
stddev_img = flex.sqrt(stddev_img / (nmemb - 1))
stddev_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=stddev_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.stddev_path, d)
return 0
def process_event(self, run, timestamp):
"""
Process a single event from a run
@param run psana run object
@param timestamp psana timestamp object
"""
ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6))
if ts is None:
print "No timestamp, skipping shot"
return
if len(self.params_cache.debug.event_timestamp) > 0 and ts not in self.params_cache.debug.event_timestamp:
return
if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events:
if ts in self.known_events:
if self.known_events[ts] == "unknown":
if self.params_cache.debug.skip_bad_events and self.known_events[ts] == "unknown":
print "Skipping event %s: possibly caused an unknown exception previously"%ts
return
elif self.params_cache.debug.skip_processed_events:
print "Skipping event %s: processed successfully previously"%ts
return
else:
if self.params_cache.debug.skip_unprocessed_events:
print "Skipping event %s: not processed previously"%ts
return
print "Accepted", ts
self.debug_file_handle.write("%s,%s"%(socket.gethostname(), ts))
self.params = copy.deepcopy(self.params_cache)
evt = run.event(timestamp)
id = evt.get(psana.EventId)
if evt.get("skip_event"):
print "Skipping event",id
self.debug_file_handle.write(",psana_skip\n")
return
# the data needs to have already been processed and put into the event by psana
if self.params.format.file_format == 'cbf':
# get numpy array, 32x185x388
data = self.psana_det.calib(evt) # applies psana's complex run-dependent calibrations
if data is None:
print "No data"
self.debug_file_handle.write(",no_data\n")
return
if self.params.format.cbf.gain_mask_value is not None:
# apply gain mask
data *= self.gain_mask
distance = cspad_tbx.env_distance(self.params.input.address, run.env(), self.params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
self.debug_file_handle.write(",no_distance\n")
return
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
self.debug_file_handle.write(",no_wavelength\n")
return
else:
wavelength = 12398.4187/self.params.format.cbf.override_energy
if self.params.format.file_format == 'pickle':
image_dict = evt.get(self.params.format.pickle.out_key)
data = image_dict['DATA']
timestamp = t = ts
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]
print "Processing shot", s
if self.params.format.file_format == 'cbf':
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address)
elif self.params.format.file_format == 'pickle':
from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory
cspad_img = FormatPYunspecifiedStillInMemory(image_dict)
cspad_img.timestamp = s
if self.params.dispatch.dump_all:
self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s))
self.cache_ranges(cspad_img, self.params)
imgset = MemImageSet([cspad_img])
datablock = DataBlockFactory.from_imageset(imgset)[0]
# before calling DIALS for processing, set output paths according to the templates
if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template:
self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.indexed_filename_template%("idx-" + s))
if "%s" in self.refined_experiments_filename_template:
self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.refined_experiments_filename_template%("idx-" + s))
if "%s" in self.integrated_filename_template:
self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.integrated_filename_template%("idx-" + s))
# if border is requested, generate a border only mask
if self.params.border_mask.border > 0:
from dials.command_line.generate_mask import MaskGenerator
generator = MaskGenerator(self.params.border_mask)
mask = generator.generate(imgset)
self.params.spotfinder.lookup.mask = mask
try:
observed = self.find_spots(datablock)
except Exception, e:
import traceback; traceback.print_exc()
print str(e), "event", timestamp
self.debug_file_handle.write(",spotfinding_exception\n")
return
def run(args=None):
dxtbx.util.encode_output_as_utf8()
args = args or sys.argv[1:]
progname = os.getenv("LIBTBX_DISPATCHER_NAME")
if not progname or progname.endswith(".python"):
progname = "%prog"
command_line = (libtbx.option_parser.option_parser(
usage=
f"{progname} [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress",
).option(
None,
"--crop",
"-c",
action="store_true",
default=False,
dest="crop",
help="Crop the image such that the beam center is in the middle",
).option(
None,
"--skip_converted",
"-s",
action="store_true",
default=False,
dest="skip_converted",
help=
"Skip converting if an image already exist that matches the destination file name",
).option(
None,
"--wavelength",
"-w",
type="float",
default=None,
dest="wavelength",
help="Override the image's wavelength (angstroms)",
).option(
None,
"--distance",
"-d",
type="float",
default=None,
dest="distance",
help="Override the detector distance (mm)",
).option(
None,
"--pixel_size",
"-p",
type="float",
default=None,
dest="pixel_size",
help="Override the detector pixel size (mm)",
).option(
None,
"--beam_x",
"-x",
type="float",
default=None,
dest="beam_center_x",
help="Override the beam x position (pixels)",
).option(
None,
"--beam_y",
"-y",
type="float",
default=None,
dest="beam_center_y",
help="Override the beam y position (pixels)",
).option(
None,
"--overload",
"-o",
type="float",
default=None,
dest="overload",
help="Override the detector overload value (ADU)",
)).process(args)
paths = command_line.args
if len(paths) <= 0:
raise Usage("No files specified")
for imgpath in paths:
if command_line.options.verbose:
print("Reading %s" % (imgpath))
try:
img = dxtbx.load(imgpath)
except OSError:
img = None
if img is None:
try:
raw_data = np.loadtxt(imgpath)
raw_data = flex.double(raw_data.astype(np.double))
except ValueError:
raise Usage("Couldn't load %s, no supported readers" % imgpath)
detector = None
beam = None
scan = None
is_multi_image = False
else:
try:
raw_data = img.get_raw_data()
is_multi_image = False
except TypeError:
raw_data = img.get_raw_data(0)
is_multi_image = True
detector = img.get_detector()
beam = img.get_beam()
scan = img.get_scan()
if detector is None:
if command_line.options.distance is None:
raise Usage("Can't get distance from image. Override with -d")
if command_line.options.pixel_size is None:
raise Usage(
"Can't get pixel size from image. Override with -p")
if command_line.options.overload is None:
raise Usage(
"Can't get overload value from image. Override with -o")
distance = command_line.options.distance
pixel_size = command_line.options.pixel_size
overload = command_line.options.overload
else:
detector = detector[0]
if command_line.options.distance is None:
distance = detector.get_distance()
else:
distance = command_line.options.distance
if command_line.options.pixel_size is None:
pixel_size = detector.get_pixel_size()[0]
else:
pixel_size = command_line.options.pixel_size
if command_line.options.overload is None:
overload = detector.get_trusted_range()[1]
else:
overload = command_line.options.overload
if beam is None:
if command_line.options.wavelength is None:
raise Usage(
"Can't get wavelength from image. Override with -w")
wavelength = command_line.options.wavelength
else:
if command_line.options.wavelength is None:
wavelength = beam.get_wavelength()
else:
wavelength = command_line.options.wavelength
if beam is None and detector is None:
if command_line.options.beam_center_x is None:
print(
"Can't get beam x position from image. Using image center. Override with -x"
)
beam_x = raw_data.focus()[0] * pixel_size
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
print(
"Can't get beam y position from image. Using image center. Override with -y"
)
beam_y = raw_data.focus()[1] * pixel_size
else:
beam_y = command_line.options.beam_center_y * pixel_size
else:
if command_line.options.beam_center_x is None:
beam_x = detector.get_beam_centre(beam.get_s0())[0]
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
beam_y = detector.get_beam_centre(beam.get_s0())[1]
else:
beam_y = command_line.options.beam_center_y * pixel_size
if scan is None:
timestamp = None
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
if is_multi_image:
for i in range(img.get_num_images()):
save_image(
command_line,
imgpath,
scan,
img.get_raw_data(i),
distance,
pixel_size,
wavelength,
beam_x,
beam_y,
overload,
timestamp,
image_number=i,
)
else:
save_image(
command_line,
imgpath,
scan,
raw_data,
distance,
pixel_size,
wavelength,
beam_x,
beam_y,
overload,
timestamp,
)
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except Exception as e:
print 'IOTA IMPORT ERROR:', e
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
img_type = 'pickle'
else:
img_type = 'raw'
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec,msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp
)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
else:
data = None
img_type = 'not imported'
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
from dxtbx.datablock import DataBlockFactory
from dials.command_line.estimate_gain import estimate_gain
with misc.Capturing() as junk_output:
try:
datablock = DataBlockFactory.from_filenames([self.raw_img])[0]
imageset = datablock.extract_imagesets()[0]
self.gain = estimate_gain(imageset)
except Exception as e:
self.gain = 1.0
else:
self.gain = 1.0
return data, img_type
def beginjob(self, evt, env):
"""The beginjob() function does one-time initialisation from
event- or environment data. It is called at an XTC configure
transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).beginjob(evt, env)
# Load the dark image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing. If a dark image
# is provided, a standard deviation image is required, and all the
# ADU scales must match up.
#
# XXX Can we zap all the ADU_SCALE stuff?
#
# XXX Do we really need to store the substituted values in the
# instance here? At least self._dark_path is referenced later on.
#
# Note that this will load the dark, standard deviation and gain
# once (SEVERAL TIMES) for each process, but the gain is that we
# can do substitutions. But it is only done at the beginning of
# the job.
self.dark_img = None
if self._dark_path is not None:
self._dark_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_path, evt, env))
assert self._dark_stddev_path is not None
dark_dict = easy_pickle.load(self._dark_path)
#assert "ADU_SCALE" not in dark_dict # force use of recalculated dark
self.dark_img = dark_dict['DATA']
assert isinstance(self.dark_img, flex.double)
self._dark_stddev_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_stddev_path, evt, env))
self.dark_stddev = easy_pickle.load(self._dark_stddev_path)['DATA']
assert isinstance(self.dark_stddev, flex.double)
self.dark_mask = (self.dark_stddev > 0)
# Load the mask image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing.
self.gain_map = None
if self._gain_map_path is not None:
self._gain_map_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._gain_map_path, evt, env))
self.gain_map = easy_pickle.load(self._gain_map_path)['DATA']
if self.gain_map_level is not None:
sel = flex.bool([bool(f) for f in self.gain_map])
sel.reshape(flex.grid(self.gain_map.focus()))
self.gain_map = self.gain_map.set_selected(~sel, self.gain_map_level)
self.gain_map = self.gain_map.set_selected(sel, 1)
assert isinstance(self.gain_map, flex.double)
self.mask_img = None
if self._mask_path is not None:
self._mask_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._mask_path, evt, env))
self.mask_img = easy_pickle.load(self._mask_path)['DATA']
assert isinstance(self.mask_img, flex.double) \
or isinstance(self.mask_img, flex.int)
if self.address == 'XppGon-0|marccd-0':
#mod_mar.py will set these during its event function
self.active_areas = None
self.beam_center = None
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
assert self.override_beam_x is not None
assert self.override_beam_y is not None
from xfel.cxi.cspad_ana import rayonix_tbx
maxx, maxy = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
if self.crop_rayonix:
bx = int(round(self.override_beam_x))
by = int(round(self.override_beam_y))
minsize = min([bx,by,maxx-bx,maxy-by])
self.beam_center = minsize,minsize
self.active_areas = flex.int([0,0,2*minsize,2*minsize])
self.rayonix_crop_slice = slice(by-minsize,by+minsize), slice(bx-minsize,bx+minsize)
else:
self.beam_center = self.override_beam_x,self.override_beam_y
self.active_areas = flex.int([0,0,maxx,maxy])
elif self.address == 'XppGon-0|Cspad-0':
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = cspad_tbx.cbcaa(
cspad_tbx.getConfig(self.address, env), self.sections)
class InMemScript(DialsProcessScript):
""" Script to process XFEL data at LCLS """
def __init__(self):
""" Set up the option parser. Arguments come from the command line or a phil file """
self.usage = """
%s input.experiment=experimentname input.run_num=N input.address=address
format.file_format=cbf format.cbf.detz_offset=N
%s input.experiment=experimentname input.run_num=N input.address=address
format.file_format=pickle input.cfg=filename
""" % (libtbx.env.dispatcher_name, libtbx.env.dispatcher_name)
self.parser = OptionParser(usage=self.usage, phil=phil_scope)
self.debug_file_path = None
self.debug_str = None
self.mpi_log_file_path = None
self.reference_detector = None
def debug_start(self, ts):
self.debug_str = "%s,%s" % (socket.gethostname(), ts)
self.debug_str += ",%s,%s,%s\n"
self.debug_write("start")
def debug_write(self, string, state=None):
ts = cspad_tbx.evt_timestamp() # Now
debug_file_handle = open(self.debug_file_path, 'a')
if string == "":
debug_file_handle.write("\n")
else:
if state is None:
state = " "
debug_file_handle.write(self.debug_str % (ts, state, string))
debug_file_handle.close()
def mpi_log_write(self, string):
mpi_log_file_handle = open(self.mpi_log_file_path, 'a')
mpi_log_file_handle.write(string)
mpi_log_file_handle.close()
def psana_mask_to_dials_mask(self, psana_mask):
if psana_mask.dtype == np.bool:
psana_mask = flex.bool(psana_mask)
else:
psana_mask = flex.bool(psana_mask == 1)
assert psana_mask.focus() == (32, 185, 388)
dials_mask = []
for i in xrange(32):
dials_mask.append(psana_mask[i:i + 1, :, :194])
dials_mask[-1].reshape(flex.grid(185, 194))
dials_mask.append(psana_mask[i:i + 1, :, 194:])
dials_mask[-1].reshape(flex.grid(185, 194))
return dials_mask
def run(self):
""" Process all images assigned to this thread """
params, options = self.parser.parse_args(show_diff_phil=True)
# Check inputs
if params.input.experiment is None or \
params.input.run_num is None or \
params.input.address is None:
raise Usage(self.usage)
if params.format.file_format == "cbf":
if params.format.cbf.detz_offset is None:
raise Usage(self.usage)
elif params.format.file_format == "pickle":
if params.input.cfg is None:
raise Usage(self.usage)
else:
raise Usage(self.usage)
if not os.path.exists(params.output.output_dir):
raise Sorry("Output path not found:" + params.output.output_dir)
self.params = params
self.load_reference_geometry()
# The convention is to put %s in the phil parameter to add a time stamp to
# each output datafile. Save the initial templates here.
self.strong_filename_template = params.output.strong_filename
self.indexed_filename_template = params.output.indexed_filename
self.refined_experiments_filename_template = params.output.refined_experiments_filename
self.integrated_filename_template = params.output.integrated_filename
self.reindexedstrong_filename_template = params.output.reindexedstrong_filename
# Don't allow the strong reflections to be written unless there are enough to
# process
params.output.strong_filename = None
# Save the paramters
self.params_cache = copy.deepcopy(params)
self.options = options
if params.mp.method == "mpi":
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank(
) # each process in MPI has a unique id, 0-indexed
size = comm.Get_size(
) # size: number of processes running in this job
elif params.mp.method == "sge" and \
'SGE_TASK_ID' in os.environ and \
'SGE_TASK_FIRST' in os.environ and \
'SGE_TASK_LAST' in os.environ:
if 'SGE_STEP_SIZE' in os.environ:
assert int(os.environ['SGE_STEP_SIZE']) == 1
if os.environ['SGE_TASK_ID'] == 'undefined' or os.environ[
'SGE_TASK_ID'] == 'undefined' or os.environ[
'SGE_TASK_ID'] == 'undefined':
rank = 0
size = 1
else:
rank = int(os.environ['SGE_TASK_ID']) - int(
os.environ['SGE_TASK_FIRST'])
size = int(os.environ['SGE_TASK_LAST']) - int(
os.environ['SGE_TASK_FIRST']) + 1
else:
rank = 0
size = 1
# Configure the logging
if params.output.logging_dir is None:
info_path = ''
debug_path = ''
else:
log_path = os.path.join(params.output.logging_dir,
"log_rank%04d.out" % rank)
error_path = os.path.join(params.output.logging_dir,
"error_rank%04d.out" % rank)
print "Redirecting stdout to %s" % log_path
print "Redirecting stderr to %s" % error_path
sys.stdout = open(log_path, 'a', buffering=0)
sys.stderr = open(error_path, 'a', buffering=0)
print "Should be redirected now"
info_path = os.path.join(params.output.logging_dir,
"info_rank%04d.out" % rank)
debug_path = os.path.join(params.output.logging_dir,
"debug_rank%04d.out" % rank)
from dials.util import log
log.config(params.verbosity, info=info_path, debug=debug_path)
debug_dir = os.path.join(params.output.output_dir, "debug")
if not os.path.exists(debug_dir):
try:
os.makedirs(debug_dir)
except OSError, e:
pass # due to multiprocessing, makedirs can sometimes fail
assert os.path.exists(debug_dir)
if params.debug.skip_processed_events or params.debug.skip_unprocessed_events or params.debug.skip_bad_events:
print "Reading debug files..."
self.known_events = {}
for filename in os.listdir(debug_dir):
# format: hostname,timestamp_event,timestamp_now,status,detail
for line in open(os.path.join(debug_dir, filename)):
vals = line.strip().split(',')
if len(vals) != 5:
continue
_, ts, _, status, detail = vals
if status in ["done", "stop", "fail"]:
self.known_events[ts] = status
else:
self.known_events[ts] = "unknown"
self.debug_file_path = os.path.join(debug_dir, "debug_%d.txt" % rank)
write_newline = os.path.exists(self.debug_file_path)
if write_newline: # needed if the there was a crash
self.debug_write("")
if params.mp.method != 'mpi' or params.mp.mpi.method == 'client_server':
if rank == 0:
self.mpi_log_file_path = os.path.join(debug_dir, "mpilog.out")
write_newline = os.path.exists(self.mpi_log_file_path)
if write_newline: # needed if the there was a crash
self.mpi_log_write("\n")
# set up psana
if params.input.cfg is not None:
psana.setConfigFile(params.input.cfg)
dataset_name = "exp=%s:run=%s:idx" % (params.input.experiment,
params.input.run_num)
if params.input.xtc_dir is not None:
if params.input.use_ffb:
raise Sorry(
"Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % params.input.xtc_dir
elif params.input.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
params.input.experiment[0:3], params.input.experiment)
if params.input.stream is not None:
dataset_name += ":stream=%d" % params.input.stream
ds = psana.DataSource(dataset_name)
if params.format.file_format == "cbf":
self.psana_det = psana.Detector(params.input.address, ds.env())
# set this to sys.maxint to analyze all events
if params.dispatch.max_events is None:
max_events = sys.maxint
else:
max_events = params.dispatch.max_events
for run in ds.runs():
if params.format.file_format == "cbf":
# load a header only cspad cbf from the slac metrology
try:
self.base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, params.input.address)
except Exception, e:
raise Sorry(
"Couldn't load calibration file for run %d, %s" %
(run.run(), str(e)))
if self.base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
if params.format.file_format == 'cbf':
if params.format.cbf.common_mode.algorithm == "custom":
self.common_mode = params.format.cbf.common_mode.custom_parameterization
assert self.common_mode is not None
else:
self.common_mode = params.format.cbf.common_mode.algorithm # could be None or default
if params.format.cbf.invalid_pixel_mask is not None:
self.dials_mask = easy_pickle.load(
params.format.cbf.invalid_pixel_mask)
assert len(self.dials_mask) == 64
if self.params.format.cbf.mask_nonbonded_pixels:
psana_mask = self.psana_det.mask(run,
calib=False,
status=False,
edges=False,
central=False,
unbond=True,
unbondnbrs=True)
dials_mask = self.psana_mask_to_dials_mask(psana_mask)
self.dials_mask = [
self.dials_mask[i] & dials_mask[i]
for i in xrange(len(dials_mask))
]
else:
psana_mask = self.psana_det.mask(run,
calib=True,
status=True,
edges=True,
central=True,
unbond=True,
unbondnbrs=True)
self.dials_mask = self.psana_mask_to_dials_mask(psana_mask)
if self.params.spotfinder.lookup.mask is not None:
self.spotfinder_mask = easy_pickle.load(
self.params.spotfinder.lookup.mask)
else:
self.spotfinder_mask = None
if self.params.integration.lookup.mask is not None:
self.integration_mask = easy_pickle.load(
self.params.integration.lookup.mask)
else:
self.integration_mask = None
# list of all events
times = run.times()
nevents = min(len(times), max_events)
times = times[:nevents]
if params.dispatch.process_percent is not None:
import fractions
percent = params.dispatch.process_percent / 100
f = fractions.Fraction(percent).limit_denominator(100)
times = [
times[i] for i in xrange(len(times))
if i % f.denominator < f.numerator
]
print "Dividing %d of %d events (%4.1f%%) between all processes" % (
len(times), nevents, 100 * len(times) / nevents)
nevents = len(times)
else:
print "Dividing %d events between all processes" % nevents
if params.mp.method == "mpi" and params.mp.mpi.method == 'client_server' and size > 2:
print "Using MPI client server"
# use a client/server approach to be sure every process is busy as much as possible
# only do this if there are more than 2 processes, as one process will be a server
try:
if rank == 0:
# server process
self.mpi_log_write("MPI START\n")
for t in times:
# a client process will indicate it's ready by sending its rank
self.mpi_log_write(
"Getting next available process\n")
rankreq = comm.recv(source=MPI.ANY_SOURCE)
ts = cspad_tbx.evt_timestamp(
(t.seconds(), t.nanoseconds() / 1e6))
self.mpi_log_write(
"Process %s is ready, sending ts %s\n" %
(rankreq, ts))
comm.send(t, dest=rankreq)
# send a stop command to each process
self.mpi_log_write("MPI DONE, sending stops\n")
for rankreq in range(size - 1):
self.mpi_log_write(
"Getting next available process\n")
rankreq = comm.recv(source=MPI.ANY_SOURCE)
self.mpi_log_write("Sending stop to %d\n" %
rankreq)
comm.send('endrun', dest=rankreq)
else:
# client process
while True:
# inform the server this process is ready for an event
comm.send(rank, dest=0)
evttime = comm.recv(source=0)
if evttime == 'endrun': break
self.process_event(run, evttime)
except Exception, e:
print "Error caught in main loop"
print str(e)
def run(argv=None):
if argv is None:
argv = sys.argv[1:]
command_line = (
libtbx.option_parser.option_parser(
usage="%s [-v] [-c] [-s] [-w wavelength] [-d distance] [-p pixel_size] [-x beam_x] [-y beam_y] [-o overload] files"
% libtbx.env.dispatcher_name
)
.option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress",
)
.option(
None,
"--crop",
"-c",
action="store_true",
default=False,
dest="crop",
help="Crop the image such that the beam center is in the middle",
)
.option(
None,
"--skip_converted",
"-s",
action="store_true",
default=False,
dest="skip_converted",
help="Skip converting if an image already exist that matches the destination file name",
)
.option(
None,
"--wavelength",
"-w",
type="float",
default=None,
dest="wavelength",
help="Override the image's wavelength (angstroms)",
)
.option(
None,
"--distance",
"-d",
type="float",
default=None,
dest="distance",
help="Override the detector distance (mm)",
)
.option(
None,
"--pixel_size",
"-p",
type="float",
default=None,
dest="pixel_size",
help="Override the detector pixel size (mm)",
)
.option(
None,
"--beam_x",
"-x",
type="float",
default=None,
dest="beam_center_x",
help="Override the beam x position (pixels)",
)
.option(
None,
"--beam_y",
"-y",
type="float",
default=None,
dest="beam_center_y",
help="Override the beam y position (pixels)",
)
.option(
None,
"--overload",
"-o",
type="float",
default=None,
dest="overload",
help="Override the detector overload value (ADU)",
)
).process(args=argv)
paths = command_line.args
if len(paths) <= 0:
raise Usage("No files specified")
for imgpath in paths:
destpath = os.path.join(os.path.dirname(imgpath), os.path.splitext(os.path.basename(imgpath))[0] + ".pickle")
if command_line.options.skip_converted and os.path.isfile(destpath):
if command_line.options.verbose:
print "Skipping %s, file exists" % imgpath
continue
if command_line.options.verbose:
print "Converting %s to %s..." % (imgpath, destpath)
try:
img = dxtbx.load(imgpath)
except IOError:
img = None
pass
if img is None:
import numpy as np
try:
raw_data = np.loadtxt(imgpath)
from scitbx.array_family import flex
raw_data = flex.double(raw_data.astype(np.double))
except ValueError:
raise Usage("Couldn't load %s, no supported readers" % imgpath)
detector = None
beam = None
scan = None
else:
raw_data = img.get_raw_data()
detector = img.get_detector()
beam = img.get_beam()
scan = img.get_scan()
if detector is None:
if command_line.options.distance is None:
raise Usage("Can't get distance from image. Override with -d")
if command_line.options.pixel_size is None:
raise Usage("Can't get pixel size from image. Override with -p")
if command_line.options.overload is None:
raise Usage("Can't get overload value from image. Override with -o")
distance = command_line.options.distance
pixel_size = command_line.options.pixel_size
overload = command_line.options.overload
else:
detector = detector[0]
if command_line.options.distance is None:
distance = detector.get_distance()
else:
distance = command_line.options.distance
if command_line.options.pixel_size is None:
pixel_size = detector.get_pixel_size()[0]
else:
pixel_size = command_line.options.pixel_size
if command_line.options.overload is None:
overload = detector.get_trusted_range()[1]
else:
overload = command_line.options.overload
if beam is None:
if command_line.options.wavelength is None:
raise Usage("Can't get wavelength from image. Override with -w")
wavelength = command_line.options.wavelength
else:
if command_line.options.wavelength is None:
wavelength = beam.get_wavelength()
else:
wavelength = command_line.options.wavelength
if beam is None and detector is None:
if command_line.options.beam_center_x is None:
print "Can't get beam x position from image. Using image center. Override with -x"
beam_x = raw_data.focus()[0] * pixel_size
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
print "Can't get beam y position from image. Using image center. Override with -y"
beam_y = raw_data.focus()[1] * pixel_size
else:
beam_y = command_line.options.beam_center_y * pixel_size
else:
if command_line.options.beam_center_x is None:
beam_x = detector.get_beam_centre(beam.get_s0())[0]
else:
beam_x = command_line.options.beam_center_x * pixel_size
if command_line.options.beam_center_y is None:
beam_y = detector.get_beam_centre(beam.get_s0())[1]
else:
beam_y = command_line.options.beam_center_y * pixel_size
if scan is None:
timestamp = None
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
data = dpack(
data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp,
)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data["OSC_START"] = osc_start
data["OSC_RANGE"] = osc_range
data["TIME"] = scan.get_exposure_times()[0]
if command_line.options.crop:
data = crop_image_pickle(data)
easy_pickle.dump(destpath, data)
def process_event(self, run, timestamp):
"""
Process a single event from a run
@param run psana run object
@param timestamp psana timestamp object
"""
ts = cspad_tbx.evt_timestamp(
(timestamp.seconds(), timestamp.nanoseconds() / 1e6))
if ts is None:
print "No timestamp, skipping shot"
return
if len(self.params_cache.debug.event_timestamp
) > 0 and ts not in self.params_cache.debug.event_timestamp:
return
if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events:
if ts in self.known_events:
if self.known_events[ts] not in ["stop", "done", "fail"]:
if self.params_cache.debug.skip_bad_events:
print "Skipping event %s: possibly caused an unknown exception previously" % ts
return
elif self.params_cache.debug.skip_processed_events:
print "Skipping event %s: processed successfully previously" % ts
return
else:
if self.params_cache.debug.skip_unprocessed_events:
print "Skipping event %s: not processed previously" % ts
return
self.debug_start(ts)
evt = run.event(timestamp)
if evt.get("skip_event") or "skip_event" in [
key.key() for key in evt.keys()
]:
print "Skipping event", ts
self.debug_write("psana_skip", "skip")
return
print "Accepted", ts
self.params = copy.deepcopy(self.params_cache)
# the data needs to have already been processed and put into the event by psana
if self.params.format.file_format == 'cbf':
# get numpy array, 32x185x388
data = cspad_cbf_tbx.get_psana_corrected_data(
self.psana_det,
evt,
use_default=False,
dark=True,
common_mode=self.common_mode,
apply_gain_mask=self.params.format.cbf.gain_mask_value
is not None,
gain_mask_value=self.params.format.cbf.gain_mask_value,
per_pixel_gain=False)
if data is None:
print "No data"
self.debug_write("no_data", "skip")
return
if self.params.format.cbf.override_distance is None:
distance = cspad_tbx.env_distance(
self.params.input.address, run.env(),
self.params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
self.debug_write("no_distance", "skip")
return
else:
distance = self.params.format.cbf.override_distance
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
self.debug_write("no_wavelength", "skip")
return
else:
wavelength = 12398.4187 / self.params.format.cbf.override_energy
if self.params.format.file_format == 'pickle':
image_dict = evt.get(self.params.format.pickle.out_key)
data = image_dict['DATA']
timestamp = t = ts
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
20:23]
print "Processing shot", s
if self.params.format.file_format == 'cbf':
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(
self.base_dxtbx, data, distance, wavelength, timestamp,
self.params.input.address)
if self.params.input.reference_geometry is not None:
from dxtbx.model import Detector
# copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead.
cspad_img._detector_instance = Detector.from_dict(
self.reference_detector.to_dict())
cspad_img.sync_detector_to_cbf()
elif self.params.format.file_format == 'pickle':
from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory
cspad_img = FormatPYunspecifiedStillInMemory(image_dict)
cspad_img.timestamp = s
if self.params.dispatch.dump_all:
self.save_image(
cspad_img, self.params,
os.path.join(self.params.output.output_dir, "shot-" + s))
self.cache_ranges(cspad_img, self.params)
imgset = MemImageSet([cspad_img])
if self.params.dispatch.estimate_gain_only:
from dials.command_line.estimate_gain import estimate_gain
estimate_gain(imgset)
return
if not self.params.dispatch.find_spots:
self.debug_write("data_loaded", "done")
return
datablock = DataBlockFactory.from_imageset(imgset)[0]
# before calling DIALS for processing, set output paths according to the templates
if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template:
self.params.output.indexed_filename = os.path.join(
self.params.output.output_dir,
self.indexed_filename_template % ("idx-" + s))
if "%s" in self.refined_experiments_filename_template:
self.params.output.refined_experiments_filename = os.path.join(
self.params.output.output_dir,
self.refined_experiments_filename_template % ("idx-" + s))
if "%s" in self.integrated_filename_template:
self.params.output.integrated_filename = os.path.join(
self.params.output.output_dir,
self.integrated_filename_template % ("idx-" + s))
if "%s" in self.reindexedstrong_filename_template:
self.params.output.reindexedstrong_filename = os.path.join(
self.params.output.output_dir,
self.reindexedstrong_filename_template % ("idx-" + s))
# Load a dials mask from the trusted range and psana mask
from dials.util.masking import MaskGenerator
generator = MaskGenerator(self.params.border_mask)
mask = generator.generate(imgset)
if self.params.format.file_format == "cbf":
mask = tuple([a & b for a, b in zip(mask, self.dials_mask)])
if self.spotfinder_mask is None:
self.params.spotfinder.lookup.mask = mask
else:
self.params.spotfinder.lookup.mask = tuple(
[a & b for a, b in zip(mask, self.spotfinder_mask)])
if self.integration_mask is None:
self.params.integration.lookup.mask = mask
else:
self.params.integration.lookup.mask = tuple(
[a & b for a, b in zip(mask, self.integration_mask)])
self.debug_write("spotfind_start")
try:
observed = self.find_spots(datablock)
except Exception, e:
import traceback
traceback.print_exc()
print str(e), "event", timestamp
self.debug_write("spotfinding_exception", "fail")
return
def endjob(self, obj1, obj2=None):
"""The endjob() function writes the mean and standard deviation images
to disk.
@param evt Event object (psana only)
@param env Environment object
"""
if obj2 is None:
env = obj1
else:
evt = obj1
env = obj2
stats = super(mod_average, self).endjob(env)
if stats is None:
return
device = cspad_tbx.address_split(self.address)[2]
if device == 'Andor':
beam_center = (0, 0) # XXX Fiction!
pixel_size = 13.5e-3 # XXX Should not be hardcoded here!
saturated_value = 10000
elif device == 'Cspad' or device == 'Cspad2x2':
beam_center = self.beam_center
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
elif device == 'marccd':
beam_center = tuple(t // 2 for t in d['mean_img'].focus())
pixel_size = 0.079346
saturated_value = 2**16 - 1
if stats['nmemb'] > 0:
if self.avg_dirname is not None or \
self.avg_basename is not None or \
self._mean_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['mean_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._mean_out is not None:
p = cspad_tbx.dwritef2(d, self._mean_out)
else:
p = cspad_tbx.dwritef(d, self.avg_dirname,
self.avg_basename)
self.logger.info("Average written to %s" % p)
if self.stddev_dirname is not None or \
self.stddev_basename is not None or \
self._std_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['std_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._std_out is not None:
p = cspad_tbx.dwritef2(d, self._std_out)
else:
p = cspad_tbx.dwritef(d, self.stddev_dirname,
self.stddev_basename)
self.logger.info("Standard deviation written to %s" % p)
if self.max_dirname is not None or \
self.max_basename is not None or \
self._max_out is not None:
d = cspad_tbx.dpack(active_areas=self.active_areas,
address=self.address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=stats['max_img'],
distance=stats['distance'],
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=cspad_tbx.evt_timestamp(
stats['time']),
wavelength=stats['wavelength'])
if self._max_out is not None:
p = cspad_tbx.dwritef2(d, self._max_out)
else:
p = cspad_tbx.dwritef(d, self.max_dirname,
self.max_basename)
self.logger.info("Max written to %s" % p)
if stats['nfail'] == 0:
self.logger.info("%d images processed" % stats['nmemb'])
else:
self.logger.warning("%d images processed, %d failed" %
(stats['nmemb'], stats['nfail']))
def event(self,evt,evn):
"""The event() function puts a "skip_event" object with value @c
True into the event if the shot is to be skipped.
@param evt Event data object, a configure object
@param env Environment object
"""
#import pdb; pdb.set_trace()
if (evt.get("skip_event")):
return
# check if FEE data is one or two dimensional
data = evt.get(Camera.FrameV1, self.src)
if data is None:
one_D = True
data = evt.get(Bld.BldDataSpectrometerV1, self.src)
else:
one_D = False
# get event timestamp
timestamp = cspad_tbx.evt_timestamp(cspad_tbx.evt_time(evt)) # human readable format
if data is None:
self.nnodata +=1
#self.logger.warning("event(): No spectrum data")
evt.put(skip_event_flag(),"skip_event")
if timestamp is None:
evt.put(skip_event_flag(),"skip_event")
#self.logger.warning("event(): No TIMESTAMP, skipping shot")
elif data is not None:
self.nshots +=1
# get data as array and split into two half to find each peak
if one_D:
# filtering out outlier spikes in FEE data
data = np.array(data.hproj().astype(np.float64))
for i in xrange(len(data)):
if data[i]>1000000000:
data[i]=data[i]-(2**32)
if self.dark is not None:
data = data - self.dark
spectrum = data
spectrum1 = data[:data.shape[0]//2]
spectrum2 = data[data.shape[0]//2:]
else:
data = np.array(data.data16().astype(np.int32))
if self.dark is not None:
data = data - self.dark
data_split1 = data[:,:data.shape[1]//2]
data_split2 = data[:,data.shape[1]//2:]
# make a 1D trace of entire spectrum and each half to find peaks
spectrum = np.sum(data,0)/data.shape[0]
spectrum1 = np.sum(data_split1,0)/data_split1.shape[0]
spectrum2 = np.sum(data_split2,0)/data_split2.shape[0]
if not one_D:
# the x-coordinate of the weighted center of peak region
weighted_peak_one_positions = []
for i in xrange(self.peak_one_range_min,self.peak_one_range_max):
weighted_peak_one_positions.append(spectrum[i]*i)
weighted_sum_peak_one = sum(weighted_peak_one_positions)
weighted_peak_one_center_position = weighted_sum_peak_one//sum(spectrum[self.peak_one_range_min:self.peak_one_range_max])
weighted_peak_two_positions = []
for i in xrange(self.peak_two_range_min,self.peak_two_range_max):
weighted_peak_two_positions.append(spectrum[i]*i)
weighted_sum_peak_two = sum(weighted_peak_two_positions)
weighted_peak_two_center_position = weighted_sum_peak_two//sum(spectrum[self.peak_two_range_min:self.peak_two_range_max])
# normalized integrated regions between the peaks
int_left_region_norm = np.sum(spectrum[0:(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])/len(spectrum[0:(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_middle_region_norm = np.sum(spectrum[(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
int_right_region_norm = np.sum(spectrum[(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):])/len(spectrum[(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):])
# normalized integrated peaks
int_peak_one_norm = np.sum(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])/len(spectrum[(weighted_peak_one_center_position-len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2):(weighted_peak_one_center_position+len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)])
int_peak_two_norm = np.sum(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])/len(spectrum[(weighted_peak_two_center_position-len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2):(weighted_peak_two_center_position+len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)])
else:
# convert eV range of iron edge (7112 eV) to pixels:
metal_edge_max=(self.forbidden_range_eV/2.)/0.059+738
metal_edge_min=(-self.forbidden_range_eV/2.)/0.059+738
int_metal_region = np.sum(spectrum[metal_edge_min:metal_edge_max])
#peak one region integrate:
int_peak_one=np.sum(spectrum[0:metal_edge_min])
int_peak_two=np.sum(spectrum[metal_edge_max:])
# now to do the filtering
if not one_D:
if min(int_peak_one_norm,int_peak_two_norm)/max(int_peak_one_norm,int_peak_two_norm) < self.peak_ratio:
print "event(): too low"
evt.put(skip_event_flag(), "skip_event")
return
if (np.argmax(spectrum2)+len(spectrum2)) > (weighted_peak_two_center_position+(len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)) or (np.argmax(spectrum2)+len(spectrum2)) < (weighted_peak_two_center_position-(len(spectrum[self.peak_two_range_min:self.peak_two_range_max])/2)):
print "event(): out of range high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if np.argmax(spectrum1) > (weighted_peak_one_center_position+(len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)) or np.argmax(spectrum1) < (weighted_peak_one_center_position-(len(spectrum[self.peak_one_range_min:self.peak_one_range_max])/2)):
print "event(): out of range low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if int_left_region_norm/int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print "event(): noisy left of low energy peak"
evt.put(skip_event_flag(), "skip_event")
return
if int_middle_region_norm/int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if int_middle_region_norm/int_peak_one_norm > self.normalized_peak_to_noise_ratio:
print "event(): noisy middle"
evt.put(skip_event_flag(), "skip_event")
return
if int_right_region_norm/int_peak_two_norm > self.normalized_peak_to_noise_ratio:
print "event(): noisy right of high energy peak"
evt.put(skip_event_flag(), "skip_event")
return
else:
# filter for cxih8015
#iron edge at 738 pixels on FFE detetor
if int_metal_region>=0.10*int_peak_two:
print "event(): high intensity at metal edge"
evt.put(skip_event_flag(), "skip_event")
return
if int_metal_region>=0.10*int_peak_one:
print "event(): high intensity at metal edge"
evt.put(skip_event_flag(), "skip_event")
return
if min(int_peak_one,int_peak_two)/max(int_peak_one,int_peak_two) < self.peak_ratio:
print "event(): peak ratio too low"
evt.put(skip_event_flag(), "skip_event")
return
#self.logger.info("TIMESTAMP %s accepted" %timestamp)
self.naccepted += 1
self.ntwo_color += 1
print "%d Two Color shots" %self.ntwo_color
