def _start(self):
import os
if isinstance(self._image_file, basestring) and os.path.isfile(
self._image_file):
stream = FormatPYunspecified.open_file(self._image_file, 'rb')
import cPickle as pickle
data = pickle.load(stream)
else:
data = self._image_file
if not "DETECTOR_ADDRESS" in data:
# legacy format; try to guess the address
self.LCLS_detector_address = 'CxiDs1-0|Cspad-0'
if "DISTANCE" in data and data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
self.LCLS_detector_address = 'CxiDsd-0|Cspad-0'
else:
self.LCLS_detector_address = data["DETECTOR_ADDRESS"]
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
self._timesec = reverse_timestamp(data["TIMESTAMP"])[0]
from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function
version_lookup = detector_format_function(self.LCLS_detector_address,
self._timesec)
self.start_helper(version_token="distl.detector_format_version=%s" %
version_lookup)
def get_timing_info(root, rank=0, size=1):
steps_d = {}
for ii, filename in enumerate(os.listdir(root)):
if os.path.splitext(filename)[1] != ".txt": continue
if (ii + rank) % size != 0: continue
current_ts = ""
prev_step = ""
prev_time = None
with open(os.path.join(root, filename)) as logfile:
for line in logfile:
try:
hostname, ts, now, status, step = line.split(',')
except ValueError:
print 'ERROR', line
raise
now_s, now_ms = reverse_timestamp(now)
now = now_s + (1e-3 * now_ms)
#from IPython import embed; embed(); exit()
if step.strip() == 'start':
if prev_time is not None:
#print prev_step.strip(), "took", now - prev_time, "seconds"
steps_d = add_step(prev_step, now - prev_time, steps_d,
ts, filename, step)
else:
#print prev_step.strip(), "took", now - prev_time, "seconds"
steps_d = add_step(prev_step, now - prev_time, steps_d, ts,
filename, step)
current_ts = ts
prev_step = step
prev_time = now
#if 'index' in step:
# print 'FWWWWW',step,filename,ts
#print steps_d.keys()
return steps_d
def _start(self):
if isinstance(self._image_file, six.string_types) and os.path.isfile(
self._image_file):
with FormatPYunspecified.open_file(self._image_file, "rb") as fh:
if six.PY3:
data = pickle.load(fh, encoding="bytes")
data = image_dict_to_unicode(data)
else:
data = pickle.load(fh)
else:
data = self._image_file
if "DETECTOR_ADDRESS" not in data:
# legacy format; try to guess the address
self.LCLS_detector_address = "CxiDs1-0|Cspad-0"
if "DISTANCE" in data and data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
self.LCLS_detector_address = "CxiDsd-0|Cspad-0"
else:
self.LCLS_detector_address = data["DETECTOR_ADDRESS"]
self._timesec = reverse_timestamp(data["TIMESTAMP"])[0]
version_lookup = detector_format_version(self.LCLS_detector_address,
self._timesec)
self.start_helper(version_token="distl.detector_format_version=%s" %
version_lookup)
def distl_filter(self,
address,
cspad_img,
distance,
timestamp,
wavelength):
self.hitfinder_d["DATA"] = cspad_img
self.hitfinder_d["DISTANCE"] = distance
self.hitfinder_d["TIMESTAMP"] = timestamp
self.hitfinder_d["WAVELENGTH"] = wavelength
self.hitfinder_d["DETECTOR_ADDRESS"] = address
args = ["indexing.data=dummy",
"distl.bins.verbose=False",
self.asic_filter,
]
detector_format_version = detector_format_function(
address, reverse_timestamp(timestamp)[0])
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = self.hitfinder_d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
imgdata = info.Files.images[0].linearintdata
active_data = self.get_active_data(info.Files.images[0],horizons_phil)
peak_heights = flex.int( [
imgdata[ spot.max_pxl_x(), spot.max_pxl_y() ]
for spot in info.S.images[info.frames[0]]["spots_total"]
])
outscale = 256
corrected = peak_heights.as_double() * self.correction
outvalue = outscale *(1.0-corrected)
outvalue.set_selected(outvalue<0.0,0.)
outvalue.set_selected(outvalue>=outscale,int(outscale)-1)
outvalue = flex.int(outvalue.as_numpy_array().astype(numpy.int32))
# essentially, select a peak if the peak's ADU value is > 2.5 * the 90-percentile pixel value
#work = display_spots.wrapper_of_callback(info)
#work.display_with_callback(horizons_phil.indexing.data)
return peak_heights,outvalue
def distl_filter(self,
address,
cspad_img,
distance,
timestamp,
wavelength):
self.hitfinder_d["DATA"] = cspad_img
self.hitfinder_d["DISTANCE"] = distance
self.hitfinder_d["TIMESTAMP"] = timestamp
self.hitfinder_d["WAVELENGTH"] = wavelength
self.hitfinder_d["DETECTOR_ADDRESS"] = address
args = ["indexing.data=dummy",
"distl.bins.verbose=False",
self.asic_filter,
]
detector_format_version = detector_format_function(
address, reverse_timestamp(timestamp)[0])
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(self.m_xtal_target, args)
horizons_phil.indexing.data = self.hitfinder_d
from xfel.cxi import display_spots
display_spots.parameters.horizons_phil = horizons_phil
from rstbx.new_horizons.index import pre_indexing_validation,pack_names
pre_indexing_validation(horizons_phil)
imagefile_arguments = pack_names(horizons_phil)
from spotfinder.applications import signal_strength
info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments)
imgdata = info.Files.images[0].linearintdata
active_data = self.get_active_data(info.Files.images[0],horizons_phil)
peak_heights = flex.int( [
imgdata[ spot.max_pxl_x(), spot.max_pxl_y() ]
for spot in info.S.images[info.frames[0]]["spots_total"]
])
outscale = 256
corrected = peak_heights.as_double() * self.correction
outvalue = outscale *(1.0-corrected)
outvalue.set_selected(outvalue<0.0,0.)
outvalue.set_selected(outvalue>=outscale,int(outscale)-1)
outvalue = flex.int(outvalue.as_numpy_array().astype(numpy.int32))
# essentially, select a peak if the peak's ADU value is > 2.5 * the 90-percentile pixel value
#work = display_spots.wrapper_of_callback(info)
#work.display_with_callback(horizons_phil.indexing.data)
return peak_heights,outvalue
def integrate_one_image(data, **kwargs):
from xfel.cxi.display_spots import run_one_index_core
from labelit.dptbx.error import NoAutoIndex
from libtbx.utils import Sorry
from spotfinder.exception import SpotfinderError
from labelit.exception import AutoIndexError
from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
basename = kwargs.get("integration_basename")
if (basename is None):
basename = ""
dirname = kwargs.get("integration_dirname")
if (dirname is None):
dirname = "integration"
if (not os.path.isdir(dirname)):
import errno
try:
os.makedirs(dirname)
except OSError as exc:
if exc.errno==errno.EEXIST: pass
path = os.path.join(dirname, basename \
+ data['TIMESTAMP'] \
+ ("_%05d.pickle" % data['SEQUENCE_NUMBER']))
args = ["indexing.data=dummy",
"beam_search_scope=0.5",
"lepage_max_delta = 3.0",
"spots_pickle = None",
"subgroups_pickle = None",
"refinements_pickle = None",
"rmsd_tolerance = 5.0",
"mosflm_rmsd_tolerance = 5.0",
"indexing.completeness_pickle=%s"%path,
"difflimit_sigma_cutoff=2.0",
#"indexing.open_wx_viewer=True"
]
detector_format_version = detector_format_function(
data['DETECTOR_ADDRESS'], reverse_timestamp(data['TIMESTAMP'])[0])
args += ["distl.detector_format_version=%s" % detector_format_version]
from xfel.phil_preferences import load_cxi_phil
horizons_phil = load_cxi_phil(data["xtal_target"], args)
horizons_phil.indexing.data = data
print "XFEL processing: %s"%path
try:
return run_one_index_core(horizons_phil)
except NoAutoIndex,e:
print "NoAutoIndex", data['TIMESTAMP'], e
info = e.info
def run(params):
counter = 0
reference = None
root=os.path.join(params.input_path, 'out', 'debug')
fig_object = plt.figure()
for filename in os.listdir(root):
if os.path.splitext(filename)[1] != '.txt': continue
if 'debug' not in filename: continue
timepoints = []
rank = int(filename.split('_')[1].split('.')[0])
counter += 1
print (filename)
for line in open(os.path.join(root,filename)):
hostname, psanats, ts, status, result = line.strip().split(',')
if reference is None:
sec, ms = reverse_timestamp(ts)
reference = sec+ms*1e-3
if status in ['stop','done']:
sec, ms = reverse_timestamp(ts)
timepoints.append((sec + ms*1.e-3)-reference)
ok = True
else:
ok = False
plt.plot(timepoints, [rank]*len(timepoints), 'b.')
if not ok:
sec, ms = reverse_timestamp(ts)
plt.plot([(sec+ms*1e-3) - reference], [rank], 'rx')
#if counter > 100: break
for i in range(params.num_nodes):
plt.plot([0,params.wall_time], [i*params.num_cores_per_node-0.5, i*params.num_cores_per_node-0.5], 'r-')
plt.xlabel('Wall time (sec)')
plt.ylabel('MPI Rank Number')
plt.title(params.plot_title)
if params.pickle_plot:
from libtbx.easy_pickle import dump
dump('%s'%params.pickle_filename, fig_object)
if params.show_plot:
plt.show()
def _start(self):
import os
if isinstance(self._image_file, basestring) and os.path.isfile(self._image_file):
stream = FormatPYunspecified.open_file(self._image_file, 'rb')
import cPickle as pickle
data = pickle.load(stream)
else:
data = self._image_file
if not "DETECTOR_ADDRESS" in data:
# legacy format; try to guess the address
self.LCLS_detector_address = 'CxiDs1-0|Cspad-0'
if "DISTANCE" in data and data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
self.LCLS_detector_address = 'CxiDsd-0|Cspad-0'
else:
self.LCLS_detector_address = data["DETECTOR_ADDRESS"]
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
self._timesec = reverse_timestamp( data["TIMESTAMP"] )[0]
from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function
version_lookup = detector_format_function(self.LCLS_detector_address,self._timesec)
self.start_helper(version_token="distl.detector_format_version=%s"%version_lookup)
def get_hits_and_indexing_stats(filenames, debug_root, rank=0):
print('starting hits_and_indexing_stats', rank)
num_of_xray_events = 0
num_of_images_analyzed = 0
hits = []
events_list = []
indexing_time = {}
indexing_time_all = {} # Both for failed and successful indexing trials
current_ts = ""
current_reverse_ts = ""
prev_step = ""
prev_time = None
for filename in filenames:
with open(os.path.join(debug_root, filename)) as logfile:
for line in logfile:
try:
if line == '\n': continue
hostname, ts, now, status, step = line.strip().split(',')
except ValueError as e:
print(line)
raise
now_s, now_ms = reverse_timestamp(now)
now = now_s + (1e-3 * now_ms)
if step.strip() == 'spotfind_start':
num_of_images_analyzed += 1
if step.strip() == 'start':
num_of_xray_events += 1
curr_ts, curr_ms = reverse_timestamp(ts)
current_reverse_ts = curr_ts + (1e-3 * curr_ms)
#if '2018-05-01T14:50Z21.976' == ts:
# print (line + 'THIS MIGHT BE A DUPLICATE\n')
events_list.append(ts)
#if prev_time is not None:
#print (prev_step.strip(), "took", now - prev_time, "seconds")
#add_step(prev_step, now-prev_time)
#else:
#print (prev_step.strip(), "took", now - prev_time, "seconds")
#add_step(prev_step, now-prev_time)
if prev_step.strip() == 'index_start':
indexing_time_all[ts] = now - prev_time
if step.strip() == 'index_start':
hits.append(ts)
if step.strip() == 'refine_start':
indexing_time[ts] = now - prev_time
#print ('DEBUG',prev_step, step, now,prev_time, ts)
current_ts = ts
prev_step = step
prev_time = now
#print steps_d.keys()
#print indexing_time.keys()
#print len(hits)
total_idx_time = 0
idx_cutoff_time_exceeded_event = []
for event in indexing_time.keys():
# print event, indexing_time[event]
total_idx_time += indexing_time[event]
recorded_hits = []
idx_attempt_time = []
idx_successful_time = []
if params.write_out_timings:
fout = open('indexing_timing_' + str(rank) + '.dat', 'w')
fout.write(
'Event Number hits indexed t_indexed t_indexed_attempted \n'
)
for ii, event in enumerate(events_list):
if event in hits:
is_hit = 1
if event not in recorded_hits:
recorded_hits.append(event)
else:
if debug_mode:
print('Duplicate Event ? = ', event)
else:
is_hit = 0
if event in indexing_time:
is_idx = 1
t_idx = indexing_time[event]
idx_successful_time.append(t_idx)
assert event in indexing_time_all, 'Event not present in indexing_time_all'
t_idx2 = indexing_time_all[event]
idx_attempt_time.append(t_idx2)
elif event in indexing_time_all:
is_idx = 0.5
assert event not in indexing_time, 'Event should not be present in indexing_time'
t_idx = 0.0
t_idx2 = indexing_time_all[event]
idx_attempt_time.append(t_idx2)
if t_idx2 > indexing_time_cutoff:
idx_cutoff_time_exceeded_event.append(event)
else:
is_idx = 0
t_idx = 0.0
t_idx2 = 0.0
if write_out_timings:
fout.write('%s %3.1f %3.1f %12.7f %12.7f\n' %
(event, is_hit, is_idx, t_idx, t_idx2))
if write_out_timings:
fout.close()
return (len(hits),
len(idx_successful_time), sum(idx_attempt_time) / 3600.0,
sum(idx_successful_time) / 3600.0, idx_cutoff_time_exceeded_event,
num_of_xray_events, num_of_images_analyzed)
def beginjob(self, evt, env):
"""The beginjob() function does one-time initialisation from
event- or environment data. It is called at an XTC configure
transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).beginjob(evt, env)
# Load the dark image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing. If a dark image
# is provided, a standard deviation image is required, and all the
# ADU scales must match up.
#
# XXX Can we zap all the ADU_SCALE stuff?
#
# XXX Do we really need to store the substituted values in the
# instance here? At least self._dark_path is referenced later on.
#
# Note that this will load the dark, standard deviation and gain
# once (SEVERAL TIMES) for each process, but the gain is that we
# can do substitutions. But it is only done at the beginning of
# the job.
self.dark_img = None
if self._dark_path is not None:
self._dark_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_path, evt, env))
assert self._dark_stddev_path is not None
dark_dict = easy_pickle.load(self._dark_path)
#assert "ADU_SCALE" not in dark_dict # force use of recalculated dark
self.dark_img = dark_dict['DATA']
assert isinstance(self.dark_img, flex.double)
self._dark_stddev_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._dark_stddev_path, evt, env))
self.dark_stddev = easy_pickle.load(self._dark_stddev_path)['DATA']
assert isinstance(self.dark_stddev, flex.double)
self.dark_mask = (self.dark_stddev > 0)
# Load the mask image and ensure it is signed and at least 32 bits
# wide, since it will be used for differencing.
self.gain_map = None
if self._gain_map_path is not None:
self._gain_map_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._gain_map_path, evt, env))
self.gain_map = easy_pickle.load(self._gain_map_path)['DATA']
if self.gain_map_level is not None:
sel = flex.bool([bool(f) for f in self.gain_map])
sel.reshape(flex.grid(self.gain_map.focus()))
self.gain_map = self.gain_map.set_selected(~sel, self.gain_map_level)
self.gain_map = self.gain_map.set_selected(sel, 1)
assert isinstance(self.gain_map, flex.double)
self.mask_img = None
if self._mask_path is not None:
self._mask_path = cspad_tbx.getOptEvalOrString(
cspad_tbx.pathsubst(self._mask_path, evt, env))
self.mask_img = easy_pickle.load(self._mask_path)['DATA']
assert isinstance(self.mask_img, flex.double) \
or isinstance(self.mask_img, flex.int)
if self.address == 'XppGon-0|marccd-0':
#mod_mar.py will set these during its event function
self.active_areas = None
self.beam_center = None
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
assert self.override_beam_x is not None
assert self.override_beam_y is not None
from xfel.cxi.cspad_ana import rayonix_tbx
maxx, maxy = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size)
if self.crop_rayonix:
bx = int(round(self.override_beam_x))
by = int(round(self.override_beam_y))
minsize = min([bx,by,maxx-bx,maxy-by])
self.beam_center = minsize,minsize
self.active_areas = flex.int([0,0,2*minsize,2*minsize])
self.rayonix_crop_slice = slice(by-minsize,by+minsize), slice(bx-minsize,bx+minsize)
else:
self.beam_center = self.override_beam_x,self.override_beam_y
self.active_areas = flex.int([0,0,maxx,maxy])
elif self.address == 'XppGon-0|Cspad-0':
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = cspad_tbx.cbcaa(
cspad_tbx.getConfig(self.address, env), self.sections)
def __call__(self):
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
from xfel.ui.components.timeit import duration
import time
t1 = time.time()
run_numbers = [r.run for r in self.trial.runs]
assert self.run.run in run_numbers
rungroup_ids = [rg.id for rg in self.trial.rungroups]
assert self.rungroup.id in rungroup_ids
if len(self.trial.isoforms) > 0:
cells = [isoform.cell for isoform in self.trial.isoforms]
else:
cells = self.app.get_trial_cells(self.trial.id, self.rungroup.id,
self.run.id)
low_res_bin_ids = []
for cell in cells:
bins = cell.bins
d_mins = [float(b.d_min) for b in bins]
if len(d_mins) == 0: continue
low_res_bin_ids.append(str(bins[d_mins.index(max(d_mins))].id))
tag = self.app.params.experiment_tag
timestamps = flex.double()
xtal_ids = flex.double()
n_strong = flex.int()
if len(low_res_bin_ids) > 0:
# Get the spotfinding results from the selected runs
query = """SELECT bin.id, crystal.id, event.timestamp, event.n_strong
FROM `%s_event` event
JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id
JOIN `%s_imageset` imgset ON imgset.id = is_e.imageset_id
JOIN `%s_experiment` exp ON exp.imageset_id = imgset.id
JOIN `%s_crystal` crystal ON crystal.id = exp.crystal_id
JOIN `%s_cell` cell ON cell.id = crystal.cell_id
JOIN `%s_bin` bin ON bin.cell_id = cell.id
JOIN `%s_cell_bin` cb ON cb.bin_id = bin.id AND cb.crystal_id = crystal.id
WHERE event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d AND
cb.bin_id IN (%s)
""" % (tag, tag, tag, tag, tag, tag, tag, tag, self.trial.id,
self.run.id, self.rungroup.id, ", ".join(low_res_bin_ids))
cursor = self.app.execute_query(query)
sample = -1
for row in cursor.fetchall():
b_id, xtal_id, ts, n_s = row
rts = reverse_timestamp(ts)
rts = rts[0] + (rts[1] / 1000)
if xtal_id not in xtal_ids:
sample += 1
if sample % self.sampling != 0:
continue
timestamps.append(rts)
xtal_ids.append(xtal_id)
n_strong.append(n_s)
# This left join query finds the events with no imageset, meaning they failed to index
query = """SELECT event.timestamp, event.n_strong
FROM `%s_event` event
LEFT JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id
WHERE is_e.event_id IS NULL AND
event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d
""" % (tag, tag, self.trial.id, self.run.id, self.rungroup.id)
cursor = self.app.execute_query(query)
for row in cursor.fetchall():
ts, n_s = row
rts = reverse_timestamp(ts)
timestamps.append(rts[0] + (rts[1] / 1000))
n_strong.append(n_s)
order = flex.sort_permutation(timestamps)
timestamps = timestamps.select(order)
n_strong = n_strong.select(order)
t2 = time.time()
return timestamps, n_strong
if params.output_file is None:
logger = sys.stdout
else:
logger = open(params.output_file, 'w')
logger.write("%s " % params.output_file)
if not "DETECTOR_ADDRESS" in source_data:
# legacy format; try to guess the address
LCLS_detector_address = 'CxiDs1-0|Cspad-0'
if "DISTANCE" in source_data and source_data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
LCLS_detector_address = 'CxiDsd-0|Cspad-0'
else:
LCLS_detector_address = source_data["DETECTOR_ADDRESS"]
timesec = reverse_timestamp(source_data["TIMESTAMP"])[0]
version_lookup = detector_format_function(LCLS_detector_address, timesec)
args = [
"distl.detector_format_version=%s" % version_lookup,
"viewer.powder_arcs.show=False",
"viewer.powder_arcs.code=3n9c",
]
horizons_phil = cxi_phil.cxi_versioned_extract(args).persist.commands
img = NpyImage(params.file_path, source_data)
img.readHeader(horizons_phil)
img.translate_tiles(horizons_phil)
if params.verbose:
img.show_header()
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name).option(
None,
"--as_pickle",
"-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files"
).option(
None,
"--raw_data",
"-R",
action="store_true",
default=False,
dest="raw_data",
help=
"Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)"
).option(
None,
"--background_pickle",
"-B",
default=None,
dest="background_pickle",
help=""
).option(
None,
"--config",
"-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file"
).option(
None,
"--experiment",
"-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)"
).option(
None,
"--run",
"-r",
type="int",
default=None,
dest="run",
help="run number"
).option(
None,
"--address",
"-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)"
).option(
None,
"--detz_offset",
"-d",
type="float",
default=None,
dest="detz_offset",
help=
"offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)"
).option(
None,
"--outputdir",
"-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files"
).option(
None,
"--averagebase",
"-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help=
"Path to output average image without extension. String substitution allowed"
).option(
None,
"--stddevbase",
"-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help=
"Path to output standard deviation image without extension. String substitution allowed"
).option(
None,
"--maxbase",
"-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help=
"Path to output maximum projection image without extension. String substitution allowed"
).option(
None,
"--numevents",
"-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all"
).option(
None,
"--skipevents",
"-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0"
).option(
None,
"--verbose",
"-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress"
).option(
None,
"--pickle-optical-metrology",
"-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help=
"If writing pickle files, use the optical metrology in the experiment's calib directory"
).option(
None,
"--bin_size",
"-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size"
).option(
None,
"--override_beam_x",
"-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate"
).option(
None,
"--override_beam_y",
"-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate"
).option(
None,
"--calib_dir",
"-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory"
).option(
None,
"--pickle_calib_dir",
"-P",
type="string",
default=None,
dest="pickle_calib_dir",
metavar="PATH",
help=
"pickle calibration directory specification. Replaces --calib_dir functionality."
).option(
None,
"--xtc_dir",
"-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory"
).option(
None,
"--use_ffb",
"-f",
action="store_true",
default=False,
dest="use_ffb",
help=
"Use the fast feedback filesystem at LCLS. Only for the active experiment!"
).option(
None,
"--gain_mask_value",
"-g",
type="float",
default=None,
dest="gain_mask_value",
help=
"Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode."
).option(
None,
"--min",
None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection"
).option(
None,
"--minpath",
None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help=
"Path to output minimum image without extension. String substitution allowed"
)).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxsize
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment,
command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s" % command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % (
command_line.options.experiment[0:3],
command_line.options.experiment)
if command_line.options.calib_dir is not None:
psana.setOption('psana.calib-dir', command_line.options.calib_dir)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)' % address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(
command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print("Skipping first %d events" % command_line.options.skipevents)
elif "Rayonix" in command_line.options.address:
print("Skipping first image in the Rayonix detector"
) # Shuttering issue
command_line.options.skipevents = 1
for i, evt in enumerate(run.events()):
if i % size != rank: continue
if i < command_line.options.skipevents: continue
if i >= maxevents: break
if i % 10 == 0: print('Rank', rank, 'processing event', i)
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1, src)
if data is None:
print("No data")
continue
data = data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det,
evt,
use_default=False,
dark=False,
common_mode=None,
apply_gain_mask=False,
per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det,
evt,
use_default=True)
else:
data = get_psana_corrected_data(
psana_det,
evt,
use_default=False,
dark=True,
common_mode=None,
apply_gain_mask=True,
gain_mask_value=command_line.options.
gain_mask_value,
per_pixel_gain=False)
if data is None:
print("No data")
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(),
command_line.options.detz_offset)
if d is None:
print("No distance, using distance",
command_line.options.detz_offset)
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print("No wavelength")
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print("No timestamp, skipping shot")
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1] / 1000)
else:
timestamp = np.array([t[0] + (t[1] / 1000)])
if 'sum' in locals():
sum += data
else:
sum = np.array(data, copy=True)
if 'sumsq' in locals():
sumsq += data * data
else:
sumsq = data * data
if 'maximum' in locals():
maximum = np.maximum(maximum, data)
else:
maximum = np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum = np.minimum(minimum, data)
else:
minimum = np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print("Synchronizing rank", rank)
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent, totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum, sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq, sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum, maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum, minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength, waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance, distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp, timeall)
if rank == 0:
if size > 1:
print("Synchronized")
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [
cspad_tbx.pathsubst(command_line.options.averagepath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension,
evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt,
ds.env())
]
if command_line.options.do_minimum_projection:
dest_paths.append(
cspad_tbx.pathsubst(command_line.options.minpath + extension,
evt, ds.env()))
dest_paths = [
os.path.join(command_line.options.outputdir, path)
for path in dest_paths
]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(
command_line.options.bin_size)
beam_center = [
command_line.options.override_beam_x,
command_line.options.override_beam_y
]
active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(address=old_style_address,
data=flex.double(data),
distance=distance,
pixel_size=0.1,
timestamp=timestamp,
wavelength=wavelength)
print("Saving", path)
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[
1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(
evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(
evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
old_style_address,
reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.pickle_calib_dir is not None:
metro_path = command_line.options.pickle_calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories(
"xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.image_xpp(old_style_address,
None,
ds.env(),
active_areas,
quads=quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [
fake_quad(i, mean[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
mean = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [
fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
stddev = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [
fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
maxall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [
fake_quad(i, minall[i * 8:(i + 1) * 8, :, :])
for i in range(4)
]
minall = cspad_tbx.CsPadDetector(address,
evt,
ds.env(),
sections,
quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
d = cspad_tbx.dpack(active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(
run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d" %
run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print("Saving", path)
cspad_img = cspad_cbf_tbx.format_object_from_data(
base_dxtbx,
data,
distance,
wavelength,
timestamp,
address,
round_to_int=False)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def __call__(self):
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
from xfel.ui.components.timeit import duration
#import time
#t1 = time.time()
run_numbers = [r.run for r in self.trial.runs]
assert self.run.run in run_numbers
rungroup_ids = [rg.id for rg in self.trial.rungroups]
assert self.rungroup.id in rungroup_ids
if len(self.trial.isoforms) > 0:
cells = [isoform.cell for isoform in self.trial.isoforms]
else:
cells = self.app.get_trial_cells(self.trial.id, self.rungroup.id,
self.run.id)
high_res_bin_ids = []
for cell in cells:
bins = cell.bins
d_mins = [float(b.d_min) for b in bins]
if len(d_mins) == 0: continue
if self.d_min is None:
min_bin_index = d_mins.index(min(d_mins))
else:
d_maxes = [float(b.d_max) for b in bins]
qualified_bin_indices = [
i for i in xrange(len(bins))
if d_maxes[i] >= self.d_min and d_mins[i] <= self.d_min
]
if len(qualified_bin_indices) == 0: continue
min_bin_index = qualified_bin_indices[0]
high_res_bin_ids.append(str(bins[min_bin_index].id))
resolutions = flex.double()
two_theta_low = flex.double()
two_theta_high = flex.double()
tag = self.app.params.experiment_tag
timestamps = flex.double()
n_strong = flex.int()
n_lattices = flex.int()
if len(high_res_bin_ids) > 0:
# Get the stats in one query.
query = """SELECT event.timestamp, event.n_strong, MIN(bin.d_min), event.two_theta_low, event.two_theta_high, COUNT(crystal.id)
FROM `%s_event` event
JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id
JOIN `%s_imageset` imgset ON imgset.id = is_e.imageset_id
JOIN `%s_experiment` exp ON exp.imageset_id = imgset.id
JOIN `%s_crystal` crystal ON crystal.id = exp.crystal_id
JOIN `%s_cell` cell ON cell.id = crystal.cell_id
JOIN `%s_bin` bin ON bin.cell_id = cell.id
JOIN `%s_cell_bin` cb ON cb.bin_id = bin.id AND cb.crystal_id = crystal.id
WHERE event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d AND
cb.avg_i_sigi >= %f
GROUP BY event.id
""" % (tag, tag, tag, tag, tag, tag, tag, tag, self.trial.id,
self.run.id, self.rungroup.id, self.i_sigi_cutoff)
cursor = self.app.execute_query(query)
sample = -1
for row in cursor.fetchall():
ts, n_s, d_min, tt_low, tt_high, n_xtal = row
try:
d_min = float(d_min)
except ValueError:
d_min = None
rts = reverse_timestamp(ts)
rts = rts[0] + (rts[1] / 1000)
sample += 1
if sample % self.sampling != 0:
continue
timestamps.append(rts)
n_strong.append(n_s)
two_theta_low.append(tt_low or -1)
two_theta_high.append(tt_high or -1)
resolutions.append(d_min or 0)
n_lattices.append(n_xtal or 0)
# This left join query finds the events with no imageset, meaning they failed to index
query = """SELECT event.timestamp, event.n_strong, event.two_theta_low, event.two_theta_high
FROM `%s_event` event
LEFT JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id
WHERE is_e.event_id IS NULL AND
event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d
""" % (tag, tag, self.trial.id, self.run.id, self.rungroup.id)
cursor = self.app.execute_query(query)
for row in cursor.fetchall():
ts, n_s, tt_low, tt_high = row
rts = reverse_timestamp(ts)
timestamps.append(rts[0] + (rts[1] / 1000))
n_strong.append(n_s)
two_theta_low.append(tt_low or -1)
two_theta_high.append(tt_high or -1)
resolutions.append(0)
n_lattices.append(0)
order = flex.sort_permutation(timestamps)
timestamps = timestamps.select(order)
n_strong = n_strong.select(order)
two_theta_low = two_theta_low.select(order)
two_theta_high = two_theta_high.select(order)
resolutions = resolutions.select(order)
n_lattices = n_lattices.select(order)
#t2 = time.time()
#print "HitrateStats took %s" % duration(t1, t2)
return timestamps, two_theta_low, two_theta_high, n_strong, resolutions, n_lattices
def run(argv=None):
"""Compute mean, standard deviation, and maximum projection images
from a set of images given on the command line.
@param argv Command line argument list
@return @c 0 on successful termination, @c 1 on error, and @c 2
for command line syntax errors
"""
import libtbx.load_env
from libtbx import easy_pickle, option_parser
from scitbx.array_family import flex
from xfel.cxi.cspad_ana import cspad_tbx
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
if argv is None:
argv = sys.argv
command_line = (option_parser.option_parser(
usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \
"image1 image2 [image3 ...]" % libtbx.env.dispatcher_name)
.option(None, "--average-path", "-a",
type="string",
default=None,
dest="avg_path",
metavar="PATH",
help="Write average image to PATH")
.option(None, "--maximum-path", "-m",
type="string",
default=None,
dest="max_path",
metavar="PATH",
help="Write maximum projection image to PATH")
.option(None, "--stddev-path", "-s",
type="string",
default=None,
dest="stddev_path",
metavar="PATH",
help="Write standard deviation image to PATH")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
).process(args=argv[1:])
# Note that it is not an error to omit the output paths, because
# certain statistics could still be printed, e.g. with the verbose
# option.
paths = command_line.args
if len(paths) == 0:
command_line.parser.print_usage(file=sys.stderr)
return 2
# Loop over all images and accumulate statistics.
nfail = 0
nmemb = 0
for path in paths:
if command_line.options.verbose:
sys.stdout.write("Processing %s...\n" % path)
try:
# Promote the image to double-precision floating point type.
# All real-valued flex arrays have the as_double() function.
d = easy_pickle.load(path)
distance = d['DISTANCE']
img = d['DATA'].as_1d().as_double()
wavelength = d['WAVELENGTH']
time_tuple = reverse_timestamp(d['TIMESTAMP'])
# Warn if the header items across the set of images do not match
# up. Note that discrepancies regarding the image size are
# fatal.
if 'active_areas' in locals():
if (active_areas != d['ACTIVE_AREAS']).count(True) != 0:
sys.stderr.write("Active areas do not match\n")
else:
active_areas = d['ACTIVE_AREAS']
if 'beam_center' in locals():
if beam_center != (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']):
sys.stderr.write("Beam centers do not match\n")
else:
beam_center = (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y'])
if 'detector_address' in locals():
if detector_address != d['DETECTOR_ADDRESS']:
sys.stderr.write("Detector addresses do not match\n")
else:
detector_address = d['DETECTOR_ADDRESS']
if 'saturated_value' in locals():
if saturated_value != d['SATURATED_VALUE']:
sys.stderr.write("Saturated values do not match\n")
else:
saturated_value = d['SATURATED_VALUE']
if 'size' in locals():
if size != (d['SIZE1'], d['SIZE2']):
sys.stderr.write("Image sizes do not match\n")
return 1
else:
size = (d['SIZE1'], d['SIZE2'])
if size != d['DATA'].focus():
sys.stderr.write("Image size does not match pixel array\n")
return 1
if 'pixel_size' in locals():
if pixel_size != d['PIXEL_SIZE']:
sys.stderr.write("Pixel sizes do not match\n")
return 1
else:
if 'PIXEL_SIZE' in d:
pixel_size = d['PIXEL_SIZE']
else:
pixel_size = None
except Exception:
try:
# Fall back on reading the image with dxtbx, and shoehorn the
# extracted information into what would have been found in a
# pickle file. XXX This code assumes a monolithic detector!
from dxtbx.format.Registry import Registry
format_class = Registry.find(path)
i = format_class(path)
beam = i.get_beam()
assert len(i.get_detector()) == 1
detector = i.get_detector()[0]
beam_center = detector.get_beam_centre(beam.get_s0())
detector_address = format_class.__name__
distance = detector.get_distance()
img = i.get_raw_data().as_1d().as_double()
pixel_size = 0.5 * sum(detector.get_pixel_size())
saturated_value = int(round(detector.get_trusted_range()[1]))
size = detector.get_image_size()
time_tuple = (i.get_scan().get_epochs()[0], 0)
wavelength = beam.get_wavelength()
active_areas = flex.int((0, 0, size[0], size[1]))
except Exception:
nfail += 1
continue
# See also event() in xfel.cxi.cspad_ana.average_tbx. Record the
# base time as the timestamp of the first image.
#
# The sum-of-squares image is accumulated using long integers, as
# this delays the point where overflow occurs. But really, this
# is just a band-aid...
if nmemb == 0:
max_img = img.deep_copy()
sum_distance = distance
sum_img = img.deep_copy()
ssq_img = flex.pow2(img)
sum_wavelength = wavelength
sum_time = (0, 0)
time_base = time_tuple
else:
sel = (img > max_img).as_1d()
max_img.set_selected(sel, img.select(sel))
sum_distance += distance
sum_img += img
ssq_img += flex.pow2(img)
sum_wavelength += wavelength
sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]),
sum_time[1] + (time_tuple[1] - time_base[1]))
nmemb += 1
# Early exit if no statistics were accumulated.
if command_line.options.verbose:
sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail))
if nmemb == 0:
return 0
# Calculate averages for measures where other statistics do not make
# sense. Note that avg_img is required for stddev_img.
avg_img = sum_img.as_double() / nmemb
avg_distance = sum_distance / nmemb
avg_timestamp = cspad_tbx.evt_timestamp(
(time_base[0] + int(round(sum_time[0] / nmemb)),
time_base[1] + int(round(sum_time[1] / nmemb))))
avg_wavelength = sum_wavelength / nmemb
# Output the average image, maximum projection image, and standard
# deviation image, if requested.
if command_line.options.avg_path is not None:
avg_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=avg_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.avg_path, d)
if command_line.options.max_path is not None:
max_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=max_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.max_path, d)
if command_line.options.stddev_path is not None:
stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img
# Accumulating floating-point numbers introduces errors, which may
# cause negative variances. Since a two-pass approach is
# unacceptable, the standard deviation is clamped at zero.
stddev_img.set_selected(stddev_img < 0, 0)
if nmemb == 1:
stddev_img = flex.sqrt(stddev_img)
else:
stddev_img = flex.sqrt(stddev_img / (nmemb - 1))
stddev_img.resize(flex.grid(size[0], size[1]))
d = cspad_tbx.dpack(
active_areas=active_areas,
address=detector_address,
beam_center_x=beam_center[0],
beam_center_y=beam_center[1],
data=stddev_img,
distance=avg_distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=avg_timestamp,
wavelength=avg_wavelength)
easy_pickle.dump(command_line.options.stddev_path, d)
return 0
def run(args, source_data=None):
from xfel import radial_average
from scitbx.array_family import flex
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function
from spotfinder.applications.xfel import cxi_phil
from iotbx.detectors.npy import NpyImage
import os, sys
from iotbx.detectors.npy import NpyImage
user_phil = []
# TODO: replace this stuff with iotbx.phil.process_command_line_with_files
# as soon as I can safely modify it
for arg in args:
if (not "=" in arg):
try:
user_phil.append(libtbx.phil.parse("""file_path=%s""" % arg))
except ValueError as e:
raise Sorry("Unrecognized argument '%s'" % arg)
else:
try:
user_phil.append(libtbx.phil.parse(arg))
except RuntimeError as e:
raise Sorry("Unrecognized argument '%s' (error: %s)" %
(arg, str(e)))
params = master_phil.fetch(sources=user_phil).extract()
if params.file_path is None or not os.path.isfile(
params.file_path) and source_data is None:
master_phil.show()
raise Usage(
"file_path must be defined (either file_path=XXX, or the path alone)."
)
assert params.handedness is not None
assert params.n_bins is not None
assert params.verbose is not None
assert params.output_bins is not None
if source_data is None:
from libtbx import easy_pickle
source_data = easy_pickle.load(params.file_path)
if params.output_file is None:
logger = sys.stdout
else:
logger = open(params.output_file, 'w')
logger.write("%s " % params.output_file)
if not "DETECTOR_ADDRESS" in source_data:
# legacy format; try to guess the address
LCLS_detector_address = 'CxiDs1-0|Cspad-0'
if "DISTANCE" in source_data and source_data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
LCLS_detector_address = 'CxiDsd-0|Cspad-0'
else:
LCLS_detector_address = source_data["DETECTOR_ADDRESS"]
timesec = reverse_timestamp(source_data["TIMESTAMP"])[0]
version_lookup = detector_format_function(LCLS_detector_address, timesec)
args = [
"distl.detector_format_version=%s" % version_lookup,
"viewer.powder_arcs.show=False",
"viewer.powder_arcs.code=3n9c",
]
horizons_phil = cxi_phil.cxi_versioned_extract(args).persist.commands
img = NpyImage(params.file_path, source_data)
img.readHeader(horizons_phil)
img.translate_tiles(horizons_phil)
if params.verbose:
img.show_header()
the_tiles = img.get_tile_manager(
horizons_phil).effective_tiling_as_flex_int(
reapply_peripheral_margin=False, encode_inactive_as_zeroes=True)
if params.beam_x is None:
params.beam_x = img.beamx / img.pixel_size
if params.beam_y is None:
params.beam_y = img.beamy / img.pixel_size
if params.verbose:
logger.write("I think the beam center is (%s,%s)\n" %
(params.beam_x, params.beam_y))
bc = (int(params.beam_x), int(params.beam_y))
extent = int(
math.ceil(
max(distance((0, 0), bc), distance((img.size1, 0), bc),
distance((0, img.size2), bc),
distance((img.size1, img.size2), bc))))
if params.n_bins < extent:
params.n_bins = extent
extent_in_mm = extent * img.pixel_size
extent_two_theta = math.atan(extent_in_mm / img.distance) * 180 / math.pi
sums = flex.double(params.n_bins) * 0
sums_sq = flex.double(params.n_bins) * 0
counts = flex.int(params.n_bins) * 0
data = img.get_raw_data()
if hasattr(data, "as_double"):
data = data.as_double()
logger.write("Average intensity: %9.3f\n" % flex.mean(data))
if params.verbose:
logger.write("Generating average...tile:")
logger.flush()
for tile in xrange(len(the_tiles) // 4):
if params.verbose:
logger.write(" %d" % tile)
logger.flush()
x1, y1, x2, y2 = get_tile_coords(the_tiles, tile)
radial_average(data, bc, sums, sums_sq, counts, img.pixel_size,
img.distance, (x1, y1), (x2, y2))
if params.verbose:
logger.write(" Finishing...\n")
# average, avoiding division by zero
results = sums.set_selected(counts <= 0, 0)
results /= counts.set_selected(counts <= 0, 1).as_double()
# calculte standard devations
std_devs = [
math.sqrt((sums_sq[i] - sums[i] * results[i]) /
counts[i]) if counts[i] > 0 else 0 for i in xrange(len(sums))
]
xvals = flex.double(len(results))
max_twotheta = float('-inf')
max_result = float('-inf')
for i in xrange(len(results)):
twotheta = i * extent_two_theta / params.n_bins
xvals[i] = twotheta
if params.output_bins and "%.3f" % results[i] != "nan":
#logger.write("%9.3f %9.3f\n"% (twotheta,results[i])) #.xy format for Rex.cell.
logger.write(
"%9.3f %9.3f %9.3f\n" %
(twotheta, results[i], std_devs[i])) #.xye format for GSASII
#logger.write("%.3f %.3f %.3f\n"%(twotheta,results[i],ds[i])) # include calculated d spacings
if results[i] > max_result:
max_twotheta = twotheta
max_result = results[i]
logger.write(
"Maximum 2theta for %s, TS %s: %f, value: %f\n" %
(params.file_path, source_data['TIMESTAMP'], max_twotheta, max_result))
if params.verbose:
from pylab import scatter, show, xlabel, ylabel, ylim
scatter(xvals, results)
xlabel("2 theta")
ylabel("Avg ADUs")
if params.plot_y_max is not None:
ylim(0, params.plot_y_max)
show()
return xvals, results
if params.output_file is None:
logger = sys.stdout
else:
logger = open(params.output_file, 'w')
logger.write("%s "%params.output_file)
if not "DETECTOR_ADDRESS" in source_data:
# legacy format; try to guess the address
LCLS_detector_address = 'CxiDs1-0|Cspad-0'
if "DISTANCE" in source_data and source_data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
LCLS_detector_address = 'CxiDsd-0|Cspad-0'
else:
LCLS_detector_address = source_data["DETECTOR_ADDRESS"]
timesec = reverse_timestamp( source_data["TIMESTAMP"] )[0]
version_lookup = detector_format_function(LCLS_detector_address,timesec)
args = [
"distl.detector_format_version=%s"%version_lookup,
"viewer.powder_arcs.show=False",
"viewer.powder_arcs.code=3n9c",
]
horizons_phil = cxi_phil.cxi_versioned_extract(args).persist.commands
img = NpyImage(params.file_path, source_data)
img.readHeader(horizons_phil)
img.translate_tiles(horizons_phil)
if params.verbose:
img.show_header()
def __call__(self):
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
run_numbers = [r.run for r in self.trial.runs]
assert self.run.run in run_numbers
rungroup_ids = [rg.id for rg in self.trial.rungroups]
assert self.rungroup.id in rungroup_ids
isoforms = self.trial.isoforms
assert len(isoforms) > 0
low_res_bin_ids = []
high_res_bin_ids = []
for isoform in isoforms:
bins = isoform.cell.bins
d_mins = [float(b.d_min) for b in bins]
low_res_bin_ids.append(str(bins[d_mins.index(max(d_mins))].id))
if self.d_min is None:
min_bin_index = d_mins.index(min(d_mins))
else:
d_maxes = [float(b.d_max) for b in bins]
qualified_bin_indices = [i for i in xrange(len(bins)) if d_maxes[i] >= self.d_min and d_mins[i] <= self.d_min]
assert len(qualified_bin_indices) == 1
min_bin_index = qualified_bin_indices[0]
high_res_bin_ids.append(str(bins[min_bin_index].id))
assert len(low_res_bin_ids) > 0
assert len(high_res_bin_ids) > 0
assert len(low_res_bin_ids) == len(high_res_bin_ids)
tag = self.app.params.experiment_tag
# Get the high and low res avg_i_sigi in one query. Means there will be 2x timestamps retrieved, where each is found twice
query = """SELECT bin.id, event.timestamp, event.n_strong, cb.avg_i_sigi, event.two_theta_low, event.two_theta_high
FROM `%s_event` event
JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id
JOIN `%s_imageset` imgset ON imgset.id = is_e.imageset_id
JOIN `%s_experiment` exp ON exp.imageset_id = imgset.id
JOIN `%s_crystal` crystal ON crystal.id = exp.crystal_id
JOIN `%s_cell` cell ON cell.id = crystal.cell_id
JOIN `%s_bin` bin ON bin.cell_id = cell.id
JOIN `%s_cell_bin` cb ON cb.bin_id = bin.id AND cb.crystal_id = crystal.id
WHERE event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d AND
cb.bin_id IN (%s)
""" % (tag, tag, tag, tag, tag, tag, tag, tag, self.trial.id, self.run.id, self.rungroup.id, ", ".join(low_res_bin_ids + high_res_bin_ids))
cursor = self.app.execute_query(query)
timestamps = flex.double()
n_strong = flex.int()
average_i_sigi_low = flex.double()
average_i_sigi_high = flex.double()
two_theta_low = flex.double()
two_theta_high = flex.double()
for row in cursor.fetchall():
b_id, ts, n_s, avg_i_sigi, tt_low, tt_high = row
rts = reverse_timestamp(ts)
rts = rts[0] + (rts[1]/1000)
if rts not in timestamps:
# First time through, figure out which bin is reported (high or low), add avg_i_sigi to that set of results
timestamps.append(rts)
n_strong.append(n_s)
two_theta_low.append(tt_low or -1)
two_theta_high.append(tt_high or -1)
if str(b_id) in low_res_bin_ids:
average_i_sigi_low.append(avg_i_sigi or 1e-6)
average_i_sigi_high.append(0)
elif str(b_id) in high_res_bin_ids:
average_i_sigi_low.append(0)
average_i_sigi_high.append(avg_i_sigi or 0)
else:
assert False
else:
# Second time through, already have added to timestamps and n_strong, so fill in missing avg_i_sigi
index = flex.first_index(timestamps, rts)
if str(b_id) in low_res_bin_ids:
average_i_sigi_low[index] = avg_i_sigi
elif str(b_id) in high_res_bin_ids:
average_i_sigi_high[index] = avg_i_sigi or 0
else:
assert False
# This left join query finds the events with no imageset, meaning they failed to index
query = """SELECT event.timestamp, event.n_strong, event.two_theta_low, event.two_theta_high
FROM `%s_event` event
LEFT JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id
WHERE is_e.event_id IS NULL AND
event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d
""" % (tag, tag, self.trial.id, self.run.id, self.rungroup.id)
cursor = self.app.execute_query(query)
for row in cursor.fetchall():
ts, n_s, tt_low, tt_high = row
rts = reverse_timestamp(ts)
timestamps.append(rts[0] + (rts[1]/1000))
n_strong.append(n_s)
average_i_sigi_low.append(0)
average_i_sigi_high.append(0)
two_theta_low.append(tt_low or -1)
two_theta_high.append(tt_high or -1)
order = flex.sort_permutation(timestamps)
timestamps = timestamps.select(order)
n_strong = n_strong.select(order)
average_i_sigi_low = average_i_sigi_low.select(order)
average_i_sigi_high = average_i_sigi_high.select(order)
two_theta_low = two_theta_low.select(order)
two_theta_high = two_theta_high.select(order)
return timestamps, two_theta_low, two_theta_high, n_strong, average_i_sigi_low, average_i_sigi_high
def average(argv=None):
if argv == None:
argv = sys.argv[1:]
try:
from mpi4py import MPI
except ImportError:
raise Sorry("MPI not found")
command_line = (libtbx.option_parser.option_parser(
usage="""
%s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath]
To write image pickles use -p, otherwise the program writes CSPAD CBFs.
Writing CBFs requires the geometry to be already deployed.
Examples:
cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
Use one process on the current node to process all the events from run 25 of
experiment cxi49812, using a detz_offset of 571.
mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571
As above, using 16 cores on the current node.
bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812
As above, using the psanaq and 100 cores, putting the log in average.out and
the output images in the folder cxi49812.
""" % libtbx.env.dispatcher_name)
.option(None, "--as_pickle", "-p",
action="store_true",
default=False,
dest="as_pickle",
help="Write results as image pickle files instead of cbf files")
.option(None, "--raw_data", "-R",
action="store_true",
default=False,
dest="raw_data",
help="Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)")
.option(None, "--background_pickle", "-B",
default=None,
dest="background_pickle",
help="")
.option(None, "--config", "-c",
type="string",
default=None,
dest="config",
metavar="PATH",
help="psana config file")
.option(None, "--experiment", "-x",
type="string",
default=None,
dest="experiment",
help="experiment name (eg cxi84914)")
.option(None, "--run", "-r",
type="int",
default=None,
dest="run",
help="run number")
.option(None, "--address", "-a",
type="string",
default="CxiDs2.0:Cspad.0",
dest="address",
help="detector address name (eg CxiDs2.0:Cspad.0)")
.option(None, "--detz_offset", "-d",
type="float",
default=None,
dest="detz_offset",
help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)")
.option(None, "--outputdir", "-o",
type="string",
default=".",
dest="outputdir",
metavar="PATH",
help="Optional path to output directory for output files")
.option(None, "--averagebase", "-A",
type="string",
default="{experiment!l}_avg-r{run:04d}",
dest="averagepath",
metavar="PATH",
help="Path to output average image without extension. String substitution allowed")
.option(None, "--stddevbase", "-S",
type="string",
default="{experiment!l}_stddev-r{run:04d}",
dest="stddevpath",
metavar="PATH",
help="Path to output standard deviation image without extension. String substitution allowed")
.option(None, "--maxbase", "-M",
type="string",
default="{experiment!l}_max-r{run:04d}",
dest="maxpath",
metavar="PATH",
help="Path to output maximum projection image without extension. String substitution allowed")
.option(None, "--numevents", "-n",
type="int",
default=None,
dest="numevents",
help="Maximum number of events to process. Default: all")
.option(None, "--skipevents", "-s",
type="int",
default=0,
dest="skipevents",
help="Number of events in the beginning of the run to skip. Default: 0")
.option(None, "--verbose", "-v",
action="store_true",
default=False,
dest="verbose",
help="Print more information about progress")
.option(None, "--pickle-optical-metrology", "-m",
action="store_true",
default=False,
dest="pickle_optical_metrology",
help="If writing pickle files, use the optical metrology in the experiment's calib directory")
.option(None, "--bin_size", "-b",
type="int",
default=None,
dest="bin_size",
help="Rayonix detector bin size")
.option(None, "--override_beam_x", "-X",
type="float",
default=None,
dest="override_beam_x",
help="Rayonix detector beam center x coordinate")
.option(None, "--override_beam_y", "-Y",
type="float",
default=None,
dest="override_beam_y",
help="Rayonix detector beam center y coordinate")
.option(None, "--calib_dir", "-C",
type="string",
default=None,
dest="calib_dir",
metavar="PATH",
help="calibration directory")
.option(None, "--xtc_dir", "-D",
type="string",
default=None,
dest="xtc_dir",
metavar="PATH",
help="xtc stream directory")
.option(None, "--use_ffb", "-f",
action="store_true",
default=False,
dest="use_ffb",
help="Use the fast feedback filesystem at LCLS. Only for the active experiment!")
.option(None, "--gain_mask_value", "-g",
type="float",
default=None,
dest="gain_mask_value",
help="Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode.")
.option(None, "--min", None,
action="store_true",
default=False,
dest="do_minimum_projection",
help="Output a minimum projection")
.option(None, "--minpath", None,
type="string",
default="{experiment!l}_min-r{run:04d}",
dest="minpath",
metavar="PATH",
help="Path to output minimum image without extension. String substitution allowed")
).process(args=argv)
if len(command_line.args) > 0 or \
command_line.options.as_pickle is None or \
command_line.options.experiment is None or \
command_line.options.run is None or \
command_line.options.address is None or \
command_line.options.detz_offset is None or \
command_line.options.averagepath is None or \
command_line.options.stddevpath is None or \
command_line.options.maxpath is None or \
command_line.options.pickle_optical_metrology is None:
command_line.parser.show_help()
return
# set this to sys.maxint to analyze all events
if command_line.options.numevents is None:
maxevents = sys.maxint
else:
maxevents = command_line.options.numevents
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if command_line.options.config is not None:
psana.setConfigFile(command_line.options.config)
dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run)
if command_line.options.xtc_dir is not None:
if command_line.options.use_ffb:
raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system")
dataset_name += ":dir=%s"%command_line.options.xtc_dir
elif command_line.options.use_ffb:
# as ffb is only at SLAC, ok to hardcode /reg/d here
dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment)
ds = psana.DataSource(dataset_name)
address = command_line.options.address
src = psana.Source('DetInfo(%s)'%address)
nevent = np.array([0.])
if command_line.options.background_pickle is not None:
background = easy_pickle.load(command_line.options.background_pickle)['DATA'].as_numpy_array()
for run in ds.runs():
runnumber = run.run()
if not command_line.options.as_pickle:
psana_det = psana.Detector(address, ds.env())
# list of all events
if command_line.options.skipevents > 0:
print "Skipping first %d events"%command_line.options.skipevents
elif "Rayonix" in command_line.options.address:
print "Skipping first image in the Rayonix detector" # Shuttering issue
command_line.options.skipevents = 1
times = run.times()[command_line.options.skipevents:]
nevents = min(len(times),maxevents)
# chop the list into pieces, depending on rank. This assigns each process
# events such that the get every Nth event where N is the number of processes
mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0]
for i in xrange(len(mytimes)):
if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes)
evt = run.event(mytimes[i])
#print "Event #",rank*mylength+i," has id:",evt.get(EventId)
if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
data = evt.get(psana.Camera.FrameV1,src)
if data is None:
print "No data"
continue
data=data.data16().astype(np.float64)
elif command_line.options.as_pickle:
data = evt.get(psana.ndarray_float64_3, src, 'image0')
else:
# get numpy array, 32x185x388
from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data
if command_line.options.raw_data:
data = get_psana_corrected_data(psana_det, evt, use_default=False, dark=False, common_mode=None,
apply_gain_mask=False, per_pixel_gain=False)
else:
if command_line.options.gain_mask_value is None:
data = get_psana_corrected_data(psana_det, evt, use_default=True)
else:
data = get_psana_corrected_data(psana_det, evt, use_default=False, dark=True, common_mode=None,
apply_gain_mask=True, gain_mask_value=command_line.options.gain_mask_value,
per_pixel_gain=False)
if data is None:
print "No data"
continue
if command_line.options.background_pickle is not None:
data -= background
if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
distance = np.array([0.0])
wavelength = np.array([1.0])
else:
d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset)
if d is None:
print "No distance, using distance", command_line.options.detz_offset
assert command_line.options.detz_offset is not None
if 'distance' not in locals():
distance = np.array([command_line.options.detz_offset])
else:
distance += command_line.options.detz_offset
else:
if 'distance' in locals():
distance += d
else:
distance = np.array([float(d)])
w = cspad_tbx.evt_wavelength(evt)
if w is None:
print "No wavelength"
if 'wavelength' not in locals():
wavelength = np.array([1.0])
else:
if 'wavelength' in locals():
wavelength += w
else:
wavelength = np.array([w])
t = cspad_tbx.evt_time(evt)
if t is None:
print "No timestamp, skipping shot"
continue
if 'timestamp' in locals():
timestamp += t[0] + (t[1]/1000)
else:
timestamp = np.array([t[0] + (t[1]/1000)])
if 'sum' in locals():
sum+=data
else:
sum=np.array(data, copy=True)
if 'sumsq' in locals():
sumsq+=data*data
else:
sumsq=data*data
if 'maximum' in locals():
maximum=np.maximum(maximum,data)
else:
maximum=np.array(data, copy=True)
if command_line.options.do_minimum_projection:
if 'minimum' in locals():
minimum=np.minimum(minimum,data)
else:
minimum=np.array(data, copy=True)
nevent += 1
#sum the images across mpi cores
if size > 1:
print "Synchronizing rank", rank
totevent = np.zeros(nevent.shape)
comm.Reduce(nevent,totevent)
if rank == 0 and totevent[0] == 0:
raise Sorry("No events found in the run")
sumall = np.zeros(sum.shape).astype(sum.dtype)
comm.Reduce(sum,sumall)
sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype)
comm.Reduce(sumsq,sumsqall)
maxall = np.zeros(maximum.shape).astype(maximum.dtype)
comm.Reduce(maximum,maxall, op=MPI.MAX)
if command_line.options.do_minimum_projection:
minall = np.zeros(maximum.shape).astype(minimum.dtype)
comm.Reduce(minimum,minall, op=MPI.MIN)
waveall = np.zeros(wavelength.shape).astype(wavelength.dtype)
comm.Reduce(wavelength,waveall)
distall = np.zeros(distance.shape).astype(distance.dtype)
comm.Reduce(distance,distall)
timeall = np.zeros(timestamp.shape).astype(timestamp.dtype)
comm.Reduce(timestamp,timeall)
if rank==0:
if size > 1:
print "Synchronized"
# Accumulating floating-point numbers introduces errors,
# which may cause negative variances. Since a two-pass
# approach is unacceptable, the standard deviation is
# clamped at zero.
mean = sumall / float(totevent[0])
variance = (sumsqall / float(totevent[0])) - (mean**2)
variance[variance < 0] = 0
stddev = np.sqrt(variance)
wavelength = waveall[0] / totevent[0]
distance = distall[0] / totevent[0]
pixel_size = cspad_tbx.pixel_size
saturated_value = cspad_tbx.cspad_saturated_value
timestamp = timeall[0] / totevent[0]
timestamp = (int(timestamp), timestamp % int(timestamp) * 1000)
timestamp = cspad_tbx.evt_timestamp(timestamp)
if command_line.options.as_pickle:
extension = ".pickle"
else:
extension = ".cbf"
dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()),
cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env())]
if command_line.options.do_minimum_projection:
dest_paths.append(cspad_tbx.pathsubst(command_line.options.minpath + extension, evt, ds.env()))
dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths]
if 'Rayonix' in command_line.options.address:
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
from xfel.cxi.cspad_ana import rayonix_tbx
pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size)
beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y]
detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size)
active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]])
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
for data, path in zip(all_data, dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=flex.double(data),
distance=distance,
pixel_size=pixel_size,
saturated_value=rayonix_tbx.rayonix_saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address:
all_data = [mean, stddev, maxall]
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
d = cspad_tbx.dpack(
address = old_style_address,
data = flex.double(data),
distance = distance,
pixel_size = 0.1,
timestamp=timestamp,
wavelength=wavelength
)
print "Saving", path
easy_pickle.dump(path, d)
elif command_line.options.as_pickle:
split_address = cspad_tbx.address_split(address)
old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3]
xpp = 'xpp' in address.lower()
if xpp:
evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds
timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0])
assert version_lookup is not None
active_areas = xpp_active_areas[version_lookup]['active_areas']
beam_center = [1765 // 2, 1765 // 2]
else:
if command_line.options.calib_dir is not None:
metro_path = command_line.options.calib_dir
elif command_line.options.pickle_optical_metrology:
from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path
metro_path = get_calib_file_path(run.env(), address, run)
else:
metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0")
sections = parse_calib.calib2sections(metro_path)
beam_center, active_areas = cspad_tbx.cbcaa(
cspad_tbx.getConfig(address, ds.env()), sections)
class fake_quad(object):
def __init__(self, q, d):
self.q = q
self.d = d
def quad(self):
return self.q
def data(self):
return self.d
if xpp:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [fake_quad(i, minall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
minall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads)
minall = flex.double(minall.astype(np.float64))
else:
quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)]
mean = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
mean = flex.double(mean.astype(np.float64))
quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)]
stddev = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
stddev = flex.double(stddev.astype(np.float64))
quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
maxall = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
maxall = flex.double(maxall.astype(np.float64))
if command_line.options.do_minimum_projection:
quads = [fake_quad(i, minall[i*8:(i+1)*8,:,:]) for i in xrange(4)]
minall = cspad_tbx.CsPadDetector(
address, evt, ds.env(), sections, quads=quads)
minall = flex.double(minall.astype(np.float64))
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print "Saving", path
d = cspad_tbx.dpack(
active_areas=active_areas,
address=old_style_address,
beam_center_x=pixel_size * beam_center[0],
beam_center_y=pixel_size * beam_center[1],
data=data,
distance=distance,
pixel_size=pixel_size,
saturated_value=saturated_value,
timestamp=timestamp,
wavelength=wavelength)
easy_pickle.dump(path, d)
else:
# load a header only cspad cbf from the slac metrology
from xfel.cftbx.detector import cspad_cbf_tbx
import pycbf
base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address)
if base_dxtbx is None:
raise Sorry("Couldn't load calibration file for run %d"%run.run())
all_data = [mean, stddev, maxall]
if command_line.options.do_minimum_projection:
all_data.append(minall)
for data, path in zip(all_data, dest_paths):
print "Saving", path
cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=False)
cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\
pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
for data, path in generate_data_from_streams(args, verbose=True):
if 'TIMESTAMP' in data:
# this is how FormatPYunspecified guesses the address
if not "DETECTOR_ADDRESS" in data:
# legacy format; try to guess the address
LCLS_detector_address = 'CxiDs1-0|Cspad-0'
if "DISTANCE" in data and data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
LCLS_detector_address = 'CxiDsd-0|Cspad-0'
else:
LCLS_detector_address = data["DETECTOR_ADDRESS"]
detector_format_version = detector_format_function(
LCLS_detector_address,
reverse_timestamp(data['TIMESTAMP'])[0])
print("Detector format version:", detector_format_version)
image_pickle = True
else:
print("Not an image pickle")
image_pickle = False
keywise_printout(data)
if image_pickle:
import dxtbx
image = dxtbx.load(path)
tile_manager = image.detectorbase.get_tile_manager(
image.detectorbase.horizons_phil_cache)
tiling = tile_manager.effective_tiling_as_flex_int(
reapply_peripheral_margin=True)
def __call__(self):
from iotbx.detectors.cspad_detector_formats import reverse_timestamp
run_numbers = [r.run for r in self.trial.runs]
assert self.run.run in run_numbers
rungroup_ids = [rg.id for rg in self.trial.rungroups]
assert self.rungroup.id in rungroup_ids
isoforms = self.trial.isoforms
assert len(isoforms) > 0
low_res_bin_ids = []
high_res_bin_ids = []
for isoform in isoforms:
bins = isoform.cell.bins
d_mins = [float(b.d_min) for b in bins]
low_res_bin_ids.append(str(bins[d_mins.index(max(d_mins))].id))
if self.d_min is None:
min_bin_index = d_mins.index(min(d_mins))
else:
d_maxes = [float(b.d_max) for b in bins]
qualified_bin_indices = [
i for i in xrange(len(bins))
if d_maxes[i] >= self.d_min and d_mins[i] <= self.d_min
]
assert len(qualified_bin_indices) == 1
min_bin_index = qualified_bin_indices[0]
high_res_bin_ids.append(str(bins[min_bin_index].id))
assert len(low_res_bin_ids) > 0
assert len(high_res_bin_ids) > 0
assert len(low_res_bin_ids) == len(high_res_bin_ids)
tag = self.app.params.experiment_tag
# Get the high and low res avg_i_sigi in one query. Means there will be 2x timestamps retrieved, where each is found twice
query = """SELECT bin.id, event.timestamp, event.n_strong, cb.avg_i_sigi, event.two_theta_low, event.two_theta_high
FROM `%s_event` event
JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id
JOIN `%s_imageset` imgset ON imgset.id = is_e.imageset_id
JOIN `%s_experiment` exp ON exp.imageset_id = imgset.id
JOIN `%s_crystal` crystal ON crystal.id = exp.crystal_id
JOIN `%s_cell` cell ON cell.id = crystal.cell_id
JOIN `%s_bin` bin ON bin.cell_id = cell.id
JOIN `%s_cell_bin` cb ON cb.bin_id = bin.id AND cb.crystal_id = crystal.id
WHERE event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d AND
cb.bin_id IN (%s)
""" % (tag, tag, tag, tag, tag, tag, tag, tag, self.trial.id,
self.run.id, self.rungroup.id,
", ".join(low_res_bin_ids + high_res_bin_ids))
cursor = self.app.execute_query(query)
timestamps = flex.double()
n_strong = flex.int()
average_i_sigi_low = flex.double()
average_i_sigi_high = flex.double()
two_theta_low = flex.double()
two_theta_high = flex.double()
for row in cursor.fetchall():
b_id, ts, n_s, avg_i_sigi, tt_low, tt_high = row
rts = reverse_timestamp(ts)
rts = rts[0] + (rts[1] / 1000)
if rts not in timestamps:
# First time through, figure out which bin is reported (high or low), add avg_i_sigi to that set of results
timestamps.append(rts)
n_strong.append(n_s)
two_theta_low.append(tt_low or -1)
two_theta_high.append(tt_high or -1)
if str(b_id) in low_res_bin_ids:
average_i_sigi_low.append(avg_i_sigi or 1e-6)
average_i_sigi_high.append(0)
elif str(b_id) in high_res_bin_ids:
average_i_sigi_low.append(0)
average_i_sigi_high.append(avg_i_sigi or 0)
else:
assert False
else:
# Second time through, already have added to timestamps and n_strong, so fill in missing avg_i_sigi
index = flex.first_index(timestamps, rts)
if str(b_id) in low_res_bin_ids:
average_i_sigi_low[index] = avg_i_sigi
elif str(b_id) in high_res_bin_ids:
average_i_sigi_high[index] = avg_i_sigi or 0
else:
assert False
# This left join query finds the events with no imageset, meaning they failed to index
query = """SELECT event.timestamp, event.n_strong, event.two_theta_low, event.two_theta_high
FROM `%s_event` event
LEFT JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id
WHERE is_e.event_id IS NULL AND
event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d
""" % (tag, tag, self.trial.id, self.run.id, self.rungroup.id)
cursor = self.app.execute_query(query)
for row in cursor.fetchall():
ts, n_s, tt_low, tt_high = row
rts = reverse_timestamp(ts)
timestamps.append(rts[0] + (rts[1] / 1000))
n_strong.append(n_s)
average_i_sigi_low.append(0)
average_i_sigi_high.append(0)
two_theta_low.append(tt_low or -1)
two_theta_high.append(tt_high or -1)
order = flex.sort_permutation(timestamps)
timestamps = timestamps.select(order)
n_strong = n_strong.select(order)
average_i_sigi_low = average_i_sigi_low.select(order)
average_i_sigi_high = average_i_sigi_high.select(order)
two_theta_low = two_theta_low.select(order)
two_theta_high = two_theta_high.select(order)
return timestamps, two_theta_low, two_theta_high, n_strong, average_i_sigi_low, average_i_sigi_high
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).event(evt, env)
if (evt.get("skip_event")):
return
if not hasattr(self, 'active_areas') or self.active_areas is None or \
not hasattr(self, 'beam_center') or self.beam_center is None:
if self.address == 'XppGon-0|marccd-0':
# The mod_mar module needs to have been called before this one
# to set this up. The MAR does not have a configure object.
self.beam_center = evt.get("marccd_beam_center")
self.active_areas = evt.get("marccd_active_areas")
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
pass # bc and aa set in the beginjob function
elif self.address == 'XppGon-0|Cspad-0':
# Load the active areas as determined from the optical metrology
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(self.address, reverse_timestamp(self.timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup]['active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = \
cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)
if self.filter_laser_1_status is not None:
if (self.laser_1_status.status != self.filter_laser_1_status or
(self.laser_1_ms_since_change is not None and
self.laser_1_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
if self.filter_laser_4_status is not None:
if (self.laser_4_status.status != self.filter_laser_4_status or
(self.laser_4_ms_since_change is not None and
self.laser_4_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
# Early return if the full detector image is already stored in the
# event. Otherwise, get it from the stream as a double-precision
# floating-point flex array. XXX It is probably not safe to key
# the image on self.address, so we should come up with our own
# namespace. XXX Misnomer--could be CAMP, too
self.cspad_img = evt.get(self.address)
if self.cspad_img is not None:
return
if self.address == 'XppGon-0|Cspad-0':
# Kludge until cspad_tbx.image() can be rewritten to handle the
# XPP metrology.
self.cspad_img = cspad_tbx.image_xpp(
self.address, evt, env, self.active_areas)
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
from psana import Source, Camera
import numpy as np
address = cspad_tbx.old_address_to_new_address(self.address)
src=Source('DetInfo(%s)'%address)
self.cspad_img = evt.get(Camera.FrameV1,src)
if self.cspad_img is not None:
self.cspad_img = self.cspad_img.data16().astype(np.float64)
elif self.address=='CxiDg3-0|Opal1000-0':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='CxiEndstation-0|Opal1000-2':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='FeeHxSpectrometer-0|Opal1000-1':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address=='NoDetector-0|Cspad2x2-0':
import numpy as np
from pypdsdata import xtc
test=[]
self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,self.address).data()
self.cspad_img=np.reshape(self.cspad_img,(370, 388))
else:
try:
self.cspad_img = cspad_tbx.image(
self.address, cspad_tbx.getConfig(self.address, env),
evt, env, self.sections)
except Exception, e:
self.logger.error("Error reading image data: " + str(e))
evt.put(skip_event_flag(), "skip_event")
return
def event(self, evt, env):
"""The event() function is called for every L1Accept transition.
@param evt Event data object, a configure object
@param env Environment object
"""
super(common_mode_correction, self).event(evt, env)
if (evt.get("skip_event")):
return
if not hasattr(self, 'active_areas') or self.active_areas is None or \
not hasattr(self, 'beam_center') or self.beam_center is None:
if self.address == 'XppGon-0|marccd-0':
# The mod_mar module needs to have been called before this one
# to set this up. The MAR does not have a configure object.
self.beam_center = evt.get("marccd_beam_center")
self.active_areas = evt.get("marccd_active_areas")
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
pass # bc and aa set in the beginjob function
elif self.address == 'XppGon-0|Cspad-0':
# Load the active areas as determined from the optical metrology
from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp
from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas
version_lookup = detector_format_version(
self.address,
reverse_timestamp(self.timestamp)[0])
assert version_lookup is not None
self.active_areas = xpp_active_areas[version_lookup][
'active_areas']
self.beam_center = [1765 // 2, 1765 // 2]
else:
(self.beam_center, self.active_areas) = \
cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections)
if self.filter_laser_1_status is not None:
if (self.laser_1_status.status != self.filter_laser_1_status or
(self.laser_1_ms_since_change is not None and
self.laser_1_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
if self.filter_laser_4_status is not None:
if (self.laser_4_status.status != self.filter_laser_4_status or
(self.laser_4_ms_since_change is not None and
self.laser_4_ms_since_change < self.filter_laser_wait_time)):
evt.put(skip_event_flag(), "skip_event")
return
# Early return if the full detector image is already stored in the
# event. Otherwise, get it from the stream as a double-precision
# floating-point flex array. XXX It is probably not safe to key
# the image on self.address, so we should come up with our own
# namespace. XXX Misnomer--could be CAMP, too
self.cspad_img = evt.get(self.address)
if self.cspad_img is not None:
return
if self.address == 'XppGon-0|Cspad-0':
# Kludge until cspad_tbx.image() can be rewritten to handle the
# XPP metrology.
self.cspad_img = cspad_tbx.image_xpp(self.address, evt, env,
self.active_areas)
elif self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0':
from psana import Source, Camera
import numpy as np
address = cspad_tbx.old_address_to_new_address(self.address)
src = Source('DetInfo(%s)' % address)
self.cspad_img = evt.get(Camera.FrameV1, src)
if self.cspad_img is not None:
self.cspad_img = self.cspad_img.data16().astype(np.float64)
elif self.address == 'CxiDg3-0|Opal1000-0':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'CxiEndstation-0|Opal1000-2':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'FeeHxSpectrometer-0|Opal1000-1':
if evt.getFrameValue(self.address) is not None:
self.cspad_img = evt.getFrameValue(self.address).data()
elif self.address == 'NoDetector-0|Cspad2x2-0':
import numpy as np
from pypdsdata import xtc
test = []
self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,
self.address).data()
self.cspad_img = np.reshape(self.cspad_img, (370, 388))
else:
try:
self.cspad_img = cspad_tbx.image(
self.address, cspad_tbx.getConfig(self.address, env), evt,
env, self.sections)
except Exception as e:
self.logger.error("Error reading image data: " + str(e))
evt.put(skip_event_flag(), "skip_event")
return
if self.cspad_img is None:
if cspad_tbx.address_split(self.address)[2] != 'Andor':
self.nfail += 1
self.logger.warning("event(): no image, shot skipped")
evt.put(skip_event_flag(), "skip_event")
return
self.cspad_img = flex.double(self.cspad_img.astype(numpy.float64))
# If a dark image was provided, subtract it from the image. There
# is no point in doing common-mode correction unless the dark
# image was subtracted.
if (self.dark_img is not None):
self.cspad_img -= self.dark_img
if (self.common_mode_correction != "none"):
# Mask out inactive pixels prior to common mode correction.
# Pixels are marked as inactive either due to low ADU values
# or non-positive standard deviations in dark image. XXX Make
# the threshold tunable?
cspad_mask = self.dark_mask.deep_copy()
if self.roi is not None and self.common_mode_correction == "chebyshev":
roi_mask = cspad_mask[self.roi[2]:self.roi[3], :]
roi_mask = flex.bool(roi_mask.accessor(), False)
cspad_mask.matrix_paste_block_in_place(block=roi_mask,
i_row=self.roi[2],
i_column=0)
# Extract each active section from the assembled detector
# image and apply the common mode correction. XXX Make up a
# quadrant mask for the emission detector. Needs to be
# checked!
config = cspad_tbx.getConfig(self.address, env)
if len(self.sections) == 1:
q_mask = 1
else:
q_mask = config.quadMask()
for q in range(len(self.sections)):
if (not ((1 << q) & q_mask)):
continue
# XXX Make up section mask for the emission detector. Needs
# to be checked!
import _pdsdata
if len(self.sections) == 1 and type(config) in (
_pdsdata.cspad2x2.ConfigV1,
_pdsdata.cspad2x2.ConfigV2):
s_mask = config.roiMask()
else:
s_mask = config.roiMask(q)
for s in range(len(self.sections[q])):
# XXX DAQ misconfiguration? This mask appears not to work
# reliably for the Sc1 detector.
# if (not((1 << s) & s_mask)):
# continue
corners = self.sections[q][s].corners()
i_row = int(round(min(c[0] for c in corners)))
i_column = int(round(min(c[1] for c in corners)))
n_rows = int(round(max(c[0] for c in corners))) - i_row
n_columns = int(round(max(
c[1] for c in corners))) - i_column
section_img = self.cspad_img.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
section_mask = cspad_mask.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
section_stddev = self.dark_stddev.matrix_copy_block(
i_row=i_row,
i_column=i_column,
n_rows=n_rows,
n_columns=n_columns)
if section_mask.count(True) == 0: continue
if self.common_mode_correction == "chebyshev":
assert len(self.sections[q]) == 2
if s == 0:
section_imgs = [section_img]
section_masks = [section_mask]
i_rows = [i_row]
i_columns = [i_column]
continue
else:
section_imgs.append(section_img)
section_masks.append(section_mask)
i_rows.append(i_row)
i_columns.append(i_column)
chebyshev_corrected_imgs = self.chebyshev_common_mode(
section_imgs, section_masks)
for i in range(2):
section_imgs[i].as_1d().copy_selected(
section_masks[i].as_1d().iselection(),
chebyshev_corrected_imgs[i].as_1d())
self.cspad_img.matrix_paste_block_in_place(
block=section_imgs[i],
i_row=i_rows[i],
i_column=i_columns[i])
else:
common_mode = self.common_mode(
section_img, section_stddev, section_mask)
self.sum_common_mode += common_mode
self.sumsq_common_mode += common_mode**2
# Apply the common mode correction to the
# section, and paste it back into the image.
self.cspad_img.matrix_paste_block_in_place(
block=section_img - common_mode,
i_row=i_row,
i_column=i_column)
if self.gain_map is not None:
self.cspad_img *= self.gain_map
if (self.mask_img is not None):
sel = (self.mask_img == -2) | (self.mask_img
== cspad_tbx.cspad_mask_value)
self.cspad_img.set_selected(sel, cspad_tbx.cspad_mask_value)
if (self.address == 'XppEndstation-0|Rayonix-0' or \
self.address == 'MfxEndstation-0|Rayonix-0') and \
self.crop_rayonix:
# Crop the masked data so that the beam center is in the center of the image
self.cspad_img = self.cspad_img[self.rayonix_crop_slice[0],
self.rayonix_crop_slice[1]]
if self.cache_image:
# Store the image in the event.
evt.put(self.cspad_img, self.address)
doplots = False
for data in generate_data_from_streams(args, verbose=True):
if 'TIMESTAMP' in data:
# this is how FormatPYunspecified guesses the address
if not "DETECTOR_ADDRESS" in data:
# legacy format; try to guess the address
LCLS_detector_address = 'CxiDs1-0|Cspad-0'
if "DISTANCE" in data and data["DISTANCE"] > 1000:
# downstream CS-PAD detector station of CXI instrument
LCLS_detector_address = 'CxiDsd-0|Cspad-0'
else:
LCLS_detector_address = data["DETECTOR_ADDRESS"]
detector_format_version = detector_format_function(
LCLS_detector_address, reverse_timestamp(data['TIMESTAMP'])[0])
print "Detector format version:", detector_format_version
image_pickle = True
else:
print "Not an image pickle"
image_pickle = False
keywise_printout(data)
if image_pickle:
import dxtbx
image = dxtbx.load(path)
tile_manager = image.detectorbase.get_tile_manager(image.detectorbase.horizons_phil_cache)
tiling = tile_manager.effective_tiling_as_flex_int(reapply_peripheral_margin = True)
print int(len(tiling)/4), "translated active areas, first one: ", list(tiling[0:4])
def run(params):
counter = 0
root = params.input_path
fig_object = plt.figure()
good_total = fail_total = 0
all_psanats = []
all_deltas = []
fail_deltas = []
good_deltas = []
for filename in os.listdir(root):
if os.path.splitext(filename)[1] != '.txt': continue
if 'debug' not in filename: continue
reference = None
fail_timepoints = []
good_timepoints = []
rank = int(filename.split('_')[1].split('.')[0])
counter += 1
print(filename)
run_timepoints = []
for line in open(os.path.join(root, filename)):
try:
hostname, psanats, ts, status, result = line.strip().split(',')
except ValueError:
continue
if reference is None:
sec, ms = reverse_timestamp(ts)
reference = sec + ms * 1e-3
run_timepoints.append(0)
assert status not in ['stop', 'done', 'fail']
if status in ['stop', 'done', 'fail']:
sec, ms = reverse_timestamp(ts)
run_timepoints.append((sec + ms * 1.e-3) - reference)
if status == 'done':
good_timepoints.append((sec + ms * 1.e-3) - reference)
good_deltas.append(good_timepoints[-1] -
run_timepoints[-2])
else:
fail_timepoints.append((sec + ms * 1.e-3) - reference)
fail_deltas.append(fail_timepoints[-1] -
run_timepoints[-2])
all_psanats.append(psanats)
all_deltas.append(run_timepoints[-1] - run_timepoints[-2])
ok = True
else:
ok = False
plt.plot(fail_timepoints, [rank] * len(fail_timepoints), 'b.')
plt.plot(good_timepoints, [rank] * len(good_timepoints), 'g.')
fail_total += len(fail_timepoints)
good_total += len(good_timepoints)
if not ok:
sec, ms = reverse_timestamp(ts)
plt.plot([(sec + ms * 1e-3) - reference], [rank], 'rx')
#if counter > 100: break
if fail_deltas:
print(
"Five number summary of %d fail image processing times:" %
fail_total, five_number_summary(flex.double(fail_deltas)))
if good_deltas:
print(
"Five number summary of %d good image processing times:" %
good_total, five_number_summary(flex.double(good_deltas)))
if params.wall_time and params.num_nodes and params.num_cores_per_node - 0.5:
for i in range(params.num_nodes):
plt.plot([0, params.wall_time], [
i * params.num_cores_per_node - 0.5,
i * params.num_cores_per_node - 0.5
], 'r-')
plt.xlabel('Wall time (sec)')
plt.ylabel('MPI Rank Number')
plt.title(params.plot_title)
if params.pickle_plot:
from libtbx.easy_pickle import dump
dump('%s' % params.pickle_filename, fig_object)
if params.show_plot:
plt.show()
