def _start(self): import os if isinstance(self._image_file, basestring) and os.path.isfile( self._image_file): stream = FormatPYunspecified.open_file(self._image_file, 'rb') import cPickle as pickle data = pickle.load(stream) else: data = self._image_file if not "DETECTOR_ADDRESS" in data: # legacy format; try to guess the address self.LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in data and data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument self.LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: self.LCLS_detector_address = data["DETECTOR_ADDRESS"] from iotbx.detectors.cspad_detector_formats import reverse_timestamp self._timesec = reverse_timestamp(data["TIMESTAMP"])[0] from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function version_lookup = detector_format_function(self.LCLS_detector_address, self._timesec) self.start_helper(version_token="distl.detector_format_version=%s" % version_lookup)
def get_timing_info(root, rank=0, size=1): steps_d = {} for ii, filename in enumerate(os.listdir(root)): if os.path.splitext(filename)[1] != ".txt": continue if (ii + rank) % size != 0: continue current_ts = "" prev_step = "" prev_time = None with open(os.path.join(root, filename)) as logfile: for line in logfile: try: hostname, ts, now, status, step = line.split(',') except ValueError: print 'ERROR', line raise now_s, now_ms = reverse_timestamp(now) now = now_s + (1e-3 * now_ms) #from IPython import embed; embed(); exit() if step.strip() == 'start': if prev_time is not None: #print prev_step.strip(), "took", now - prev_time, "seconds" steps_d = add_step(prev_step, now - prev_time, steps_d, ts, filename, step) else: #print prev_step.strip(), "took", now - prev_time, "seconds" steps_d = add_step(prev_step, now - prev_time, steps_d, ts, filename, step) current_ts = ts prev_step = step prev_time = now #if 'index' in step: # print 'FWWWWW',step,filename,ts #print steps_d.keys() return steps_d
def _start(self): if isinstance(self._image_file, six.string_types) and os.path.isfile( self._image_file): with FormatPYunspecified.open_file(self._image_file, "rb") as fh: if six.PY3: data = pickle.load(fh, encoding="bytes") data = image_dict_to_unicode(data) else: data = pickle.load(fh) else: data = self._image_file if "DETECTOR_ADDRESS" not in data: # legacy format; try to guess the address self.LCLS_detector_address = "CxiDs1-0|Cspad-0" if "DISTANCE" in data and data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument self.LCLS_detector_address = "CxiDsd-0|Cspad-0" else: self.LCLS_detector_address = data["DETECTOR_ADDRESS"] self._timesec = reverse_timestamp(data["TIMESTAMP"])[0] version_lookup = detector_format_version(self.LCLS_detector_address, self._timesec) self.start_helper(version_token="distl.detector_format_version=%s" % version_lookup)
def distl_filter(self, address, cspad_img, distance, timestamp, wavelength): self.hitfinder_d["DATA"] = cspad_img self.hitfinder_d["DISTANCE"] = distance self.hitfinder_d["TIMESTAMP"] = timestamp self.hitfinder_d["WAVELENGTH"] = wavelength self.hitfinder_d["DETECTOR_ADDRESS"] = address args = ["indexing.data=dummy", "distl.bins.verbose=False", self.asic_filter, ] detector_format_version = detector_format_function( address, reverse_timestamp(timestamp)[0]) args += ["distl.detector_format_version=%s" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = self.hitfinder_d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil from rstbx.new_horizons.index import pre_indexing_validation,pack_names pre_indexing_validation(horizons_phil) imagefile_arguments = pack_names(horizons_phil) from spotfinder.applications import signal_strength info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments) imgdata = info.Files.images[0].linearintdata active_data = self.get_active_data(info.Files.images[0],horizons_phil) peak_heights = flex.int( [ imgdata[ spot.max_pxl_x(), spot.max_pxl_y() ] for spot in info.S.images[info.frames[0]]["spots_total"] ]) outscale = 256 corrected = peak_heights.as_double() * self.correction outvalue = outscale *(1.0-corrected) outvalue.set_selected(outvalue<0.0,0.) outvalue.set_selected(outvalue>=outscale,int(outscale)-1) outvalue = flex.int(outvalue.as_numpy_array().astype(numpy.int32)) # essentially, select a peak if the peak's ADU value is > 2.5 * the 90-percentile pixel value #work = display_spots.wrapper_of_callback(info) #work.display_with_callback(horizons_phil.indexing.data) return peak_heights,outvalue
def distl_filter(self, address, cspad_img, distance, timestamp, wavelength): self.hitfinder_d["DATA"] = cspad_img self.hitfinder_d["DISTANCE"] = distance self.hitfinder_d["TIMESTAMP"] = timestamp self.hitfinder_d["WAVELENGTH"] = wavelength self.hitfinder_d["DETECTOR_ADDRESS"] = address args = ["indexing.data=dummy", "distl.bins.verbose=False", self.asic_filter, ] detector_format_version = detector_format_function( address, reverse_timestamp(timestamp)[0]) args += ["distl.detector_format_version=%s" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = self.hitfinder_d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil from rstbx.new_horizons.index import pre_indexing_validation,pack_names pre_indexing_validation(horizons_phil) imagefile_arguments = pack_names(horizons_phil) from spotfinder.applications import signal_strength info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments) imgdata = info.Files.images[0].linearintdata active_data = self.get_active_data(info.Files.images[0],horizons_phil) peak_heights = flex.int( [ imgdata[ spot.max_pxl_x(), spot.max_pxl_y() ] for spot in info.S.images[info.frames[0]]["spots_total"] ]) outscale = 256 corrected = peak_heights.as_double() * self.correction outvalue = outscale *(1.0-corrected) outvalue.set_selected(outvalue<0.0,0.) outvalue.set_selected(outvalue>=outscale,int(outscale)-1) outvalue = flex.int(outvalue.as_numpy_array().astype(numpy.int32)) # essentially, select a peak if the peak's ADU value is > 2.5 * the 90-percentile pixel value #work = display_spots.wrapper_of_callback(info) #work.display_with_callback(horizons_phil.indexing.data) return peak_heights,outvalue
def integrate_one_image(data, **kwargs): from xfel.cxi.display_spots import run_one_index_core from labelit.dptbx.error import NoAutoIndex from libtbx.utils import Sorry from spotfinder.exception import SpotfinderError from labelit.exception import AutoIndexError from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function from iotbx.detectors.cspad_detector_formats import reverse_timestamp basename = kwargs.get("integration_basename") if (basename is None): basename = "" dirname = kwargs.get("integration_dirname") if (dirname is None): dirname = "integration" if (not os.path.isdir(dirname)): import errno try: os.makedirs(dirname) except OSError as exc: if exc.errno==errno.EEXIST: pass path = os.path.join(dirname, basename \ + data['TIMESTAMP'] \ + ("_%05d.pickle" % data['SEQUENCE_NUMBER'])) args = ["indexing.data=dummy", "beam_search_scope=0.5", "lepage_max_delta = 3.0", "spots_pickle = None", "subgroups_pickle = None", "refinements_pickle = None", "rmsd_tolerance = 5.0", "mosflm_rmsd_tolerance = 5.0", "indexing.completeness_pickle=%s"%path, "difflimit_sigma_cutoff=2.0", #"indexing.open_wx_viewer=True" ] detector_format_version = detector_format_function( data['DETECTOR_ADDRESS'], reverse_timestamp(data['TIMESTAMP'])[0]) args += ["distl.detector_format_version=%s" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(data["xtal_target"], args) horizons_phil.indexing.data = data print "XFEL processing: %s"%path try: return run_one_index_core(horizons_phil) except NoAutoIndex,e: print "NoAutoIndex", data['TIMESTAMP'], e info = e.info
def run(params): counter = 0 reference = None root=os.path.join(params.input_path, 'out', 'debug') fig_object = plt.figure() for filename in os.listdir(root): if os.path.splitext(filename)[1] != '.txt': continue if 'debug' not in filename: continue timepoints = [] rank = int(filename.split('_')[1].split('.')[0]) counter += 1 print (filename) for line in open(os.path.join(root,filename)): hostname, psanats, ts, status, result = line.strip().split(',') if reference is None: sec, ms = reverse_timestamp(ts) reference = sec+ms*1e-3 if status in ['stop','done']: sec, ms = reverse_timestamp(ts) timepoints.append((sec + ms*1.e-3)-reference) ok = True else: ok = False plt.plot(timepoints, [rank]*len(timepoints), 'b.') if not ok: sec, ms = reverse_timestamp(ts) plt.plot([(sec+ms*1e-3) - reference], [rank], 'rx') #if counter > 100: break for i in range(params.num_nodes): plt.plot([0,params.wall_time], [i*params.num_cores_per_node-0.5, i*params.num_cores_per_node-0.5], 'r-') plt.xlabel('Wall time (sec)') plt.ylabel('MPI Rank Number') plt.title(params.plot_title) if params.pickle_plot: from libtbx.easy_pickle import dump dump('%s'%params.pickle_filename, fig_object) if params.show_plot: plt.show()
def _start(self): import os if isinstance(self._image_file, basestring) and os.path.isfile(self._image_file): stream = FormatPYunspecified.open_file(self._image_file, 'rb') import cPickle as pickle data = pickle.load(stream) else: data = self._image_file if not "DETECTOR_ADDRESS" in data: # legacy format; try to guess the address self.LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in data and data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument self.LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: self.LCLS_detector_address = data["DETECTOR_ADDRESS"] from iotbx.detectors.cspad_detector_formats import reverse_timestamp self._timesec = reverse_timestamp( data["TIMESTAMP"] )[0] from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function version_lookup = detector_format_function(self.LCLS_detector_address,self._timesec) self.start_helper(version_token="distl.detector_format_version=%s"%version_lookup)
def get_hits_and_indexing_stats(filenames, debug_root, rank=0): print('starting hits_and_indexing_stats', rank) num_of_xray_events = 0 num_of_images_analyzed = 0 hits = [] events_list = [] indexing_time = {} indexing_time_all = {} # Both for failed and successful indexing trials current_ts = "" current_reverse_ts = "" prev_step = "" prev_time = None for filename in filenames: with open(os.path.join(debug_root, filename)) as logfile: for line in logfile: try: if line == '\n': continue hostname, ts, now, status, step = line.strip().split(',') except ValueError as e: print(line) raise now_s, now_ms = reverse_timestamp(now) now = now_s + (1e-3 * now_ms) if step.strip() == 'spotfind_start': num_of_images_analyzed += 1 if step.strip() == 'start': num_of_xray_events += 1 curr_ts, curr_ms = reverse_timestamp(ts) current_reverse_ts = curr_ts + (1e-3 * curr_ms) #if '2018-05-01T14:50Z21.976' == ts: # print (line + 'THIS MIGHT BE A DUPLICATE\n') events_list.append(ts) #if prev_time is not None: #print (prev_step.strip(), "took", now - prev_time, "seconds") #add_step(prev_step, now-prev_time) #else: #print (prev_step.strip(), "took", now - prev_time, "seconds") #add_step(prev_step, now-prev_time) if prev_step.strip() == 'index_start': indexing_time_all[ts] = now - prev_time if step.strip() == 'index_start': hits.append(ts) if step.strip() == 'refine_start': indexing_time[ts] = now - prev_time #print ('DEBUG',prev_step, step, now,prev_time, ts) current_ts = ts prev_step = step prev_time = now #print steps_d.keys() #print indexing_time.keys() #print len(hits) total_idx_time = 0 idx_cutoff_time_exceeded_event = [] for event in indexing_time.keys(): # print event, indexing_time[event] total_idx_time += indexing_time[event] recorded_hits = [] idx_attempt_time = [] idx_successful_time = [] if params.write_out_timings: fout = open('indexing_timing_' + str(rank) + '.dat', 'w') fout.write( 'Event Number hits indexed t_indexed t_indexed_attempted \n' ) for ii, event in enumerate(events_list): if event in hits: is_hit = 1 if event not in recorded_hits: recorded_hits.append(event) else: if debug_mode: print('Duplicate Event ? = ', event) else: is_hit = 0 if event in indexing_time: is_idx = 1 t_idx = indexing_time[event] idx_successful_time.append(t_idx) assert event in indexing_time_all, 'Event not present in indexing_time_all' t_idx2 = indexing_time_all[event] idx_attempt_time.append(t_idx2) elif event in indexing_time_all: is_idx = 0.5 assert event not in indexing_time, 'Event should not be present in indexing_time' t_idx = 0.0 t_idx2 = indexing_time_all[event] idx_attempt_time.append(t_idx2) if t_idx2 > indexing_time_cutoff: idx_cutoff_time_exceeded_event.append(event) else: is_idx = 0 t_idx = 0.0 t_idx2 = 0.0 if write_out_timings: fout.write('%s %3.1f %3.1f %12.7f %12.7f\n' % (event, is_hit, is_idx, t_idx, t_idx2)) if write_out_timings: fout.close() return (len(hits), len(idx_successful_time), sum(idx_attempt_time) / 3600.0, sum(idx_successful_time) / 3600.0, idx_cutoff_time_exceeded_event, num_of_xray_events, num_of_images_analyzed)
def beginjob(self, evt, env): """The beginjob() function does one-time initialisation from event- or environment data. It is called at an XTC configure transition. @param evt Event data object, a configure object @param env Environment object """ super(common_mode_correction, self).beginjob(evt, env) # Load the dark image and ensure it is signed and at least 32 bits # wide, since it will be used for differencing. If a dark image # is provided, a standard deviation image is required, and all the # ADU scales must match up. # # XXX Can we zap all the ADU_SCALE stuff? # # XXX Do we really need to store the substituted values in the # instance here? At least self._dark_path is referenced later on. # # Note that this will load the dark, standard deviation and gain # once (SEVERAL TIMES) for each process, but the gain is that we # can do substitutions. But it is only done at the beginning of # the job. self.dark_img = None if self._dark_path is not None: self._dark_path = cspad_tbx.getOptEvalOrString( cspad_tbx.pathsubst(self._dark_path, evt, env)) assert self._dark_stddev_path is not None dark_dict = easy_pickle.load(self._dark_path) #assert "ADU_SCALE" not in dark_dict # force use of recalculated dark self.dark_img = dark_dict['DATA'] assert isinstance(self.dark_img, flex.double) self._dark_stddev_path = cspad_tbx.getOptEvalOrString( cspad_tbx.pathsubst(self._dark_stddev_path, evt, env)) self.dark_stddev = easy_pickle.load(self._dark_stddev_path)['DATA'] assert isinstance(self.dark_stddev, flex.double) self.dark_mask = (self.dark_stddev > 0) # Load the mask image and ensure it is signed and at least 32 bits # wide, since it will be used for differencing. self.gain_map = None if self._gain_map_path is not None: self._gain_map_path = cspad_tbx.getOptEvalOrString( cspad_tbx.pathsubst(self._gain_map_path, evt, env)) self.gain_map = easy_pickle.load(self._gain_map_path)['DATA'] if self.gain_map_level is not None: sel = flex.bool([bool(f) for f in self.gain_map]) sel.reshape(flex.grid(self.gain_map.focus())) self.gain_map = self.gain_map.set_selected(~sel, self.gain_map_level) self.gain_map = self.gain_map.set_selected(sel, 1) assert isinstance(self.gain_map, flex.double) self.mask_img = None if self._mask_path is not None: self._mask_path = cspad_tbx.getOptEvalOrString( cspad_tbx.pathsubst(self._mask_path, evt, env)) self.mask_img = easy_pickle.load(self._mask_path)['DATA'] assert isinstance(self.mask_img, flex.double) \ or isinstance(self.mask_img, flex.int) if self.address == 'XppGon-0|marccd-0': #mod_mar.py will set these during its event function self.active_areas = None self.beam_center = None elif self.address == 'XppEndstation-0|Rayonix-0' or \ self.address == 'MfxEndstation-0|Rayonix-0': assert self.override_beam_x is not None assert self.override_beam_y is not None from xfel.cxi.cspad_ana import rayonix_tbx maxx, maxy = rayonix_tbx.get_rayonix_detector_dimensions(self.bin_size) if self.crop_rayonix: bx = int(round(self.override_beam_x)) by = int(round(self.override_beam_y)) minsize = min([bx,by,maxx-bx,maxy-by]) self.beam_center = minsize,minsize self.active_areas = flex.int([0,0,2*minsize,2*minsize]) self.rayonix_crop_slice = slice(by-minsize,by+minsize), slice(bx-minsize,bx+minsize) else: self.beam_center = self.override_beam_x,self.override_beam_y self.active_areas = flex.int([0,0,maxx,maxy]) elif self.address == 'XppGon-0|Cspad-0': evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version(self.address, reverse_timestamp(timestamp)[0]) assert version_lookup is not None self.active_areas = xpp_active_areas[version_lookup]['active_areas'] self.beam_center = [1765 // 2, 1765 // 2] else: (self.beam_center, self.active_areas) = cspad_tbx.cbcaa( cspad_tbx.getConfig(self.address, env), self.sections)
def __call__(self): from iotbx.detectors.cspad_detector_formats import reverse_timestamp from xfel.ui.components.timeit import duration import time t1 = time.time() run_numbers = [r.run for r in self.trial.runs] assert self.run.run in run_numbers rungroup_ids = [rg.id for rg in self.trial.rungroups] assert self.rungroup.id in rungroup_ids if len(self.trial.isoforms) > 0: cells = [isoform.cell for isoform in self.trial.isoforms] else: cells = self.app.get_trial_cells(self.trial.id, self.rungroup.id, self.run.id) low_res_bin_ids = [] for cell in cells: bins = cell.bins d_mins = [float(b.d_min) for b in bins] if len(d_mins) == 0: continue low_res_bin_ids.append(str(bins[d_mins.index(max(d_mins))].id)) tag = self.app.params.experiment_tag timestamps = flex.double() xtal_ids = flex.double() n_strong = flex.int() if len(low_res_bin_ids) > 0: # Get the spotfinding results from the selected runs query = """SELECT bin.id, crystal.id, event.timestamp, event.n_strong FROM `%s_event` event JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id JOIN `%s_imageset` imgset ON imgset.id = is_e.imageset_id JOIN `%s_experiment` exp ON exp.imageset_id = imgset.id JOIN `%s_crystal` crystal ON crystal.id = exp.crystal_id JOIN `%s_cell` cell ON cell.id = crystal.cell_id JOIN `%s_bin` bin ON bin.cell_id = cell.id JOIN `%s_cell_bin` cb ON cb.bin_id = bin.id AND cb.crystal_id = crystal.id WHERE event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d AND cb.bin_id IN (%s) """ % (tag, tag, tag, tag, tag, tag, tag, tag, self.trial.id, self.run.id, self.rungroup.id, ", ".join(low_res_bin_ids)) cursor = self.app.execute_query(query) sample = -1 for row in cursor.fetchall(): b_id, xtal_id, ts, n_s = row rts = reverse_timestamp(ts) rts = rts[0] + (rts[1] / 1000) if xtal_id not in xtal_ids: sample += 1 if sample % self.sampling != 0: continue timestamps.append(rts) xtal_ids.append(xtal_id) n_strong.append(n_s) # This left join query finds the events with no imageset, meaning they failed to index query = """SELECT event.timestamp, event.n_strong FROM `%s_event` event LEFT JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id WHERE is_e.event_id IS NULL AND event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d """ % (tag, tag, self.trial.id, self.run.id, self.rungroup.id) cursor = self.app.execute_query(query) for row in cursor.fetchall(): ts, n_s = row rts = reverse_timestamp(ts) timestamps.append(rts[0] + (rts[1] / 1000)) n_strong.append(n_s) order = flex.sort_permutation(timestamps) timestamps = timestamps.select(order) n_strong = n_strong.select(order) t2 = time.time() return timestamps, n_strong
if params.output_file is None: logger = sys.stdout else: logger = open(params.output_file, 'w') logger.write("%s " % params.output_file) if not "DETECTOR_ADDRESS" in source_data: # legacy format; try to guess the address LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in source_data and source_data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: LCLS_detector_address = source_data["DETECTOR_ADDRESS"] timesec = reverse_timestamp(source_data["TIMESTAMP"])[0] version_lookup = detector_format_function(LCLS_detector_address, timesec) args = [ "distl.detector_format_version=%s" % version_lookup, "viewer.powder_arcs.show=False", "viewer.powder_arcs.code=3n9c", ] horizons_phil = cxi_phil.cxi_versioned_extract(args).persist.commands img = NpyImage(params.file_path, source_data) img.readHeader(horizons_phil) img.translate_tiles(horizons_phil) if params.verbose: img.show_header()
def average(argv=None): if argv == None: argv = sys.argv[1:] try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") command_line = (libtbx.option_parser.option_parser(usage=""" %s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath] To write image pickles use -p, otherwise the program writes CSPAD CBFs. Writing CBFs requires the geometry to be already deployed. Examples: cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 Use one process on the current node to process all the events from run 25 of experiment cxi49812, using a detz_offset of 571. mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 As above, using 16 cores on the current node. bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812 As above, using the psanaq and 100 cores, putting the log in average.out and the output images in the folder cxi49812. """ % libtbx.env.dispatcher_name).option( None, "--as_pickle", "-p", action="store_true", default=False, dest="as_pickle", help="Write results as image pickle files instead of cbf files" ).option( None, "--raw_data", "-R", action="store_true", default=False, dest="raw_data", help= "Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)" ).option( None, "--background_pickle", "-B", default=None, dest="background_pickle", help="" ).option( None, "--config", "-c", type="string", default=None, dest="config", metavar="PATH", help="psana config file" ).option( None, "--experiment", "-x", type="string", default=None, dest="experiment", help="experiment name (eg cxi84914)" ).option( None, "--run", "-r", type="int", default=None, dest="run", help="run number" ).option( None, "--address", "-a", type="string", default="CxiDs2.0:Cspad.0", dest="address", help="detector address name (eg CxiDs2.0:Cspad.0)" ).option( None, "--detz_offset", "-d", type="float", default=None, dest="detz_offset", help= "offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)" ).option( None, "--outputdir", "-o", type="string", default=".", dest="outputdir", metavar="PATH", help="Optional path to output directory for output files" ).option( None, "--averagebase", "-A", type="string", default="{experiment!l}_avg-r{run:04d}", dest="averagepath", metavar="PATH", help= "Path to output average image without extension. String substitution allowed" ).option( None, "--stddevbase", "-S", type="string", default="{experiment!l}_stddev-r{run:04d}", dest="stddevpath", metavar="PATH", help= "Path to output standard deviation image without extension. String substitution allowed" ).option( None, "--maxbase", "-M", type="string", default="{experiment!l}_max-r{run:04d}", dest="maxpath", metavar="PATH", help= "Path to output maximum projection image without extension. String substitution allowed" ).option( None, "--numevents", "-n", type="int", default=None, dest="numevents", help="Maximum number of events to process. Default: all" ).option( None, "--skipevents", "-s", type="int", default=0, dest="skipevents", help="Number of events in the beginning of the run to skip. Default: 0" ).option( None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress" ).option( None, "--pickle-optical-metrology", "-m", action="store_true", default=False, dest="pickle_optical_metrology", help= "If writing pickle files, use the optical metrology in the experiment's calib directory" ).option( None, "--bin_size", "-b", type="int", default=None, dest="bin_size", help="Rayonix detector bin size" ).option( None, "--override_beam_x", "-X", type="float", default=None, dest="override_beam_x", help="Rayonix detector beam center x coordinate" ).option( None, "--override_beam_y", "-Y", type="float", default=None, dest="override_beam_y", help="Rayonix detector beam center y coordinate" ).option( None, "--calib_dir", "-C", type="string", default=None, dest="calib_dir", metavar="PATH", help="calibration directory" ).option( None, "--pickle_calib_dir", "-P", type="string", default=None, dest="pickle_calib_dir", metavar="PATH", help= "pickle calibration directory specification. Replaces --calib_dir functionality." ).option( None, "--xtc_dir", "-D", type="string", default=None, dest="xtc_dir", metavar="PATH", help="xtc stream directory" ).option( None, "--use_ffb", "-f", action="store_true", default=False, dest="use_ffb", help= "Use the fast feedback filesystem at LCLS. Only for the active experiment!" ).option( None, "--gain_mask_value", "-g", type="float", default=None, dest="gain_mask_value", help= "Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode." ).option( None, "--min", None, action="store_true", default=False, dest="do_minimum_projection", help="Output a minimum projection" ).option( None, "--minpath", None, type="string", default="{experiment!l}_min-r{run:04d}", dest="minpath", metavar="PATH", help= "Path to output minimum image without extension. String substitution allowed" )).process(args=argv) if len(command_line.args) > 0 or \ command_line.options.as_pickle is None or \ command_line.options.experiment is None or \ command_line.options.run is None or \ command_line.options.address is None or \ command_line.options.detz_offset is None or \ command_line.options.averagepath is None or \ command_line.options.stddevpath is None or \ command_line.options.maxpath is None or \ command_line.options.pickle_optical_metrology is None: command_line.parser.show_help() return # set this to sys.maxint to analyze all events if command_line.options.numevents is None: maxevents = sys.maxsize else: maxevents = command_line.options.numevents comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if command_line.options.config is not None: psana.setConfigFile(command_line.options.config) dataset_name = "exp=%s:run=%d:smd" % (command_line.options.experiment, command_line.options.run) if command_line.options.xtc_dir is not None: if command_line.options.use_ffb: raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system") dataset_name += ":dir=%s" % command_line.options.xtc_dir elif command_line.options.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc" % ( command_line.options.experiment[0:3], command_line.options.experiment) if command_line.options.calib_dir is not None: psana.setOption('psana.calib-dir', command_line.options.calib_dir) ds = psana.DataSource(dataset_name) address = command_line.options.address src = psana.Source('DetInfo(%s)' % address) nevent = np.array([0.]) if command_line.options.background_pickle is not None: background = easy_pickle.load( command_line.options.background_pickle)['DATA'].as_numpy_array() for run in ds.runs(): runnumber = run.run() if not command_line.options.as_pickle: psana_det = psana.Detector(address, ds.env()) # list of all events if command_line.options.skipevents > 0: print("Skipping first %d events" % command_line.options.skipevents) elif "Rayonix" in command_line.options.address: print("Skipping first image in the Rayonix detector" ) # Shuttering issue command_line.options.skipevents = 1 for i, evt in enumerate(run.events()): if i % size != rank: continue if i < command_line.options.skipevents: continue if i >= maxevents: break if i % 10 == 0: print('Rank', rank, 'processing event', i) #print "Event #",rank*mylength+i," has id:",evt.get(EventId) if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: data = evt.get(psana.Camera.FrameV1, src) if data is None: print("No data") continue data = data.data16().astype(np.float64) elif command_line.options.as_pickle: data = evt.get(psana.ndarray_float64_3, src, 'image0') else: # get numpy array, 32x185x388 from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data if command_line.options.raw_data: data = get_psana_corrected_data(psana_det, evt, use_default=False, dark=False, common_mode=None, apply_gain_mask=False, per_pixel_gain=False) else: if command_line.options.gain_mask_value is None: data = get_psana_corrected_data(psana_det, evt, use_default=True) else: data = get_psana_corrected_data( psana_det, evt, use_default=False, dark=True, common_mode=None, apply_gain_mask=True, gain_mask_value=command_line.options. gain_mask_value, per_pixel_gain=False) if data is None: print("No data") continue if command_line.options.background_pickle is not None: data -= background if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: distance = np.array([0.0]) wavelength = np.array([1.0]) else: d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset) if d is None: print("No distance, using distance", command_line.options.detz_offset) assert command_line.options.detz_offset is not None if 'distance' not in locals(): distance = np.array([command_line.options.detz_offset]) else: distance += command_line.options.detz_offset else: if 'distance' in locals(): distance += d else: distance = np.array([float(d)]) w = cspad_tbx.evt_wavelength(evt) if w is None: print("No wavelength") if 'wavelength' not in locals(): wavelength = np.array([1.0]) else: if 'wavelength' in locals(): wavelength += w else: wavelength = np.array([w]) t = cspad_tbx.evt_time(evt) if t is None: print("No timestamp, skipping shot") continue if 'timestamp' in locals(): timestamp += t[0] + (t[1] / 1000) else: timestamp = np.array([t[0] + (t[1] / 1000)]) if 'sum' in locals(): sum += data else: sum = np.array(data, copy=True) if 'sumsq' in locals(): sumsq += data * data else: sumsq = data * data if 'maximum' in locals(): maximum = np.maximum(maximum, data) else: maximum = np.array(data, copy=True) if command_line.options.do_minimum_projection: if 'minimum' in locals(): minimum = np.minimum(minimum, data) else: minimum = np.array(data, copy=True) nevent += 1 #sum the images across mpi cores if size > 1: print("Synchronizing rank", rank) totevent = np.zeros(nevent.shape) comm.Reduce(nevent, totevent) if rank == 0 and totevent[0] == 0: raise Sorry("No events found in the run") sumall = np.zeros(sum.shape).astype(sum.dtype) comm.Reduce(sum, sumall) sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype) comm.Reduce(sumsq, sumsqall) maxall = np.zeros(maximum.shape).astype(maximum.dtype) comm.Reduce(maximum, maxall, op=MPI.MAX) if command_line.options.do_minimum_projection: minall = np.zeros(maximum.shape).astype(minimum.dtype) comm.Reduce(minimum, minall, op=MPI.MIN) waveall = np.zeros(wavelength.shape).astype(wavelength.dtype) comm.Reduce(wavelength, waveall) distall = np.zeros(distance.shape).astype(distance.dtype) comm.Reduce(distance, distall) timeall = np.zeros(timestamp.shape).astype(timestamp.dtype) comm.Reduce(timestamp, timeall) if rank == 0: if size > 1: print("Synchronized") # Accumulating floating-point numbers introduces errors, # which may cause negative variances. Since a two-pass # approach is unacceptable, the standard deviation is # clamped at zero. mean = sumall / float(totevent[0]) variance = (sumsqall / float(totevent[0])) - (mean**2) variance[variance < 0] = 0 stddev = np.sqrt(variance) wavelength = waveall[0] / totevent[0] distance = distall[0] / totevent[0] pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value timestamp = timeall[0] / totevent[0] timestamp = (int(timestamp), timestamp % int(timestamp) * 1000) timestamp = cspad_tbx.evt_timestamp(timestamp) if command_line.options.as_pickle: extension = ".pickle" else: extension = ".cbf" dest_paths = [ cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env()) ] if command_line.options.do_minimum_projection: dest_paths.append( cspad_tbx.pathsubst(command_line.options.minpath + extension, evt, ds.env())) dest_paths = [ os.path.join(command_line.options.outputdir, path) for path in dest_paths ] if 'Rayonix' in command_line.options.address: all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size( command_line.options.bin_size) beam_center = [ command_line.options.override_beam_x, command_line.options.override_beam_y ] active_areas = flex.int([0, 0, mean.shape[1], mean.shape[0]]) split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] for data, path in zip(all_data, dest_paths): print("Saving", path) d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=flex.double(data), distance=distance, pixel_size=pixel_size, saturated_value=rayonix_tbx.rayonix_saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: all_data = [mean, stddev, maxall] split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): d = cspad_tbx.dpack(address=old_style_address, data=flex.double(data), distance=distance, pixel_size=0.1, timestamp=timestamp, wavelength=wavelength) print("Saving", path) easy_pickle.dump(path, d) elif command_line.options.as_pickle: split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[ 1] + "|" + split_address[2] + "-" + split_address[3] xpp = 'xpp' in address.lower() if xpp: evt_time = cspad_tbx.evt_time( evt) # tuple of seconds, milliseconds timestamp = cspad_tbx.evt_timestamp( evt_time) # human readable format from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version( old_style_address, reverse_timestamp(timestamp)[0]) assert version_lookup is not None active_areas = xpp_active_areas[version_lookup]['active_areas'] beam_center = [1765 // 2, 1765 // 2] else: if command_line.options.pickle_calib_dir is not None: metro_path = command_line.options.pickle_calib_dir elif command_line.options.pickle_optical_metrology: from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path metro_path = get_calib_file_path(run.env(), address, run) else: metro_path = libtbx.env.find_in_repositories( "xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0") sections = parse_calib.calib2sections(metro_path) beam_center, active_areas = cspad_tbx.cbcaa( cspad_tbx.getConfig(address, ds.env()), sections) class fake_quad(object): def __init__(self, q, d): self.q = q self.d = d def quad(self): return self.q def data(self): return self.d if xpp: quads = [ fake_quad(i, mean[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [ fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [ fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [ fake_quad(i, minall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] minall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads=quads) minall = flex.double(minall.astype(np.float64)) else: quads = [ fake_quad(i, mean[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] mean = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [ fake_quad(i, stddev[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] stddev = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [ fake_quad(i, maxall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] maxall = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [ fake_quad(i, minall[i * 8:(i + 1) * 8, :, :]) for i in range(4) ] minall = cspad_tbx.CsPadDetector(address, evt, ds.env(), sections, quads=quads) minall = flex.double(minall.astype(np.float64)) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print("Saving", path) d = cspad_tbx.dpack(active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=data, distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) else: # load a header only cspad cbf from the slac metrology from xfel.cftbx.detector import cspad_cbf_tbx import pycbf base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology( run, address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d" % run.run()) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print("Saving", path) cspad_img = cspad_cbf_tbx.format_object_from_data( base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=False) cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
def __call__(self): from iotbx.detectors.cspad_detector_formats import reverse_timestamp from xfel.ui.components.timeit import duration #import time #t1 = time.time() run_numbers = [r.run for r in self.trial.runs] assert self.run.run in run_numbers rungroup_ids = [rg.id for rg in self.trial.rungroups] assert self.rungroup.id in rungroup_ids if len(self.trial.isoforms) > 0: cells = [isoform.cell for isoform in self.trial.isoforms] else: cells = self.app.get_trial_cells(self.trial.id, self.rungroup.id, self.run.id) high_res_bin_ids = [] for cell in cells: bins = cell.bins d_mins = [float(b.d_min) for b in bins] if len(d_mins) == 0: continue if self.d_min is None: min_bin_index = d_mins.index(min(d_mins)) else: d_maxes = [float(b.d_max) for b in bins] qualified_bin_indices = [ i for i in xrange(len(bins)) if d_maxes[i] >= self.d_min and d_mins[i] <= self.d_min ] if len(qualified_bin_indices) == 0: continue min_bin_index = qualified_bin_indices[0] high_res_bin_ids.append(str(bins[min_bin_index].id)) resolutions = flex.double() two_theta_low = flex.double() two_theta_high = flex.double() tag = self.app.params.experiment_tag timestamps = flex.double() n_strong = flex.int() n_lattices = flex.int() if len(high_res_bin_ids) > 0: # Get the stats in one query. query = """SELECT event.timestamp, event.n_strong, MIN(bin.d_min), event.two_theta_low, event.two_theta_high, COUNT(crystal.id) FROM `%s_event` event JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id JOIN `%s_imageset` imgset ON imgset.id = is_e.imageset_id JOIN `%s_experiment` exp ON exp.imageset_id = imgset.id JOIN `%s_crystal` crystal ON crystal.id = exp.crystal_id JOIN `%s_cell` cell ON cell.id = crystal.cell_id JOIN `%s_bin` bin ON bin.cell_id = cell.id JOIN `%s_cell_bin` cb ON cb.bin_id = bin.id AND cb.crystal_id = crystal.id WHERE event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d AND cb.avg_i_sigi >= %f GROUP BY event.id """ % (tag, tag, tag, tag, tag, tag, tag, tag, self.trial.id, self.run.id, self.rungroup.id, self.i_sigi_cutoff) cursor = self.app.execute_query(query) sample = -1 for row in cursor.fetchall(): ts, n_s, d_min, tt_low, tt_high, n_xtal = row try: d_min = float(d_min) except ValueError: d_min = None rts = reverse_timestamp(ts) rts = rts[0] + (rts[1] / 1000) sample += 1 if sample % self.sampling != 0: continue timestamps.append(rts) n_strong.append(n_s) two_theta_low.append(tt_low or -1) two_theta_high.append(tt_high or -1) resolutions.append(d_min or 0) n_lattices.append(n_xtal or 0) # This left join query finds the events with no imageset, meaning they failed to index query = """SELECT event.timestamp, event.n_strong, event.two_theta_low, event.two_theta_high FROM `%s_event` event LEFT JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id WHERE is_e.event_id IS NULL AND event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d """ % (tag, tag, self.trial.id, self.run.id, self.rungroup.id) cursor = self.app.execute_query(query) for row in cursor.fetchall(): ts, n_s, tt_low, tt_high = row rts = reverse_timestamp(ts) timestamps.append(rts[0] + (rts[1] / 1000)) n_strong.append(n_s) two_theta_low.append(tt_low or -1) two_theta_high.append(tt_high or -1) resolutions.append(0) n_lattices.append(0) order = flex.sort_permutation(timestamps) timestamps = timestamps.select(order) n_strong = n_strong.select(order) two_theta_low = two_theta_low.select(order) two_theta_high = two_theta_high.select(order) resolutions = resolutions.select(order) n_lattices = n_lattices.select(order) #t2 = time.time() #print "HitrateStats took %s" % duration(t1, t2) return timestamps, two_theta_low, two_theta_high, n_strong, resolutions, n_lattices
def run(argv=None): """Compute mean, standard deviation, and maximum projection images from a set of images given on the command line. @param argv Command line argument list @return @c 0 on successful termination, @c 1 on error, and @c 2 for command line syntax errors """ import libtbx.load_env from libtbx import easy_pickle, option_parser from scitbx.array_family import flex from xfel.cxi.cspad_ana import cspad_tbx from iotbx.detectors.cspad_detector_formats import reverse_timestamp if argv is None: argv = sys.argv command_line = (option_parser.option_parser( usage="%s [-v] [-a PATH] [-m PATH] [-s PATH] " \ "image1 image2 [image3 ...]" % libtbx.env.dispatcher_name) .option(None, "--average-path", "-a", type="string", default=None, dest="avg_path", metavar="PATH", help="Write average image to PATH") .option(None, "--maximum-path", "-m", type="string", default=None, dest="max_path", metavar="PATH", help="Write maximum projection image to PATH") .option(None, "--stddev-path", "-s", type="string", default=None, dest="stddev_path", metavar="PATH", help="Write standard deviation image to PATH") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") ).process(args=argv[1:]) # Note that it is not an error to omit the output paths, because # certain statistics could still be printed, e.g. with the verbose # option. paths = command_line.args if len(paths) == 0: command_line.parser.print_usage(file=sys.stderr) return 2 # Loop over all images and accumulate statistics. nfail = 0 nmemb = 0 for path in paths: if command_line.options.verbose: sys.stdout.write("Processing %s...\n" % path) try: # Promote the image to double-precision floating point type. # All real-valued flex arrays have the as_double() function. d = easy_pickle.load(path) distance = d['DISTANCE'] img = d['DATA'].as_1d().as_double() wavelength = d['WAVELENGTH'] time_tuple = reverse_timestamp(d['TIMESTAMP']) # Warn if the header items across the set of images do not match # up. Note that discrepancies regarding the image size are # fatal. if 'active_areas' in locals(): if (active_areas != d['ACTIVE_AREAS']).count(True) != 0: sys.stderr.write("Active areas do not match\n") else: active_areas = d['ACTIVE_AREAS'] if 'beam_center' in locals(): if beam_center != (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']): sys.stderr.write("Beam centers do not match\n") else: beam_center = (d['BEAM_CENTER_X'], d['BEAM_CENTER_Y']) if 'detector_address' in locals(): if detector_address != d['DETECTOR_ADDRESS']: sys.stderr.write("Detector addresses do not match\n") else: detector_address = d['DETECTOR_ADDRESS'] if 'saturated_value' in locals(): if saturated_value != d['SATURATED_VALUE']: sys.stderr.write("Saturated values do not match\n") else: saturated_value = d['SATURATED_VALUE'] if 'size' in locals(): if size != (d['SIZE1'], d['SIZE2']): sys.stderr.write("Image sizes do not match\n") return 1 else: size = (d['SIZE1'], d['SIZE2']) if size != d['DATA'].focus(): sys.stderr.write("Image size does not match pixel array\n") return 1 if 'pixel_size' in locals(): if pixel_size != d['PIXEL_SIZE']: sys.stderr.write("Pixel sizes do not match\n") return 1 else: if 'PIXEL_SIZE' in d: pixel_size = d['PIXEL_SIZE'] else: pixel_size = None except Exception: try: # Fall back on reading the image with dxtbx, and shoehorn the # extracted information into what would have been found in a # pickle file. XXX This code assumes a monolithic detector! from dxtbx.format.Registry import Registry format_class = Registry.find(path) i = format_class(path) beam = i.get_beam() assert len(i.get_detector()) == 1 detector = i.get_detector()[0] beam_center = detector.get_beam_centre(beam.get_s0()) detector_address = format_class.__name__ distance = detector.get_distance() img = i.get_raw_data().as_1d().as_double() pixel_size = 0.5 * sum(detector.get_pixel_size()) saturated_value = int(round(detector.get_trusted_range()[1])) size = detector.get_image_size() time_tuple = (i.get_scan().get_epochs()[0], 0) wavelength = beam.get_wavelength() active_areas = flex.int((0, 0, size[0], size[1])) except Exception: nfail += 1 continue # See also event() in xfel.cxi.cspad_ana.average_tbx. Record the # base time as the timestamp of the first image. # # The sum-of-squares image is accumulated using long integers, as # this delays the point where overflow occurs. But really, this # is just a band-aid... if nmemb == 0: max_img = img.deep_copy() sum_distance = distance sum_img = img.deep_copy() ssq_img = flex.pow2(img) sum_wavelength = wavelength sum_time = (0, 0) time_base = time_tuple else: sel = (img > max_img).as_1d() max_img.set_selected(sel, img.select(sel)) sum_distance += distance sum_img += img ssq_img += flex.pow2(img) sum_wavelength += wavelength sum_time = (sum_time[0] + (time_tuple[0] - time_base[0]), sum_time[1] + (time_tuple[1] - time_base[1])) nmemb += 1 # Early exit if no statistics were accumulated. if command_line.options.verbose: sys.stderr.write("Processed %d images (%d failed)\n" % (nmemb, nfail)) if nmemb == 0: return 0 # Calculate averages for measures where other statistics do not make # sense. Note that avg_img is required for stddev_img. avg_img = sum_img.as_double() / nmemb avg_distance = sum_distance / nmemb avg_timestamp = cspad_tbx.evt_timestamp( (time_base[0] + int(round(sum_time[0] / nmemb)), time_base[1] + int(round(sum_time[1] / nmemb)))) avg_wavelength = sum_wavelength / nmemb # Output the average image, maximum projection image, and standard # deviation image, if requested. if command_line.options.avg_path is not None: avg_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=avg_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.avg_path, d) if command_line.options.max_path is not None: max_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=max_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.max_path, d) if command_line.options.stddev_path is not None: stddev_img = ssq_img.as_double() - sum_img.as_double() * avg_img # Accumulating floating-point numbers introduces errors, which may # cause negative variances. Since a two-pass approach is # unacceptable, the standard deviation is clamped at zero. stddev_img.set_selected(stddev_img < 0, 0) if nmemb == 1: stddev_img = flex.sqrt(stddev_img) else: stddev_img = flex.sqrt(stddev_img / (nmemb - 1)) stddev_img.resize(flex.grid(size[0], size[1])) d = cspad_tbx.dpack( active_areas=active_areas, address=detector_address, beam_center_x=beam_center[0], beam_center_y=beam_center[1], data=stddev_img, distance=avg_distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=avg_timestamp, wavelength=avg_wavelength) easy_pickle.dump(command_line.options.stddev_path, d) return 0
def run(args, source_data=None): from xfel import radial_average from scitbx.array_family import flex from iotbx.detectors.cspad_detector_formats import reverse_timestamp from iotbx.detectors.cspad_detector_formats import detector_format_version as detector_format_function from spotfinder.applications.xfel import cxi_phil from iotbx.detectors.npy import NpyImage import os, sys from iotbx.detectors.npy import NpyImage user_phil = [] # TODO: replace this stuff with iotbx.phil.process_command_line_with_files # as soon as I can safely modify it for arg in args: if (not "=" in arg): try: user_phil.append(libtbx.phil.parse("""file_path=%s""" % arg)) except ValueError as e: raise Sorry("Unrecognized argument '%s'" % arg) else: try: user_phil.append(libtbx.phil.parse(arg)) except RuntimeError as e: raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e))) params = master_phil.fetch(sources=user_phil).extract() if params.file_path is None or not os.path.isfile( params.file_path) and source_data is None: master_phil.show() raise Usage( "file_path must be defined (either file_path=XXX, or the path alone)." ) assert params.handedness is not None assert params.n_bins is not None assert params.verbose is not None assert params.output_bins is not None if source_data is None: from libtbx import easy_pickle source_data = easy_pickle.load(params.file_path) if params.output_file is None: logger = sys.stdout else: logger = open(params.output_file, 'w') logger.write("%s " % params.output_file) if not "DETECTOR_ADDRESS" in source_data: # legacy format; try to guess the address LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in source_data and source_data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: LCLS_detector_address = source_data["DETECTOR_ADDRESS"] timesec = reverse_timestamp(source_data["TIMESTAMP"])[0] version_lookup = detector_format_function(LCLS_detector_address, timesec) args = [ "distl.detector_format_version=%s" % version_lookup, "viewer.powder_arcs.show=False", "viewer.powder_arcs.code=3n9c", ] horizons_phil = cxi_phil.cxi_versioned_extract(args).persist.commands img = NpyImage(params.file_path, source_data) img.readHeader(horizons_phil) img.translate_tiles(horizons_phil) if params.verbose: img.show_header() the_tiles = img.get_tile_manager( horizons_phil).effective_tiling_as_flex_int( reapply_peripheral_margin=False, encode_inactive_as_zeroes=True) if params.beam_x is None: params.beam_x = img.beamx / img.pixel_size if params.beam_y is None: params.beam_y = img.beamy / img.pixel_size if params.verbose: logger.write("I think the beam center is (%s,%s)\n" % (params.beam_x, params.beam_y)) bc = (int(params.beam_x), int(params.beam_y)) extent = int( math.ceil( max(distance((0, 0), bc), distance((img.size1, 0), bc), distance((0, img.size2), bc), distance((img.size1, img.size2), bc)))) if params.n_bins < extent: params.n_bins = extent extent_in_mm = extent * img.pixel_size extent_two_theta = math.atan(extent_in_mm / img.distance) * 180 / math.pi sums = flex.double(params.n_bins) * 0 sums_sq = flex.double(params.n_bins) * 0 counts = flex.int(params.n_bins) * 0 data = img.get_raw_data() if hasattr(data, "as_double"): data = data.as_double() logger.write("Average intensity: %9.3f\n" % flex.mean(data)) if params.verbose: logger.write("Generating average...tile:") logger.flush() for tile in xrange(len(the_tiles) // 4): if params.verbose: logger.write(" %d" % tile) logger.flush() x1, y1, x2, y2 = get_tile_coords(the_tiles, tile) radial_average(data, bc, sums, sums_sq, counts, img.pixel_size, img.distance, (x1, y1), (x2, y2)) if params.verbose: logger.write(" Finishing...\n") # average, avoiding division by zero results = sums.set_selected(counts <= 0, 0) results /= counts.set_selected(counts <= 0, 1).as_double() # calculte standard devations std_devs = [ math.sqrt((sums_sq[i] - sums[i] * results[i]) / counts[i]) if counts[i] > 0 else 0 for i in xrange(len(sums)) ] xvals = flex.double(len(results)) max_twotheta = float('-inf') max_result = float('-inf') for i in xrange(len(results)): twotheta = i * extent_two_theta / params.n_bins xvals[i] = twotheta if params.output_bins and "%.3f" % results[i] != "nan": #logger.write("%9.3f %9.3f\n"% (twotheta,results[i])) #.xy format for Rex.cell. logger.write( "%9.3f %9.3f %9.3f\n" % (twotheta, results[i], std_devs[i])) #.xye format for GSASII #logger.write("%.3f %.3f %.3f\n"%(twotheta,results[i],ds[i])) # include calculated d spacings if results[i] > max_result: max_twotheta = twotheta max_result = results[i] logger.write( "Maximum 2theta for %s, TS %s: %f, value: %f\n" % (params.file_path, source_data['TIMESTAMP'], max_twotheta, max_result)) if params.verbose: from pylab import scatter, show, xlabel, ylabel, ylim scatter(xvals, results) xlabel("2 theta") ylabel("Avg ADUs") if params.plot_y_max is not None: ylim(0, params.plot_y_max) show() return xvals, results
if params.output_file is None: logger = sys.stdout else: logger = open(params.output_file, 'w') logger.write("%s "%params.output_file) if not "DETECTOR_ADDRESS" in source_data: # legacy format; try to guess the address LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in source_data and source_data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: LCLS_detector_address = source_data["DETECTOR_ADDRESS"] timesec = reverse_timestamp( source_data["TIMESTAMP"] )[0] version_lookup = detector_format_function(LCLS_detector_address,timesec) args = [ "distl.detector_format_version=%s"%version_lookup, "viewer.powder_arcs.show=False", "viewer.powder_arcs.code=3n9c", ] horizons_phil = cxi_phil.cxi_versioned_extract(args).persist.commands img = NpyImage(params.file_path, source_data) img.readHeader(horizons_phil) img.translate_tiles(horizons_phil) if params.verbose: img.show_header()
def __call__(self): from iotbx.detectors.cspad_detector_formats import reverse_timestamp run_numbers = [r.run for r in self.trial.runs] assert self.run.run in run_numbers rungroup_ids = [rg.id for rg in self.trial.rungroups] assert self.rungroup.id in rungroup_ids isoforms = self.trial.isoforms assert len(isoforms) > 0 low_res_bin_ids = [] high_res_bin_ids = [] for isoform in isoforms: bins = isoform.cell.bins d_mins = [float(b.d_min) for b in bins] low_res_bin_ids.append(str(bins[d_mins.index(max(d_mins))].id)) if self.d_min is None: min_bin_index = d_mins.index(min(d_mins)) else: d_maxes = [float(b.d_max) for b in bins] qualified_bin_indices = [i for i in xrange(len(bins)) if d_maxes[i] >= self.d_min and d_mins[i] <= self.d_min] assert len(qualified_bin_indices) == 1 min_bin_index = qualified_bin_indices[0] high_res_bin_ids.append(str(bins[min_bin_index].id)) assert len(low_res_bin_ids) > 0 assert len(high_res_bin_ids) > 0 assert len(low_res_bin_ids) == len(high_res_bin_ids) tag = self.app.params.experiment_tag # Get the high and low res avg_i_sigi in one query. Means there will be 2x timestamps retrieved, where each is found twice query = """SELECT bin.id, event.timestamp, event.n_strong, cb.avg_i_sigi, event.two_theta_low, event.two_theta_high FROM `%s_event` event JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id JOIN `%s_imageset` imgset ON imgset.id = is_e.imageset_id JOIN `%s_experiment` exp ON exp.imageset_id = imgset.id JOIN `%s_crystal` crystal ON crystal.id = exp.crystal_id JOIN `%s_cell` cell ON cell.id = crystal.cell_id JOIN `%s_bin` bin ON bin.cell_id = cell.id JOIN `%s_cell_bin` cb ON cb.bin_id = bin.id AND cb.crystal_id = crystal.id WHERE event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d AND cb.bin_id IN (%s) """ % (tag, tag, tag, tag, tag, tag, tag, tag, self.trial.id, self.run.id, self.rungroup.id, ", ".join(low_res_bin_ids + high_res_bin_ids)) cursor = self.app.execute_query(query) timestamps = flex.double() n_strong = flex.int() average_i_sigi_low = flex.double() average_i_sigi_high = flex.double() two_theta_low = flex.double() two_theta_high = flex.double() for row in cursor.fetchall(): b_id, ts, n_s, avg_i_sigi, tt_low, tt_high = row rts = reverse_timestamp(ts) rts = rts[0] + (rts[1]/1000) if rts not in timestamps: # First time through, figure out which bin is reported (high or low), add avg_i_sigi to that set of results timestamps.append(rts) n_strong.append(n_s) two_theta_low.append(tt_low or -1) two_theta_high.append(tt_high or -1) if str(b_id) in low_res_bin_ids: average_i_sigi_low.append(avg_i_sigi or 1e-6) average_i_sigi_high.append(0) elif str(b_id) in high_res_bin_ids: average_i_sigi_low.append(0) average_i_sigi_high.append(avg_i_sigi or 0) else: assert False else: # Second time through, already have added to timestamps and n_strong, so fill in missing avg_i_sigi index = flex.first_index(timestamps, rts) if str(b_id) in low_res_bin_ids: average_i_sigi_low[index] = avg_i_sigi elif str(b_id) in high_res_bin_ids: average_i_sigi_high[index] = avg_i_sigi or 0 else: assert False # This left join query finds the events with no imageset, meaning they failed to index query = """SELECT event.timestamp, event.n_strong, event.two_theta_low, event.two_theta_high FROM `%s_event` event LEFT JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id WHERE is_e.event_id IS NULL AND event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d """ % (tag, tag, self.trial.id, self.run.id, self.rungroup.id) cursor = self.app.execute_query(query) for row in cursor.fetchall(): ts, n_s, tt_low, tt_high = row rts = reverse_timestamp(ts) timestamps.append(rts[0] + (rts[1]/1000)) n_strong.append(n_s) average_i_sigi_low.append(0) average_i_sigi_high.append(0) two_theta_low.append(tt_low or -1) two_theta_high.append(tt_high or -1) order = flex.sort_permutation(timestamps) timestamps = timestamps.select(order) n_strong = n_strong.select(order) average_i_sigi_low = average_i_sigi_low.select(order) average_i_sigi_high = average_i_sigi_high.select(order) two_theta_low = two_theta_low.select(order) two_theta_high = two_theta_high.select(order) return timestamps, two_theta_low, two_theta_high, n_strong, average_i_sigi_low, average_i_sigi_high
def average(argv=None): if argv == None: argv = sys.argv[1:] try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") command_line = (libtbx.option_parser.option_parser( usage=""" %s [-p] -c config -x experiment -a address -r run -d detz_offset [-o outputdir] [-A averagepath] [-S stddevpath] [-M maxpath] [-n numevents] [-s skipnevents] [-v] [-m] [-b bin_size] [-X override_beam_x] [-Y override_beam_y] [-D xtc_dir] [-f] [-g gain_mask_value] [--min] [--minpath minpath] To write image pickles use -p, otherwise the program writes CSPAD CBFs. Writing CBFs requires the geometry to be already deployed. Examples: cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 Use one process on the current node to process all the events from run 25 of experiment cxi49812, using a detz_offset of 571. mpirun -n 16 cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 As above, using 16 cores on the current node. bsub -a mympi -n 100 -o average.out -q psanaq cxi.mpi_average -c cxi49812/average.cfg -x cxi49812 -a CxiDs1.0:Cspad.0 -r 25 -d 571 -o cxi49812 As above, using the psanaq and 100 cores, putting the log in average.out and the output images in the folder cxi49812. """ % libtbx.env.dispatcher_name) .option(None, "--as_pickle", "-p", action="store_true", default=False, dest="as_pickle", help="Write results as image pickle files instead of cbf files") .option(None, "--raw_data", "-R", action="store_true", default=False, dest="raw_data", help="Disable psana corrections such as dark pedestal subtraction or common mode (cbf only)") .option(None, "--background_pickle", "-B", default=None, dest="background_pickle", help="") .option(None, "--config", "-c", type="string", default=None, dest="config", metavar="PATH", help="psana config file") .option(None, "--experiment", "-x", type="string", default=None, dest="experiment", help="experiment name (eg cxi84914)") .option(None, "--run", "-r", type="int", default=None, dest="run", help="run number") .option(None, "--address", "-a", type="string", default="CxiDs2.0:Cspad.0", dest="address", help="detector address name (eg CxiDs2.0:Cspad.0)") .option(None, "--detz_offset", "-d", type="float", default=None, dest="detz_offset", help="offset (in mm) from sample interaction region to back of CSPAD detector rail (CXI), or detector distance (XPP)") .option(None, "--outputdir", "-o", type="string", default=".", dest="outputdir", metavar="PATH", help="Optional path to output directory for output files") .option(None, "--averagebase", "-A", type="string", default="{experiment!l}_avg-r{run:04d}", dest="averagepath", metavar="PATH", help="Path to output average image without extension. String substitution allowed") .option(None, "--stddevbase", "-S", type="string", default="{experiment!l}_stddev-r{run:04d}", dest="stddevpath", metavar="PATH", help="Path to output standard deviation image without extension. String substitution allowed") .option(None, "--maxbase", "-M", type="string", default="{experiment!l}_max-r{run:04d}", dest="maxpath", metavar="PATH", help="Path to output maximum projection image without extension. String substitution allowed") .option(None, "--numevents", "-n", type="int", default=None, dest="numevents", help="Maximum number of events to process. Default: all") .option(None, "--skipevents", "-s", type="int", default=0, dest="skipevents", help="Number of events in the beginning of the run to skip. Default: 0") .option(None, "--verbose", "-v", action="store_true", default=False, dest="verbose", help="Print more information about progress") .option(None, "--pickle-optical-metrology", "-m", action="store_true", default=False, dest="pickle_optical_metrology", help="If writing pickle files, use the optical metrology in the experiment's calib directory") .option(None, "--bin_size", "-b", type="int", default=None, dest="bin_size", help="Rayonix detector bin size") .option(None, "--override_beam_x", "-X", type="float", default=None, dest="override_beam_x", help="Rayonix detector beam center x coordinate") .option(None, "--override_beam_y", "-Y", type="float", default=None, dest="override_beam_y", help="Rayonix detector beam center y coordinate") .option(None, "--calib_dir", "-C", type="string", default=None, dest="calib_dir", metavar="PATH", help="calibration directory") .option(None, "--xtc_dir", "-D", type="string", default=None, dest="xtc_dir", metavar="PATH", help="xtc stream directory") .option(None, "--use_ffb", "-f", action="store_true", default=False, dest="use_ffb", help="Use the fast feedback filesystem at LCLS. Only for the active experiment!") .option(None, "--gain_mask_value", "-g", type="float", default=None, dest="gain_mask_value", help="Ratio between low and high gain pixels, if CSPAD in mixed-gain mode. Only used in CBF averaging mode.") .option(None, "--min", None, action="store_true", default=False, dest="do_minimum_projection", help="Output a minimum projection") .option(None, "--minpath", None, type="string", default="{experiment!l}_min-r{run:04d}", dest="minpath", metavar="PATH", help="Path to output minimum image without extension. String substitution allowed") ).process(args=argv) if len(command_line.args) > 0 or \ command_line.options.as_pickle is None or \ command_line.options.experiment is None or \ command_line.options.run is None or \ command_line.options.address is None or \ command_line.options.detz_offset is None or \ command_line.options.averagepath is None or \ command_line.options.stddevpath is None or \ command_line.options.maxpath is None or \ command_line.options.pickle_optical_metrology is None: command_line.parser.show_help() return # set this to sys.maxint to analyze all events if command_line.options.numevents is None: maxevents = sys.maxint else: maxevents = command_line.options.numevents comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if command_line.options.config is not None: psana.setConfigFile(command_line.options.config) dataset_name = "exp=%s:run=%d:idx"%(command_line.options.experiment, command_line.options.run) if command_line.options.xtc_dir is not None: if command_line.options.use_ffb: raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system") dataset_name += ":dir=%s"%command_line.options.xtc_dir elif command_line.options.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(command_line.options.experiment[0:3],command_line.options.experiment) ds = psana.DataSource(dataset_name) address = command_line.options.address src = psana.Source('DetInfo(%s)'%address) nevent = np.array([0.]) if command_line.options.background_pickle is not None: background = easy_pickle.load(command_line.options.background_pickle)['DATA'].as_numpy_array() for run in ds.runs(): runnumber = run.run() if not command_line.options.as_pickle: psana_det = psana.Detector(address, ds.env()) # list of all events if command_line.options.skipevents > 0: print "Skipping first %d events"%command_line.options.skipevents elif "Rayonix" in command_line.options.address: print "Skipping first image in the Rayonix detector" # Shuttering issue command_line.options.skipevents = 1 times = run.times()[command_line.options.skipevents:] nevents = min(len(times),maxevents) # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0] for i in xrange(len(mytimes)): if i%10==0: print 'Rank',rank,'processing event',rank*len(mytimes)+i,', ',i,'of',len(mytimes) evt = run.event(mytimes[i]) #print "Event #",rank*mylength+i," has id:",evt.get(EventId) if 'Rayonix' in command_line.options.address or 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: data = evt.get(psana.Camera.FrameV1,src) if data is None: print "No data" continue data=data.data16().astype(np.float64) elif command_line.options.as_pickle: data = evt.get(psana.ndarray_float64_3, src, 'image0') else: # get numpy array, 32x185x388 from xfel.cftbx.detector.cspad_cbf_tbx import get_psana_corrected_data if command_line.options.raw_data: data = get_psana_corrected_data(psana_det, evt, use_default=False, dark=False, common_mode=None, apply_gain_mask=False, per_pixel_gain=False) else: if command_line.options.gain_mask_value is None: data = get_psana_corrected_data(psana_det, evt, use_default=True) else: data = get_psana_corrected_data(psana_det, evt, use_default=False, dark=True, common_mode=None, apply_gain_mask=True, gain_mask_value=command_line.options.gain_mask_value, per_pixel_gain=False) if data is None: print "No data" continue if command_line.options.background_pickle is not None: data -= background if 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: distance = np.array([0.0]) wavelength = np.array([1.0]) else: d = cspad_tbx.env_distance(address, run.env(), command_line.options.detz_offset) if d is None: print "No distance, using distance", command_line.options.detz_offset assert command_line.options.detz_offset is not None if 'distance' not in locals(): distance = np.array([command_line.options.detz_offset]) else: distance += command_line.options.detz_offset else: if 'distance' in locals(): distance += d else: distance = np.array([float(d)]) w = cspad_tbx.evt_wavelength(evt) if w is None: print "No wavelength" if 'wavelength' not in locals(): wavelength = np.array([1.0]) else: if 'wavelength' in locals(): wavelength += w else: wavelength = np.array([w]) t = cspad_tbx.evt_time(evt) if t is None: print "No timestamp, skipping shot" continue if 'timestamp' in locals(): timestamp += t[0] + (t[1]/1000) else: timestamp = np.array([t[0] + (t[1]/1000)]) if 'sum' in locals(): sum+=data else: sum=np.array(data, copy=True) if 'sumsq' in locals(): sumsq+=data*data else: sumsq=data*data if 'maximum' in locals(): maximum=np.maximum(maximum,data) else: maximum=np.array(data, copy=True) if command_line.options.do_minimum_projection: if 'minimum' in locals(): minimum=np.minimum(minimum,data) else: minimum=np.array(data, copy=True) nevent += 1 #sum the images across mpi cores if size > 1: print "Synchronizing rank", rank totevent = np.zeros(nevent.shape) comm.Reduce(nevent,totevent) if rank == 0 and totevent[0] == 0: raise Sorry("No events found in the run") sumall = np.zeros(sum.shape).astype(sum.dtype) comm.Reduce(sum,sumall) sumsqall = np.zeros(sumsq.shape).astype(sumsq.dtype) comm.Reduce(sumsq,sumsqall) maxall = np.zeros(maximum.shape).astype(maximum.dtype) comm.Reduce(maximum,maxall, op=MPI.MAX) if command_line.options.do_minimum_projection: minall = np.zeros(maximum.shape).astype(minimum.dtype) comm.Reduce(minimum,minall, op=MPI.MIN) waveall = np.zeros(wavelength.shape).astype(wavelength.dtype) comm.Reduce(wavelength,waveall) distall = np.zeros(distance.shape).astype(distance.dtype) comm.Reduce(distance,distall) timeall = np.zeros(timestamp.shape).astype(timestamp.dtype) comm.Reduce(timestamp,timeall) if rank==0: if size > 1: print "Synchronized" # Accumulating floating-point numbers introduces errors, # which may cause negative variances. Since a two-pass # approach is unacceptable, the standard deviation is # clamped at zero. mean = sumall / float(totevent[0]) variance = (sumsqall / float(totevent[0])) - (mean**2) variance[variance < 0] = 0 stddev = np.sqrt(variance) wavelength = waveall[0] / totevent[0] distance = distall[0] / totevent[0] pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value timestamp = timeall[0] / totevent[0] timestamp = (int(timestamp), timestamp % int(timestamp) * 1000) timestamp = cspad_tbx.evt_timestamp(timestamp) if command_line.options.as_pickle: extension = ".pickle" else: extension = ".cbf" dest_paths = [cspad_tbx.pathsubst(command_line.options.averagepath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.stddevpath + extension, evt, ds.env()), cspad_tbx.pathsubst(command_line.options.maxpath + extension, evt, ds.env())] if command_line.options.do_minimum_projection: dest_paths.append(cspad_tbx.pathsubst(command_line.options.minpath + extension, evt, ds.env())) dest_paths = [os.path.join(command_line.options.outputdir, path) for path in dest_paths] if 'Rayonix' in command_line.options.address: all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) from xfel.cxi.cspad_ana import rayonix_tbx pixel_size = rayonix_tbx.get_rayonix_pixel_size(command_line.options.bin_size) beam_center = [command_line.options.override_beam_x,command_line.options.override_beam_y] detector_dimensions = rayonix_tbx.get_rayonix_detector_dimensions(command_line.options.bin_size) active_areas = flex.int([0,0,detector_dimensions[0],detector_dimensions[1]]) split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3] for data, path in zip(all_data, dest_paths): print "Saving", path d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=flex.double(data), distance=distance, pixel_size=pixel_size, saturated_value=rayonix_tbx.rayonix_saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) elif 'FeeHxSpectrometer' in command_line.options.address or 'XrayTransportDiagnostic' in command_line.options.address: all_data = [mean, stddev, maxall] split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): d = cspad_tbx.dpack( address = old_style_address, data = flex.double(data), distance = distance, pixel_size = 0.1, timestamp=timestamp, wavelength=wavelength ) print "Saving", path easy_pickle.dump(path, d) elif command_line.options.as_pickle: split_address = cspad_tbx.address_split(address) old_style_address = split_address[0] + "-" + split_address[1] + "|" + split_address[2] + "-" + split_address[3] xpp = 'xpp' in address.lower() if xpp: evt_time = cspad_tbx.evt_time(evt) # tuple of seconds, milliseconds timestamp = cspad_tbx.evt_timestamp(evt_time) # human readable format from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version(old_style_address, reverse_timestamp(timestamp)[0]) assert version_lookup is not None active_areas = xpp_active_areas[version_lookup]['active_areas'] beam_center = [1765 // 2, 1765 // 2] else: if command_line.options.calib_dir is not None: metro_path = command_line.options.calib_dir elif command_line.options.pickle_optical_metrology: from xfel.cftbx.detector.cspad_cbf_tbx import get_calib_file_path metro_path = get_calib_file_path(run.env(), address, run) else: metro_path = libtbx.env.find_in_repositories("xfel/metrology/CSPad/run4/CxiDs1.0_Cspad.0") sections = parse_calib.calib2sections(metro_path) beam_center, active_areas = cspad_tbx.cbcaa( cspad_tbx.getConfig(address, ds.env()), sections) class fake_quad(object): def __init__(self, q, d): self.q = q self.d = d def quad(self): return self.q def data(self): return self.d if xpp: quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)] mean = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) mean = flex.double(mean.astype(np.float64)) quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)] stddev = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) stddev = flex.double(stddev.astype(np.float64)) quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)] maxall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [fake_quad(i, minall[i*8:(i+1)*8,:,:]) for i in xrange(4)] minall = cspad_tbx.image_xpp(old_style_address, None, ds.env(), active_areas, quads = quads) minall = flex.double(minall.astype(np.float64)) else: quads = [fake_quad(i, mean[i*8:(i+1)*8,:,:]) for i in xrange(4)] mean = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) mean = flex.double(mean.astype(np.float64)) quads = [fake_quad(i, stddev[i*8:(i+1)*8,:,:]) for i in xrange(4)] stddev = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) stddev = flex.double(stddev.astype(np.float64)) quads = [fake_quad(i, maxall[i*8:(i+1)*8,:,:]) for i in xrange(4)] maxall = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) maxall = flex.double(maxall.astype(np.float64)) if command_line.options.do_minimum_projection: quads = [fake_quad(i, minall[i*8:(i+1)*8,:,:]) for i in xrange(4)] minall = cspad_tbx.CsPadDetector( address, evt, ds.env(), sections, quads=quads) minall = flex.double(minall.astype(np.float64)) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print "Saving", path d = cspad_tbx.dpack( active_areas=active_areas, address=old_style_address, beam_center_x=pixel_size * beam_center[0], beam_center_y=pixel_size * beam_center[1], data=data, distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=timestamp, wavelength=wavelength) easy_pickle.dump(path, d) else: # load a header only cspad cbf from the slac metrology from xfel.cftbx.detector import cspad_cbf_tbx import pycbf base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, address) if base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d"%run.run()) all_data = [mean, stddev, maxall] if command_line.options.do_minimum_projection: all_data.append(minall) for data, path in zip(all_data, dest_paths): print "Saving", path cspad_img = cspad_cbf_tbx.format_object_from_data(base_dxtbx, data, distance, wavelength, timestamp, address, round_to_int=False) cspad_img._cbf_handle.write_widefile(path, pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K, 0)
for data, path in generate_data_from_streams(args, verbose=True): if 'TIMESTAMP' in data: # this is how FormatPYunspecified guesses the address if not "DETECTOR_ADDRESS" in data: # legacy format; try to guess the address LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in data and data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: LCLS_detector_address = data["DETECTOR_ADDRESS"] detector_format_version = detector_format_function( LCLS_detector_address, reverse_timestamp(data['TIMESTAMP'])[0]) print("Detector format version:", detector_format_version) image_pickle = True else: print("Not an image pickle") image_pickle = False keywise_printout(data) if image_pickle: import dxtbx image = dxtbx.load(path) tile_manager = image.detectorbase.get_tile_manager( image.detectorbase.horizons_phil_cache) tiling = tile_manager.effective_tiling_as_flex_int( reapply_peripheral_margin=True)
def __call__(self): from iotbx.detectors.cspad_detector_formats import reverse_timestamp run_numbers = [r.run for r in self.trial.runs] assert self.run.run in run_numbers rungroup_ids = [rg.id for rg in self.trial.rungroups] assert self.rungroup.id in rungroup_ids isoforms = self.trial.isoforms assert len(isoforms) > 0 low_res_bin_ids = [] high_res_bin_ids = [] for isoform in isoforms: bins = isoform.cell.bins d_mins = [float(b.d_min) for b in bins] low_res_bin_ids.append(str(bins[d_mins.index(max(d_mins))].id)) if self.d_min is None: min_bin_index = d_mins.index(min(d_mins)) else: d_maxes = [float(b.d_max) for b in bins] qualified_bin_indices = [ i for i in xrange(len(bins)) if d_maxes[i] >= self.d_min and d_mins[i] <= self.d_min ] assert len(qualified_bin_indices) == 1 min_bin_index = qualified_bin_indices[0] high_res_bin_ids.append(str(bins[min_bin_index].id)) assert len(low_res_bin_ids) > 0 assert len(high_res_bin_ids) > 0 assert len(low_res_bin_ids) == len(high_res_bin_ids) tag = self.app.params.experiment_tag # Get the high and low res avg_i_sigi in one query. Means there will be 2x timestamps retrieved, where each is found twice query = """SELECT bin.id, event.timestamp, event.n_strong, cb.avg_i_sigi, event.two_theta_low, event.two_theta_high FROM `%s_event` event JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id JOIN `%s_imageset` imgset ON imgset.id = is_e.imageset_id JOIN `%s_experiment` exp ON exp.imageset_id = imgset.id JOIN `%s_crystal` crystal ON crystal.id = exp.crystal_id JOIN `%s_cell` cell ON cell.id = crystal.cell_id JOIN `%s_bin` bin ON bin.cell_id = cell.id JOIN `%s_cell_bin` cb ON cb.bin_id = bin.id AND cb.crystal_id = crystal.id WHERE event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d AND cb.bin_id IN (%s) """ % (tag, tag, tag, tag, tag, tag, tag, tag, self.trial.id, self.run.id, self.rungroup.id, ", ".join(low_res_bin_ids + high_res_bin_ids)) cursor = self.app.execute_query(query) timestamps = flex.double() n_strong = flex.int() average_i_sigi_low = flex.double() average_i_sigi_high = flex.double() two_theta_low = flex.double() two_theta_high = flex.double() for row in cursor.fetchall(): b_id, ts, n_s, avg_i_sigi, tt_low, tt_high = row rts = reverse_timestamp(ts) rts = rts[0] + (rts[1] / 1000) if rts not in timestamps: # First time through, figure out which bin is reported (high or low), add avg_i_sigi to that set of results timestamps.append(rts) n_strong.append(n_s) two_theta_low.append(tt_low or -1) two_theta_high.append(tt_high or -1) if str(b_id) in low_res_bin_ids: average_i_sigi_low.append(avg_i_sigi or 1e-6) average_i_sigi_high.append(0) elif str(b_id) in high_res_bin_ids: average_i_sigi_low.append(0) average_i_sigi_high.append(avg_i_sigi or 0) else: assert False else: # Second time through, already have added to timestamps and n_strong, so fill in missing avg_i_sigi index = flex.first_index(timestamps, rts) if str(b_id) in low_res_bin_ids: average_i_sigi_low[index] = avg_i_sigi elif str(b_id) in high_res_bin_ids: average_i_sigi_high[index] = avg_i_sigi or 0 else: assert False # This left join query finds the events with no imageset, meaning they failed to index query = """SELECT event.timestamp, event.n_strong, event.two_theta_low, event.two_theta_high FROM `%s_event` event LEFT JOIN `%s_imageset_event` is_e ON is_e.event_id = event.id WHERE is_e.event_id IS NULL AND event.trial_id = %d AND event.run_id = %d AND event.rungroup_id = %d """ % (tag, tag, self.trial.id, self.run.id, self.rungroup.id) cursor = self.app.execute_query(query) for row in cursor.fetchall(): ts, n_s, tt_low, tt_high = row rts = reverse_timestamp(ts) timestamps.append(rts[0] + (rts[1] / 1000)) n_strong.append(n_s) average_i_sigi_low.append(0) average_i_sigi_high.append(0) two_theta_low.append(tt_low or -1) two_theta_high.append(tt_high or -1) order = flex.sort_permutation(timestamps) timestamps = timestamps.select(order) n_strong = n_strong.select(order) average_i_sigi_low = average_i_sigi_low.select(order) average_i_sigi_high = average_i_sigi_high.select(order) two_theta_low = two_theta_low.select(order) two_theta_high = two_theta_high.select(order) return timestamps, two_theta_low, two_theta_high, n_strong, average_i_sigi_low, average_i_sigi_high
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ super(common_mode_correction, self).event(evt, env) if (evt.get("skip_event")): return if not hasattr(self, 'active_areas') or self.active_areas is None or \ not hasattr(self, 'beam_center') or self.beam_center is None: if self.address == 'XppGon-0|marccd-0': # The mod_mar module needs to have been called before this one # to set this up. The MAR does not have a configure object. self.beam_center = evt.get("marccd_beam_center") self.active_areas = evt.get("marccd_active_areas") elif self.address == 'XppEndstation-0|Rayonix-0' or \ self.address == 'MfxEndstation-0|Rayonix-0': pass # bc and aa set in the beginjob function elif self.address == 'XppGon-0|Cspad-0': # Load the active areas as determined from the optical metrology from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version(self.address, reverse_timestamp(self.timestamp)[0]) assert version_lookup is not None self.active_areas = xpp_active_areas[version_lookup]['active_areas'] self.beam_center = [1765 // 2, 1765 // 2] else: (self.beam_center, self.active_areas) = \ cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections) if self.filter_laser_1_status is not None: if (self.laser_1_status.status != self.filter_laser_1_status or (self.laser_1_ms_since_change is not None and self.laser_1_ms_since_change < self.filter_laser_wait_time)): evt.put(skip_event_flag(), "skip_event") return if self.filter_laser_4_status is not None: if (self.laser_4_status.status != self.filter_laser_4_status or (self.laser_4_ms_since_change is not None and self.laser_4_ms_since_change < self.filter_laser_wait_time)): evt.put(skip_event_flag(), "skip_event") return # Early return if the full detector image is already stored in the # event. Otherwise, get it from the stream as a double-precision # floating-point flex array. XXX It is probably not safe to key # the image on self.address, so we should come up with our own # namespace. XXX Misnomer--could be CAMP, too self.cspad_img = evt.get(self.address) if self.cspad_img is not None: return if self.address == 'XppGon-0|Cspad-0': # Kludge until cspad_tbx.image() can be rewritten to handle the # XPP metrology. self.cspad_img = cspad_tbx.image_xpp( self.address, evt, env, self.active_areas) elif self.address == 'XppEndstation-0|Rayonix-0' or \ self.address == 'MfxEndstation-0|Rayonix-0': from psana import Source, Camera import numpy as np address = cspad_tbx.old_address_to_new_address(self.address) src=Source('DetInfo(%s)'%address) self.cspad_img = evt.get(Camera.FrameV1,src) if self.cspad_img is not None: self.cspad_img = self.cspad_img.data16().astype(np.float64) elif self.address=='CxiDg3-0|Opal1000-0': if evt.getFrameValue(self.address) is not None: self.cspad_img = evt.getFrameValue(self.address).data() elif self.address=='CxiEndstation-0|Opal1000-2': if evt.getFrameValue(self.address) is not None: self.cspad_img = evt.getFrameValue(self.address).data() elif self.address=='FeeHxSpectrometer-0|Opal1000-1': if evt.getFrameValue(self.address) is not None: self.cspad_img = evt.getFrameValue(self.address).data() elif self.address=='NoDetector-0|Cspad2x2-0': import numpy as np from pypdsdata import xtc test=[] self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element,self.address).data() self.cspad_img=np.reshape(self.cspad_img,(370, 388)) else: try: self.cspad_img = cspad_tbx.image( self.address, cspad_tbx.getConfig(self.address, env), evt, env, self.sections) except Exception, e: self.logger.error("Error reading image data: " + str(e)) evt.put(skip_event_flag(), "skip_event") return
def event(self, evt, env): """The event() function is called for every L1Accept transition. @param evt Event data object, a configure object @param env Environment object """ super(common_mode_correction, self).event(evt, env) if (evt.get("skip_event")): return if not hasattr(self, 'active_areas') or self.active_areas is None or \ not hasattr(self, 'beam_center') or self.beam_center is None: if self.address == 'XppGon-0|marccd-0': # The mod_mar module needs to have been called before this one # to set this up. The MAR does not have a configure object. self.beam_center = evt.get("marccd_beam_center") self.active_areas = evt.get("marccd_active_areas") elif self.address == 'XppEndstation-0|Rayonix-0' or \ self.address == 'MfxEndstation-0|Rayonix-0': pass # bc and aa set in the beginjob function elif self.address == 'XppGon-0|Cspad-0': # Load the active areas as determined from the optical metrology from iotbx.detectors.cspad_detector_formats import detector_format_version, reverse_timestamp from xfel.cxi.cspad_ana.cspad_tbx import xpp_active_areas version_lookup = detector_format_version( self.address, reverse_timestamp(self.timestamp)[0]) assert version_lookup is not None self.active_areas = xpp_active_areas[version_lookup][ 'active_areas'] self.beam_center = [1765 // 2, 1765 // 2] else: (self.beam_center, self.active_areas) = \ cspad_tbx.cbcaa(cspad_tbx.getConfig(self.address, env), self.sections) if self.filter_laser_1_status is not None: if (self.laser_1_status.status != self.filter_laser_1_status or (self.laser_1_ms_since_change is not None and self.laser_1_ms_since_change < self.filter_laser_wait_time)): evt.put(skip_event_flag(), "skip_event") return if self.filter_laser_4_status is not None: if (self.laser_4_status.status != self.filter_laser_4_status or (self.laser_4_ms_since_change is not None and self.laser_4_ms_since_change < self.filter_laser_wait_time)): evt.put(skip_event_flag(), "skip_event") return # Early return if the full detector image is already stored in the # event. Otherwise, get it from the stream as a double-precision # floating-point flex array. XXX It is probably not safe to key # the image on self.address, so we should come up with our own # namespace. XXX Misnomer--could be CAMP, too self.cspad_img = evt.get(self.address) if self.cspad_img is not None: return if self.address == 'XppGon-0|Cspad-0': # Kludge until cspad_tbx.image() can be rewritten to handle the # XPP metrology. self.cspad_img = cspad_tbx.image_xpp(self.address, evt, env, self.active_areas) elif self.address == 'XppEndstation-0|Rayonix-0' or \ self.address == 'MfxEndstation-0|Rayonix-0': from psana import Source, Camera import numpy as np address = cspad_tbx.old_address_to_new_address(self.address) src = Source('DetInfo(%s)' % address) self.cspad_img = evt.get(Camera.FrameV1, src) if self.cspad_img is not None: self.cspad_img = self.cspad_img.data16().astype(np.float64) elif self.address == 'CxiDg3-0|Opal1000-0': if evt.getFrameValue(self.address) is not None: self.cspad_img = evt.getFrameValue(self.address).data() elif self.address == 'CxiEndstation-0|Opal1000-2': if evt.getFrameValue(self.address) is not None: self.cspad_img = evt.getFrameValue(self.address).data() elif self.address == 'FeeHxSpectrometer-0|Opal1000-1': if evt.getFrameValue(self.address) is not None: self.cspad_img = evt.getFrameValue(self.address).data() elif self.address == 'NoDetector-0|Cspad2x2-0': import numpy as np from pypdsdata import xtc test = [] self.cspad_img = evt.get(xtc.TypeId.Type.Id_Cspad2x2Element, self.address).data() self.cspad_img = np.reshape(self.cspad_img, (370, 388)) else: try: self.cspad_img = cspad_tbx.image( self.address, cspad_tbx.getConfig(self.address, env), evt, env, self.sections) except Exception as e: self.logger.error("Error reading image data: " + str(e)) evt.put(skip_event_flag(), "skip_event") return if self.cspad_img is None: if cspad_tbx.address_split(self.address)[2] != 'Andor': self.nfail += 1 self.logger.warning("event(): no image, shot skipped") evt.put(skip_event_flag(), "skip_event") return self.cspad_img = flex.double(self.cspad_img.astype(numpy.float64)) # If a dark image was provided, subtract it from the image. There # is no point in doing common-mode correction unless the dark # image was subtracted. if (self.dark_img is not None): self.cspad_img -= self.dark_img if (self.common_mode_correction != "none"): # Mask out inactive pixels prior to common mode correction. # Pixels are marked as inactive either due to low ADU values # or non-positive standard deviations in dark image. XXX Make # the threshold tunable? cspad_mask = self.dark_mask.deep_copy() if self.roi is not None and self.common_mode_correction == "chebyshev": roi_mask = cspad_mask[self.roi[2]:self.roi[3], :] roi_mask = flex.bool(roi_mask.accessor(), False) cspad_mask.matrix_paste_block_in_place(block=roi_mask, i_row=self.roi[2], i_column=0) # Extract each active section from the assembled detector # image and apply the common mode correction. XXX Make up a # quadrant mask for the emission detector. Needs to be # checked! config = cspad_tbx.getConfig(self.address, env) if len(self.sections) == 1: q_mask = 1 else: q_mask = config.quadMask() for q in range(len(self.sections)): if (not ((1 << q) & q_mask)): continue # XXX Make up section mask for the emission detector. Needs # to be checked! import _pdsdata if len(self.sections) == 1 and type(config) in ( _pdsdata.cspad2x2.ConfigV1, _pdsdata.cspad2x2.ConfigV2): s_mask = config.roiMask() else: s_mask = config.roiMask(q) for s in range(len(self.sections[q])): # XXX DAQ misconfiguration? This mask appears not to work # reliably for the Sc1 detector. # if (not((1 << s) & s_mask)): # continue corners = self.sections[q][s].corners() i_row = int(round(min(c[0] for c in corners))) i_column = int(round(min(c[1] for c in corners))) n_rows = int(round(max(c[0] for c in corners))) - i_row n_columns = int(round(max( c[1] for c in corners))) - i_column section_img = self.cspad_img.matrix_copy_block( i_row=i_row, i_column=i_column, n_rows=n_rows, n_columns=n_columns) section_mask = cspad_mask.matrix_copy_block( i_row=i_row, i_column=i_column, n_rows=n_rows, n_columns=n_columns) section_stddev = self.dark_stddev.matrix_copy_block( i_row=i_row, i_column=i_column, n_rows=n_rows, n_columns=n_columns) if section_mask.count(True) == 0: continue if self.common_mode_correction == "chebyshev": assert len(self.sections[q]) == 2 if s == 0: section_imgs = [section_img] section_masks = [section_mask] i_rows = [i_row] i_columns = [i_column] continue else: section_imgs.append(section_img) section_masks.append(section_mask) i_rows.append(i_row) i_columns.append(i_column) chebyshev_corrected_imgs = self.chebyshev_common_mode( section_imgs, section_masks) for i in range(2): section_imgs[i].as_1d().copy_selected( section_masks[i].as_1d().iselection(), chebyshev_corrected_imgs[i].as_1d()) self.cspad_img.matrix_paste_block_in_place( block=section_imgs[i], i_row=i_rows[i], i_column=i_columns[i]) else: common_mode = self.common_mode( section_img, section_stddev, section_mask) self.sum_common_mode += common_mode self.sumsq_common_mode += common_mode**2 # Apply the common mode correction to the # section, and paste it back into the image. self.cspad_img.matrix_paste_block_in_place( block=section_img - common_mode, i_row=i_row, i_column=i_column) if self.gain_map is not None: self.cspad_img *= self.gain_map if (self.mask_img is not None): sel = (self.mask_img == -2) | (self.mask_img == cspad_tbx.cspad_mask_value) self.cspad_img.set_selected(sel, cspad_tbx.cspad_mask_value) if (self.address == 'XppEndstation-0|Rayonix-0' or \ self.address == 'MfxEndstation-0|Rayonix-0') and \ self.crop_rayonix: # Crop the masked data so that the beam center is in the center of the image self.cspad_img = self.cspad_img[self.rayonix_crop_slice[0], self.rayonix_crop_slice[1]] if self.cache_image: # Store the image in the event. evt.put(self.cspad_img, self.address)
doplots = False for data in generate_data_from_streams(args, verbose=True): if 'TIMESTAMP' in data: # this is how FormatPYunspecified guesses the address if not "DETECTOR_ADDRESS" in data: # legacy format; try to guess the address LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in data and data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: LCLS_detector_address = data["DETECTOR_ADDRESS"] detector_format_version = detector_format_function( LCLS_detector_address, reverse_timestamp(data['TIMESTAMP'])[0]) print "Detector format version:", detector_format_version image_pickle = True else: print "Not an image pickle" image_pickle = False keywise_printout(data) if image_pickle: import dxtbx image = dxtbx.load(path) tile_manager = image.detectorbase.get_tile_manager(image.detectorbase.horizons_phil_cache) tiling = tile_manager.effective_tiling_as_flex_int(reapply_peripheral_margin = True) print int(len(tiling)/4), "translated active areas, first one: ", list(tiling[0:4])
def run(params): counter = 0 root = params.input_path fig_object = plt.figure() good_total = fail_total = 0 all_psanats = [] all_deltas = [] fail_deltas = [] good_deltas = [] for filename in os.listdir(root): if os.path.splitext(filename)[1] != '.txt': continue if 'debug' not in filename: continue reference = None fail_timepoints = [] good_timepoints = [] rank = int(filename.split('_')[1].split('.')[0]) counter += 1 print(filename) run_timepoints = [] for line in open(os.path.join(root, filename)): try: hostname, psanats, ts, status, result = line.strip().split(',') except ValueError: continue if reference is None: sec, ms = reverse_timestamp(ts) reference = sec + ms * 1e-3 run_timepoints.append(0) assert status not in ['stop', 'done', 'fail'] if status in ['stop', 'done', 'fail']: sec, ms = reverse_timestamp(ts) run_timepoints.append((sec + ms * 1.e-3) - reference) if status == 'done': good_timepoints.append((sec + ms * 1.e-3) - reference) good_deltas.append(good_timepoints[-1] - run_timepoints[-2]) else: fail_timepoints.append((sec + ms * 1.e-3) - reference) fail_deltas.append(fail_timepoints[-1] - run_timepoints[-2]) all_psanats.append(psanats) all_deltas.append(run_timepoints[-1] - run_timepoints[-2]) ok = True else: ok = False plt.plot(fail_timepoints, [rank] * len(fail_timepoints), 'b.') plt.plot(good_timepoints, [rank] * len(good_timepoints), 'g.') fail_total += len(fail_timepoints) good_total += len(good_timepoints) if not ok: sec, ms = reverse_timestamp(ts) plt.plot([(sec + ms * 1e-3) - reference], [rank], 'rx') #if counter > 100: break if fail_deltas: print( "Five number summary of %d fail image processing times:" % fail_total, five_number_summary(flex.double(fail_deltas))) if good_deltas: print( "Five number summary of %d good image processing times:" % good_total, five_number_summary(flex.double(good_deltas))) if params.wall_time and params.num_nodes and params.num_cores_per_node - 0.5: for i in range(params.num_nodes): plt.plot([0, params.wall_time], [ i * params.num_cores_per_node - 0.5, i * params.num_cores_per_node - 0.5 ], 'r-') plt.xlabel('Wall time (sec)') plt.ylabel('MPI Rank Number') plt.title(params.plot_title) if params.pickle_plot: from libtbx.easy_pickle import dump dump('%s' % params.pickle_filename, fig_object) if params.show_plot: plt.show()