def _start(self): import os if isinstance(self._image_file, basestring) and os.path.isfile(self._image_file): stream = FormatPYunspecified.open_file(self._image_file, "rb") import cPickle as pickle data = pickle.load(stream) else: data = self._image_file if not "DETECTOR_ADDRESS" in data: # legacy format; try to guess the address self.LCLS_detector_address = "CxiDs1-0|Cspad-0" if "DISTANCE" in data and data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument self.LCLS_detector_address = "CxiDsd-0|Cspad-0" else: self.LCLS_detector_address = data["DETECTOR_ADDRESS"] from xfel.detector_formats import reverse_timestamp self._timesec = reverse_timestamp(data["TIMESTAMP"])[0] from xfel.detector_formats import detector_format_version as detector_format_function version_lookup = detector_format_function(self.LCLS_detector_address, self._timesec) self.start_helper(version_token="distl.detector_format_version=%s" % version_lookup)
def distl_filter(self, address, cspad_img, distance, timestamp, wavelength): self.hitfinder_d["DATA"] = cspad_img self.hitfinder_d["DISTANCE"] = distance self.hitfinder_d["TIMESTAMP"] = timestamp self.hitfinder_d["WAVELENGTH"] = wavelength self.hitfinder_d["DETECTOR_ADDRESS"] = address args = ["indexing.data=dummy", "distl.bins.verbose=False", self.asic_filter, ] detector_format_version = detector_format_function( address, reverse_timestamp(timestamp)[0]) args += ["distl.detector_format_version=%s" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = self.hitfinder_d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil from rstbx.new_horizons.index import pre_indexing_validation,pack_names pre_indexing_validation(horizons_phil) imagefile_arguments = pack_names(horizons_phil) from spotfinder.applications import signal_strength info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments) imgdata = info.Files.images[0].linearintdata active_data = self.get_active_data(info.Files.images[0],horizons_phil) peak_heights = flex.int( [ imgdata[ spot.max_pxl_x(), spot.max_pxl_y() ] for spot in info.S.images[info.frames[0]]["spots_total"] ]) outscale = 256 corrected = peak_heights.as_double() * self.correction outvalue = outscale *(1.0-corrected) outvalue.set_selected(outvalue<0.0,0.) outvalue.set_selected(outvalue>=outscale,int(outscale)-1) outvalue = flex.int(outvalue.as_numpy_array().astype(numpy.int32)) # essentially, select a peak if the peak's ADU value is > 2.5 * the 90-percentile pixel value #work = display_spots.wrapper_of_callback(info) #work.display_with_callback(horizons_phil.indexing.data) return peak_heights,outvalue
def integrate_one_image(data, **kwargs): from xfel.cxi.display_spots import run_one_index_core from labelit.dptbx.error import NoAutoIndex from libtbx.utils import Sorry from spotfinder.exception import SpotfinderError from labelit.exception import AutoIndexError from xfel.detector_formats import detector_format_version as detector_format_function from xfel.detector_formats import reverse_timestamp basename = kwargs.get("integration_basename") if (basename is None): basename = "" dirname = kwargs.get("integration_dirname") if (dirname is None): dirname = "integration" if (not os.path.isdir(dirname)): import errno try: os.makedirs(dirname) except OSError as exc: if exc.errno==errno.EEXIST: pass path = os.path.join(dirname, basename \ + data['TIMESTAMP'] \ + ("_%05d.pickle" % data['SEQUENCE_NUMBER'])) args = ["indexing.data=dummy", "beam_search_scope=0.5", "lepage_max_delta = 3.0", "spots_pickle = None", "subgroups_pickle = None", "refinements_pickle = None", "rmsd_tolerance = 5.0", "mosflm_rmsd_tolerance = 5.0", "indexing.completeness_pickle=%s"%path, "difflimit_sigma_cutoff=2.0", #"indexing.open_wx_viewer=True" ] detector_format_version = detector_format_function( data['DETECTOR_ADDRESS'], reverse_timestamp(data['TIMESTAMP'])[0]) args += ["distl.detector_format_version=%s" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(data["xtal_target"], args) horizons_phil.indexing.data = data print "XFEL processing: %s"%path try: return run_one_index_core(horizons_phil) except NoAutoIndex,e: print "NoAutoIndex", data['TIMESTAMP'], e
def event(self, evt, env): """The event() function is called for every L1Accept transition. XXX more? Previously, common-mode correction was applied only after initial threshold filtering. Since the common_mode class applies the (lengthy) common-mode correction immediately after reading the image from the stream, this optimisation is currently not (elegantly) doable. @param evt Event data object, a configure object @param env Environment object """ super(mod_hitfind, self).event(evt, env) if (evt.get("skip_event")): return # This module only applies to detectors for which a distance is # available. distance = cspad_tbx.env_distance(self.address, env, self._detz_offset) if distance is None: self.nfail += 1 self.logger.warning("event(): no distance, shot skipped") evt.put(skip_event_flag(), "skip_event") return device = cspad_tbx.address_split(self.address)[2] # ***** HITFINDING ***** XXX For hitfinding it may be interesting # to look at the fraction of subzero pixels in the dark-corrected # image. if (self.m_threshold is not None): # If a threshold value is given it can be applied in one of three ways: # 1. Apply it over the whole image if (self.m_roi is None and self.m_distl_min_peaks is None): vmax = flex.max(self.cspad_img) if (vmax < self.m_threshold): if not self.m_negate_hits: # Tell downstream modules to skip this event if the threshold was not met. evt.put(skip_event_flag(), "skip_event") return elif self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return # 2. Apply threshold over a rectangular region of interest. elif (self.m_roi is not None): vmax = flex.max(self.cspad_img[self.m_roi[2]:self.m_roi[3], self.m_roi[0]:self.m_roi[1]]) if (vmax < self.m_threshold): if not self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return elif self.m_negate_hits: evt.put(skip_event_flag(), "skip_event") return # 3. Determine the spotfinder spots within the central ASICS, and accept the # image as a hit if there are m_distl_min_peaks exceeding m_threshold. # As a further requirement, the peaks must exceed 2.5 * the 90-percentile # pixel value of the central ASICS. This filter was added to avoid high-background # false positives. elif (self.m_distl_min_peaks is not None): if device == 'marccd': self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0] self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1] elif device == 'Rayonix': self.hitfinder_d['BEAM_CENTER_X'] = self.beam_center[0] self.hitfinder_d['BEAM_CENTER_Y'] = self.beam_center[1] peak_heights,outvalue = self.distl_filter( self.address, self.cspad_img.iround(), # XXX correct? distance, self.timestamp, self.wavelength) if ('permissive' in self.m_distl_flags): number_of_accepted_peaks = (peak_heights > self.m_threshold).count(True) else: number_of_accepted_peaks = ((peak_heights > self.m_threshold).__and__(outvalue==0)).count(True) sec,ms = cspad_tbx.evt_time(evt) evt_time = sec + ms/1000 self.stats_logger.info("BRAGG %.3f %d" %(evt_time, number_of_accepted_peaks)) skip_event = False if number_of_accepted_peaks < self.m_distl_min_peaks: self.logger.info("Subprocess %02d: Spotfinder NO HIT image #%05d @ %s; %d spots > %d" %( env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold)) if not self.m_negate_hits: skip_event = True else: self.logger.info("Subprocess %02d: Spotfinder YES HIT image #%05d @ %s; %d spots > %d" %( env.subprocess(), self.nshots, self.timestamp, number_of_accepted_peaks, self.m_threshold)) if self.m_negate_hits: skip_event = True if skip_event: if self.m_db_logging: # log misses to the database self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, number_of_accepted_peaks, distance, self.sifoil, self.wavelength, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, self.m_db_tags)) evt.put(skip_event_flag(), "skip_event") return # the indexer will log this hit when it is ran. Bug: if the spotfinder is ran by itself, this # hit will not be logged in the db. evt.put(number_of_accepted_peaks, 'sfspots') self.logger.info("Subprocess %02d: process image #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) # See r17537 of mod_average.py. if device == 'Cspad': pixel_size = cspad_tbx.pixel_size saturated_value = cspad_tbx.cspad_saturated_value elif device == 'marccd': pixel_size = evt.get("marccd_pixel_size") saturated_value = evt.get("marccd_saturated_value") elif device == 'Rayonix': pixel_size = rayonix_tbx.get_rayonix_pixel_size(self.bin_size) saturated_value = rayonix_tbx.rayonix_saturated_value d = cspad_tbx.dpack( active_areas=self.active_areas, address=self.address, beam_center_x=pixel_size * self.beam_center[0], beam_center_y=pixel_size * self.beam_center[1], data=self.cspad_img.iround(), # XXX ouch! distance=distance, pixel_size=pixel_size, saturated_value=saturated_value, timestamp=self.timestamp, wavelength=self.wavelength, xtal_target=self.m_xtal_target) if (self.m_dispatch == "index"): import sys from xfel.cxi.integrate_image_api import integrate_one_image info = integrate_one_image(d, integration_dirname = self.m_integration_dirname, integration_basename = self.m_integration_basename) sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ indexed = info is not None if indexed and self.m_progress_logging: # integration pickle dictionary is available here as info.last_saved_best if info.last_saved_best["identified_isoform"] is not None: #print info.last_saved_best.keys() from cxi_xdr_xes.cftbx.cspad_ana import db dbobj = db.dbconnect(self.m_db_host, self.m_db_name, self.m_db_user, self.m_db_password) cursor = dbobj.cursor() if info.last_saved_best["identified_isoform"] in self.isoforms: PM, indices, miller_id = self.isoforms[info.last_saved_best["identified_isoform"]] else: from xfel.xpp.progress_support import progress_manager PM = progress_manager(info.last_saved_best,self.m_db_experiment_tag, self.m_trial_id, self.m_rungroup_id, evt.run()) indices, miller_id = PM.get_HKL(cursor) # cache these as they don't change for a given isoform self.isoforms[info.last_saved_best["identified_isoform"]] = PM, indices, miller_id if self.m_sql_buffer_size > 1: self.queue_progress_entry(PM.scale_frame_detail(self.timestamp,cursor,do_inserts=False)) else: PM.scale_frame_detail(self.timestamp,cursor,do_inserts=True) dbobj.commit() cursor.close() dbobj.close() if self.m_db_logging: sec,ms = cspad_tbx.evt_time(evt) evt_time = sec + ms/1000 sfspots = evt.get('sfspots') if sfspots is None: if indexed: n_spots = len(info.spotfinder_results.images[info.frames[0]]['spots_total']) else: n_spots = 0 else: n_spots = sfspots if indexed: mosaic_bloc_rotation = info.last_saved_best.get('ML_half_mosaicity_deg', [0])[0] mosaic_block_size = info.last_saved_best.get('ML_domain_size_ang', [0])[0] ewald_proximal_volume = info.last_saved_best.get('ewald_proximal_volume', [0])[0] obs = info.last_saved_best['observations'][0] cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma = obs.unit_cell().parameters() pointgroup = info.last_saved_best['pointgroup'] resolution = obs.d_min() else: mosaic_bloc_rotation = mosaic_block_size = ewald_proximal_volume = cell_a = cell_b = cell_c = \ cell_alpha = cell_beta = cell_gamma = spacegroup = resolution = 0 self.queue_entry((self.trial, evt.run(), "%.3f"%evt_time, n_spots, distance, self.sifoil, self.wavelength, indexed, mosaic_bloc_rotation, mosaic_block_size, ewald_proximal_volume, pointgroup, cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma, resolution, self.m_db_tags)) if (not indexed): evt.put(skip_event_flag(), "skip_event") return elif (self.m_dispatch == "nop"): pass elif (self.m_dispatch == "view"): #interactive image viewer args = ["indexing.data=dummy"] detector_format_version = detector_format_function( self.address, evt.GetTime()) if detector_format_version is not None: args += ["distl.detector_format_version=%" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil display_spots.wrapper_of_callback().display(horizons_phil.indexing.data) elif (self.m_dispatch == "spots"): #interactive spotfinder viewer args = ["indexing.data=dummy"] detector_format_version = detector_format_function( self.address, evt.GetTime()) if detector_format_version is not None: args += ["distl.detector_format_version=%s" % detector_format_version] from xfel.phil_preferences import load_cxi_phil horizons_phil = load_cxi_phil(self.m_xtal_target, args) horizons_phil.indexing.data = d from xfel.cxi import display_spots display_spots.parameters.horizons_phil = horizons_phil from rstbx.new_horizons.index import pre_indexing_validation,pack_names pre_indexing_validation(horizons_phil) imagefile_arguments = pack_names(horizons_phil) horizons_phil.persist.show() from spotfinder.applications import signal_strength info = signal_strength.run_signal_strength_core(horizons_phil,imagefile_arguments) work = display_spots.wrapper_of_callback(info) work.display_with_callback(horizons_phil.indexing.data) elif (self.m_dispatch == "write_dict"): self.logger.warning( "event(): deprecated dispatch 'write_dict', use mod_dump instead") if (self.m_out_dirname is not None or self.m_out_basename is not None): cspad_tbx.dwritef(d, self.m_out_dirname, self.m_out_basename) # Diagnostic message emitted only when all the processing is done. if (env.subprocess() >= 0): self.logger.info("Subprocess %02d: accepted #%05d @ %s" % (env.subprocess(), self.nshots, self.timestamp)) else: self.logger.info("Accepted #%05d @ %s" % (self.nshots, self.timestamp))
continue print "Printing contents of", path if data.has_key('TIMESTAMP'): # this is how FormatPYunspecified guesses the address if not "DETECTOR_ADDRESS" in data: # legacy format; try to guess the address LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in data and data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: LCLS_detector_address = data["DETECTOR_ADDRESS"] detector_format_version = detector_format_function( LCLS_detector_address, reverse_timestamp(data['TIMESTAMP'])[0]) print "Detector format version:", detector_format_version image_pickle = True else: print "Not an image pickle" image_pickle = False for key in data: if key == 'ACTIVE_AREAS': print int(len(data[key])/4), "active areas, first one: ", list(data[key][0:4]) elif key == 'observations': print key, data[key], "Showing unit cell/spacegroup:" obs = data[key][0] uc = obs.unit_cell() uc.show_parameters() obs.space_group().info().show_summary()
if params.output_file is None: logger = sys.stdout else: logger = open(params.output_file, 'w') logger.write("%s "%params.output_file) if not "DETECTOR_ADDRESS" in source_data: # legacy format; try to guess the address LCLS_detector_address = 'CxiDs1-0|Cspad-0' if "DISTANCE" in source_data and source_data["DISTANCE"] > 1000: # downstream CS-PAD detector station of CXI instrument LCLS_detector_address = 'CxiDsd-0|Cspad-0' else: LCLS_detector_address = source_data["DETECTOR_ADDRESS"] timesec = reverse_timestamp( source_data["TIMESTAMP"] )[0] version_lookup = detector_format_function(LCLS_detector_address,timesec) args = [ "distl.detector_format_version=%s"%version_lookup, "viewer.powder_arcs.show=False", "viewer.powder_arcs.code=3n9c", ] horizons_phil = cxi_phil.cxi_versioned_extract(args).persist.commands img = NpyImage(params.file_path, source_data) img.readHeader(horizons_phil) img.translate_tiles(horizons_phil) if params.verbose: img.show_header() the_tiles = img.get_tile_manager(horizons_phil).effective_tiling_as_flex_int(