def calculate_parameters(self, experiments=None):
"""Image modification for current cctbx.xfel."""
if not experiments:
# If data are given, apply modifications as specified below
error = "IOTA IMPORT ERROR: Experiment list not found!"
return None, error
else:
error = []
# Calculate auto-threshold
# TODO: Revisit this; I REALLY don't like it.
if self.auto_threshold:
beamX = self.img_object.final["beamX"]
beamY = self.img_object.final["beamY"]
px_size = self.img_object.final["pixel"]
try:
img = dxtbx.load(self.img_object.img_path)
raw_data = img.get_raw_data()
beam_x_px = int(beamX / px_size)
beam_y_px = int(beamY / px_size)
data_array = raw_data.as_numpy_array().astype(float)
self.center_int = np.nanmax(
data_array[
beam_y_px - 20 : beam_y_px + 20,
beam_x_px - 20 : beam_x_px + 20,
]
)
except Exception as e:
error.append(
"IOTA IMPORT ERROR: Auto-threshold failed! {}".format(e)
)
# Estimate gain (or set gain to 1.00 if cannot calculate)
if self.estimate_gain:
with util.Capturing() as junk_output:
try:
assert self.img_object.experiments # Must have experiments here
imageset = self.img_object.experiments.extract_imagesets()[0]
self.img_object.gain = estimate_gain(imageset)
except Exception as e:
error.append(
"IOTA IMPORT ERROR: Estimate gain failed! ".format(e)
)
# Collect error messages for logging
if error:
error_message = "\n".join(error)
else:
error_message = None
return experiments, error_message
def calculate_parameters(self, datablock=None):
''' Image modification for current cctbx.xfel '''
if not datablock:
# If data are given, apply modifications as specified below
error = 'IOTA IMPORT ERROR: Datablock not found!'
return None, error
else:
error = []
# Calculate auto-threshold
# TODO: Revisit this; I REALLY don't like it.
if self.iparams.cctbx_xfel.auto_threshold:
beamX = self.img_object.final['beamX']
beamY = self.img_object.final['beamY']
px_size = self.img_object.final['pixel']
try:
img = dxtbx.load(self.img_object.img_path)
raw_data = img.get_raw_data()
beam_x_px = int(beamX / px_size)
beam_y_px = int(beamY / px_size)
data_array = raw_data.as_numpy_array().astype(float)
self.center_int = np.nanmax(
data_array[beam_y_px - 20:beam_y_px + 20,
beam_x_px - 20:beam_x_px + 20])
except Exception as e:
error.append(
'IOTA IMPORT ERROR: Auto-threshold failed! {}'.format(
e))
# Estimate gain (or set gain to 1.00 if cannot calculate)
if self.iparams.advanced.estimate_gain:
with util.Capturing() as junk_output:
try:
assert self.img_object.datablock # Must have datablock here
imageset = self.img_object.datablock.extract_imagesets(
)[0]
self.img_object.gain = estimate_gain(imageset)
except Exception as e:
error.append(
'IOTA IMPORT ERROR: Estimate gain failed! '.format(
e))
# Collect error messages for logging
if error:
error_message = '\n'.join(error)
else:
error_message = None
return datablock, error_message
class SingleImage(object):
def __init__(self, img, init, verbose=True, imported_grid=None):
""" Constructor for the SingleImage object using a raw image file or pickle
"""
# Initialize parameters
self.params = init.params
self.args = init.args
self.user_id = init.user_id
self.raw_img = img[2]
self.conv_img = img[2]
self.img_index = img[0]
self.center_int = 0
self.status = None
self.fail = None
self.final = None
self.log_info = []
self.gs_results = []
self.main_log = init.logfile
self.verbose = verbose
self.hmed = self.params.cctbx.grid_search.height_median
self.amed = self.params.cctbx.grid_search.area_median
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.viz_base = init.viz_base
self.log_base = init.log_base
self.tmp_base = init.tmp_base
self.abort_file = os.path.join(self.int_base, '.abort.tmp')
self.obj_path = None
self.obj_file = None
self.fin_path = None
self.fin_file = None
self.viz_path = None
# ============================== SELECTION-ONLY FUNCTIONS ============================== #
def import_int_file(self, init):
""" Replaces path settings in imported image object with new settings
NEED TO RE-DO LATER """
if os.path.isfile(self.abort_file):
self.fail = 'aborted'
return self
# Generate paths to output files
self.params = init.params
self.main_log = init.logfile
self.input_base = init.input_base
self.conv_base = init.conv_base
self.int_base = init.int_base
self.obj_base = init.obj_base
self.fin_base = init.fin_base
self.log_base = init.log_base
self.viz_base = init.viz_base
self.obj_path = misc.make_image_path(self.conv_img, self.input_base,
self.obj_base)
self.obj_file = os.path.abspath(
os.path.join(
self.obj_path,
os.path.basename(self.conv_img).split('.')[0] + ".int"))
self.fin_path = misc.make_image_path(self.conv_img, self.input_base,
self.fin_base)
self.log_path = misc.make_image_path(self.conv_img, self.input_base,
self.log_base)
self.fin_file = os.path.abspath(
os.path.join(
self.fin_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.pickle"))
self.final['final'] = self.fin_file
self.final['img'] = self.conv_img
self.viz_path = misc.make_image_path(self.conv_img, self.input_base,
self.viz_base)
self.viz_file = os.path.join(
self.viz_path,
os.path.basename(self.conv_img).split('.')[0] + "_int.png")
# Create actual folders (if necessary)
try:
if not os.path.isdir(self.obj_path):
os.makedirs(self.obj_path)
if not os.path.isdir(self.fin_path):
os.makedirs(self.fin_path)
if not os.path.isdir(self.viz_path):
os.makedirs(self.viz_path)
except OSError:
pass
# Grid search / integration log file
self.int_log = os.path.join(
self.log_path,
os.path.basename(self.conv_img).split('.')[0] + '.tmp')
# Reset status to 'grid search' to pick up at selection (if no fail)
if self.fail == None:
self.status = 'bypass grid search'
return self
def determine_gs_result_file(self):
""" For 'selection-only' cctbx.xfel runs, determine where the image objects are """
if self.params.cctbx.selection.select_only.grid_search_path != None:
obj_path = os.path.abspath(
self.params.cctbx.selection.select_only.grid_search_path)
else:
run_number = int(os.path.basename(self.int_base)) - 1
obj_path = "{}/integration/{:03d}/image_objects"\
"".format(os.path.abspath(os.curdir), run_number)
gs_result_file = os.path.join(obj_path,
os.path.basename(self.obj_file))
return gs_result_file
# =============================== IMAGE IMPORT FUNCTIONS =============================== #
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except Exception, e:
print 'IOTA IMPORT ERROR:', e
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
img_type = 'pickle'
else:
img_type = 'raw'
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
else:
data = None
img_type = 'not imported'
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
from dxtbx.datablock import DataBlockFactory
from dials.command_line.estimate_gain import estimate_gain
with misc.Capturing() as junk_output:
try:
datablock = DataBlockFactory.from_filenames([self.raw_img
])[0]
imageset = datablock.extract_imagesets()[0]
self.gain = estimate_gain(imageset)
except Exception, e:
self.gain = 1.0
def process_event(self, run, timestamp):
"""
Process a single event from a run
@param run psana run object
@param timestamp psana timestamp object
"""
ts = cspad_tbx.evt_timestamp(
(timestamp.seconds(), timestamp.nanoseconds() / 1e6))
if ts is None:
print "No timestamp, skipping shot"
return
if len(self.params_cache.debug.event_timestamp
) > 0 and ts not in self.params_cache.debug.event_timestamp:
return
if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events:
if ts in self.known_events:
if self.known_events[ts] not in ["stop", "done", "fail"]:
if self.params_cache.debug.skip_bad_events:
print "Skipping event %s: possibly caused an unknown exception previously" % ts
return
elif self.params_cache.debug.skip_processed_events:
print "Skipping event %s: processed successfully previously" % ts
return
else:
if self.params_cache.debug.skip_unprocessed_events:
print "Skipping event %s: not processed previously" % ts
return
self.debug_start(ts)
evt = run.event(timestamp)
if evt.get("skip_event") or "skip_event" in [
key.key() for key in evt.keys()
]:
print "Skipping event", ts
self.debug_write("psana_skip", "skip")
return
print "Accepted", ts
self.params = copy.deepcopy(self.params_cache)
# the data needs to have already been processed and put into the event by psana
if self.params.format.file_format == 'cbf':
# get numpy array, 32x185x388
data = cspad_cbf_tbx.get_psana_corrected_data(
self.psana_det,
evt,
use_default=False,
dark=True,
common_mode=self.common_mode,
apply_gain_mask=self.params.format.cbf.gain_mask_value
is not None,
gain_mask_value=self.params.format.cbf.gain_mask_value,
per_pixel_gain=False)
if data is None:
print "No data"
self.debug_write("no_data", "skip")
return
if self.params.format.cbf.override_distance is None:
distance = cspad_tbx.env_distance(
self.params.input.address, run.env(),
self.params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
self.debug_write("no_distance", "skip")
return
else:
distance = self.params.format.cbf.override_distance
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
self.debug_write("no_wavelength", "skip")
return
else:
wavelength = 12398.4187 / self.params.format.cbf.override_energy
if self.params.format.file_format == 'pickle':
image_dict = evt.get(self.params.format.pickle.out_key)
data = image_dict['DATA']
timestamp = t = ts
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
20:23]
print "Processing shot", s
if self.params.format.file_format == 'cbf':
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(
self.base_dxtbx, data, distance, wavelength, timestamp,
self.params.input.address)
if self.params.input.reference_geometry is not None:
from dxtbx.model import Detector
# copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead.
cspad_img._detector_instance = Detector.from_dict(
self.reference_detector.to_dict())
cspad_img.sync_detector_to_cbf()
elif self.params.format.file_format == 'pickle':
from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory
cspad_img = FormatPYunspecifiedStillInMemory(image_dict)
cspad_img.timestamp = s
if self.params.dispatch.dump_all:
self.save_image(
cspad_img, self.params,
os.path.join(self.params.output.output_dir, "shot-" + s))
self.cache_ranges(cspad_img, self.params)
imgset = MemImageSet([cspad_img])
if self.params.dispatch.estimate_gain_only:
from dials.command_line.estimate_gain import estimate_gain
estimate_gain(imgset)
return
if not self.params.dispatch.find_spots:
self.debug_write("data_loaded", "done")
return
datablock = DataBlockFactory.from_imageset(imgset)[0]
# before calling DIALS for processing, set output paths according to the templates
if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template:
self.params.output.indexed_filename = os.path.join(
self.params.output.output_dir,
self.indexed_filename_template % ("idx-" + s))
if "%s" in self.refined_experiments_filename_template:
self.params.output.refined_experiments_filename = os.path.join(
self.params.output.output_dir,
self.refined_experiments_filename_template % ("idx-" + s))
if "%s" in self.integrated_filename_template:
self.params.output.integrated_filename = os.path.join(
self.params.output.output_dir,
self.integrated_filename_template % ("idx-" + s))
if "%s" in self.reindexedstrong_filename_template:
self.params.output.reindexedstrong_filename = os.path.join(
self.params.output.output_dir,
self.reindexedstrong_filename_template % ("idx-" + s))
# Load a dials mask from the trusted range and psana mask
from dials.util.masking import MaskGenerator
generator = MaskGenerator(self.params.border_mask)
mask = generator.generate(imgset)
if self.params.format.file_format == "cbf":
mask = tuple([a & b for a, b in zip(mask, self.dials_mask)])
if self.spotfinder_mask is None:
self.params.spotfinder.lookup.mask = mask
else:
self.params.spotfinder.lookup.mask = tuple(
[a & b for a, b in zip(mask, self.spotfinder_mask)])
if self.integration_mask is None:
self.params.integration.lookup.mask = mask
else:
self.params.integration.lookup.mask = tuple(
[a & b for a, b in zip(mask, self.integration_mask)])
self.debug_write("spotfind_start")
try:
observed = self.find_spots(datablock)
except Exception, e:
import traceback
traceback.print_exc()
print str(e), "event", timestamp
self.debug_write("spotfinding_exception", "fail")
return
def load_image(self):
""" Reads raw image file and extracts data for conversion into pickle
format. Also estimates gain if turned on."""
# Load raw image or image pickle
try:
with misc.Capturing() as junk_output:
loaded_img = dxtbx.load(self.raw_img)
except Exception as e:
print 'IOTA IMPORT ERROR:', e
loaded_img = None
pass
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
img_type = 'pickle'
else:
img_type = 'raw'
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec,msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp
)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
else:
data = None
img_type = 'not imported'
# Estimate gain (or set gain to 1.00 if cannot calculate)
# Cribbed from estimate_gain.py by Richard Gildea
if self.params.advanced.estimate_gain:
from dxtbx.datablock import DataBlockFactory
from dials.command_line.estimate_gain import estimate_gain
with misc.Capturing() as junk_output:
try:
datablock = DataBlockFactory.from_filenames([self.raw_img])[0]
imageset = datablock.extract_imagesets()[0]
self.gain = estimate_gain(imageset)
except Exception as e:
self.gain = 1.0
else:
self.gain = 1.0
return data, img_type