def run(self):
''' Run the script. '''
from dials.util.masking import MaskGenerator
from dials.util.options import flatten_datablocks
from libtbx.utils import Sorry
import cPickle as pickle
from dials.util import log
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# COnfigu logging
log.config()
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Generate the mask
generator = MaskGenerator(params)
mask = generator.generate(imageset)
# Save the mask to file
print "Writing mask to %s" % params.output.mask
pickle.dump(mask, open(params.output.mask, "w"))
def UpdateMask(self):
image_viewer_frame = self.GetParent().GetParent()
# Generate the mask
from dials.util.masking import MaskGenerator
generator = MaskGenerator(self.params.masking)
imageset = image_viewer_frame.imagesets[0] # XXX
mask = generator.generate(imageset)
image_viewer_frame.mask_image_viewer = mask
image_viewer_frame.update_settings(layout=False)
def run(self):
''' Run the script. '''
from dials.util.masking import MaskGenerator
from dials.util.options import flatten_datablocks
from libtbx.utils import Sorry
import six.moves.cPickle as pickle
from dials.util import log
from dxtbx.format.image import ImageBool
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
datablocks = flatten_datablocks(params.input.datablock)
# Configure logging
log.config()
# Check number of args
if len(datablocks) == 0:
self.parser.print_help()
return
if len(datablocks) != 1:
raise Sorry('exactly 1 datablock must be specified')
datablock = datablocks[0]
imagesets = datablock.extract_imagesets()
if len(imagesets) != 1:
raise Sorry('datablock must contain exactly 1 imageset')
imageset = imagesets[0]
# Generate the mask
generator = MaskGenerator(params)
mask = generator.generate(imageset)
# Save the mask to file
print("Writing mask to %s" % params.output.mask)
with open(params.output.mask, "wb") as fh:
pickle.dump(mask, fh)
# Save the datablock
if params.output.datablock is not None:
imageset.external_lookup.mask.data = ImageBool(mask)
imageset.external_lookup.mask.filename = params.output.mask
from dxtbx.datablock import DataBlockDumper
print('Saving datablocks to {0}'.format(params.output.datablock))
dump = DataBlockDumper(datablocks)
dump.as_file(params.output.datablock)
def background(imageset, indx, n_bins, mask_params=None):
from dials.array_family import flex
from libtbx.phil import parse
from scitbx import matrix
if mask_params is None:
# Default mask params for trusted range
mask_params = phil_scope.fetch(parse("")).extract().masking
from dials.util.masking import MaskGenerator
mask_generator = MaskGenerator(mask_params)
mask = mask_generator.generate(imageset)
detector = imageset.get_detector()
beam = imageset.get_beam()
# Only working with single panel detector for now
assert len(detector) == 1
panel = detector[0]
mask = mask[0]
n = matrix.col(panel.get_normal()).normalize()
b = matrix.col(beam.get_s0()).normalize()
wavelength = beam.get_wavelength()
if math.fabs(b.dot(n)) < 0.95:
raise Sorry("Detector not perpendicular to beam")
data = imageset.get_raw_data(indx)
assert len(data) == 1
data = data[0]
data = data.as_double()
spot_params = spot_phil.fetch(source=parse("")).extract()
threshold_function = SpotFinderFactory.configure_threshold(spot_params)
peak_pixels = threshold_function.compute_threshold(data, mask)
signal = data.select(peak_pixels.iselection())
background_pixels = mask & ~peak_pixels
background = data.select(background_pixels.iselection())
# print some summary information
print("Mean background: %.3f" % (flex.sum(background) / background.size()))
if len(signal) > 0:
print("Max/total signal pixels: %.0f / %.0f" %
(flex.max(signal), flex.sum(signal)))
else:
print("No signal pixels on this image")
print("Peak/background/masked pixels: %d / %d / %d" %
(peak_pixels.count(True), background.size(), mask.count(False)))
# compute histogram of two-theta values, then same weighted
# by pixel values, finally divide latter by former to get
# the radial profile out, need to set the number of bins
# sensibly; inspired by method in PyFAI
two_theta_array = panel.get_two_theta_array(beam.get_s0())
two_theta_array = two_theta_array.as_1d().select(
background_pixels.iselection())
# Use flex.weighted_histogram
h0 = flex.weighted_histogram(two_theta_array, n_slots=n_bins)
h1 = flex.weighted_histogram(two_theta_array, background, n_slots=n_bins)
h2 = flex.weighted_histogram(two_theta_array,
background * background,
n_slots=n_bins)
d0 = h0.slots()
d1 = h1.slots()
d2 = h2.slots()
I = d1 / d0
I2 = d2 / d0
sig = flex.sqrt(I2 - flex.pow2(I))
tt = h0.slot_centers()
d_spacings = wavelength / (2.0 * flex.sin(0.5 * tt))
return d_spacings, I, sig
def from_parameters(params=None, experiments=None):
"""
Given a set of parameters, construct the spot finder
:param params: The input parameters
:returns: The spot finder instance
"""
from dials.util.masking import MaskGenerator
from dials.algorithms.spot_finding.finder import SpotFinder
from libtbx.phil import parse
from dxtbx.imageset import ImageSequence
if params is None:
params = phil_scope.fetch(source=parse("")).extract()
if params.spotfinder.force_2d and params.output.shoeboxes is False:
no_shoeboxes_2d = True
elif experiments is not None and params.output.shoeboxes is False:
no_shoeboxes_2d = False
all_stills = True
for experiment in experiments:
if isinstance(experiment.imageset, ImageSequence):
all_stills = False
break
if all_stills:
no_shoeboxes_2d = True
else:
no_shoeboxes_2d = False
# Read in the lookup files
mask = SpotFinderFactory.load_image(params.spotfinder.lookup.mask)
params.spotfinder.lookup.mask = mask
# Configure the filter options
filter_spots = SpotFinderFactory.configure_filter(params)
# Create the threshold strategy
threshold_function = SpotFinderFactory.configure_threshold(
params, experiments)
# Configure the mask generator
mask_generator = MaskGenerator(params.spotfinder.filter)
# Make sure 'none' is interpreted as None
if params.spotfinder.mp.method == "none":
params.spotfinder.mp.method = None
# Setup the spot finder
return SpotFinder(
threshold_function=threshold_function,
mask=params.spotfinder.lookup.mask,
filter_spots=filter_spots,
scan_range=params.spotfinder.scan_range,
write_hot_mask=params.spotfinder.write_hot_mask,
hot_mask_prefix=params.spotfinder.hot_mask_prefix,
mp_method=params.spotfinder.mp.method,
mp_nproc=params.spotfinder.mp.nproc,
mp_njobs=params.spotfinder.mp.njobs,
mp_chunksize=params.spotfinder.mp.chunksize,
max_strong_pixel_fraction=params.spotfinder.filter.
max_strong_pixel_fraction,
compute_mean_background=params.spotfinder.compute_mean_background,
region_of_interest=params.spotfinder.region_of_interest,
mask_generator=mask_generator,
min_spot_size=params.spotfinder.filter.min_spot_size,
max_spot_size=params.spotfinder.filter.max_spot_size,
no_shoeboxes_2d=no_shoeboxes_2d,
min_chunksize=params.spotfinder.mp.min_chunksize,
)
def process_event(self, run, timestamp):
"""
Process a single event from a run
@param run psana run object
@param timestamp psana timestamp object
"""
ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6))
if ts is None:
print "No timestamp, skipping shot"
return
if len(self.params_cache.debug.event_timestamp) > 0 and ts not in self.params_cache.debug.event_timestamp:
return
if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events:
if ts in self.known_events:
if self.known_events[ts] not in ["stop", "done", "fail"]:
if self.params_cache.debug.skip_bad_events:
print "Skipping event %s: possibly caused an unknown exception previously"%ts
return
elif self.params_cache.debug.skip_processed_events:
print "Skipping event %s: processed successfully previously"%ts
return
else:
if self.params_cache.debug.skip_unprocessed_events:
print "Skipping event %s: not processed previously"%ts
return
self.debug_start(ts)
evt = run.event(timestamp)
if evt.get("skip_event") or "skip_event" in [key.key() for key in evt.keys()]:
print "Skipping event",ts
self.debug_write("psana_skip", "skip")
return
print "Accepted", ts
self.params = copy.deepcopy(self.params_cache)
# the data needs to have already been processed and put into the event by psana
if self.params.format.file_format == 'cbf':
# get numpy array, 32x185x388
data = cspad_cbf_tbx.get_psana_corrected_data(self.psana_det, evt, use_default=False, dark=True,
common_mode=self.common_mode,
apply_gain_mask=self.params.format.cbf.gain_mask_value is not None,
gain_mask_value=self.params.format.cbf.gain_mask_value,
per_pixel_gain=False)
if data is None:
print "No data"
self.debug_write("no_data", "skip")
return
if self.params.format.cbf.override_distance is None:
distance = cspad_tbx.env_distance(self.params.input.address, run.env(), self.params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
self.debug_write("no_distance", "skip")
return
else:
distance = self.params.format.cbf.override_distance
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
self.debug_write("no_wavelength", "skip")
return
else:
wavelength = 12398.4187/self.params.format.cbf.override_energy
if self.params.format.file_format == 'pickle':
image_dict = evt.get(self.params.format.pickle.out_key)
data = image_dict['DATA']
timestamp = t = ts
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23]
print "Processing shot", s
if self.params.format.file_format == 'cbf':
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address)
if self.params.input.reference_geometry is not None:
from dxtbx.model import Detector
# copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead.
cspad_img._detector_instance = Detector.from_dict(self.reference_detector.to_dict())
cspad_img.sync_detector_to_cbf()
elif self.params.format.file_format == 'pickle':
from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory
cspad_img = FormatPYunspecifiedStillInMemory(image_dict)
cspad_img.timestamp = s
if self.params.dispatch.dump_all:
self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s))
self.cache_ranges(cspad_img, self.params)
imgset = MemImageSet([cspad_img])
if self.params.dispatch.estimate_gain_only:
from dials.command_line.estimate_gain import estimate_gain
estimate_gain(imgset)
return
if not self.params.dispatch.find_spots:
self.debug_write("data_loaded", "done")
return
datablock = DataBlockFactory.from_imageset(imgset)[0]
# before calling DIALS for processing, set output paths according to the templates
if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template:
self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.indexed_filename_template%("idx-" + s))
if "%s" in self.refined_experiments_filename_template:
self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.refined_experiments_filename_template%("idx-" + s))
if "%s" in self.integrated_filename_template:
self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.integrated_filename_template%("idx-" + s))
if "%s" in self.reindexedstrong_filename_template:
self.params.output.reindexedstrong_filename = os.path.join(self.params.output.output_dir, self.reindexedstrong_filename_template%("idx-" + s))
# Load a dials mask from the trusted range and psana mask
from dials.util.masking import MaskGenerator
generator = MaskGenerator(self.params.border_mask)
mask = generator.generate(imgset)
if self.params.format.file_format == "cbf":
mask = tuple([a&b for a, b in zip(mask,self.dials_mask)])
if self.spotfinder_mask is None:
self.params.spotfinder.lookup.mask = mask
else:
self.params.spotfinder.lookup.mask = tuple([a&b for a, b in zip(mask,self.spotfinder_mask)])
if self.integration_mask is None:
self.params.integration.lookup.mask = mask
else:
self.params.integration.lookup.mask = tuple([a&b for a, b in zip(mask,self.integration_mask)])
self.debug_write("spotfind_start")
try:
observed = self.find_spots(datablock)
except Exception, e:
import traceback; traceback.print_exc()
print str(e), "event", timestamp
self.debug_write("spotfinding_exception", "fail")
return
def process_event(self, run, timestamp):
"""
Process a single event from a run
@param run psana run object
@param timestamp psana timestamp object
"""
ts = cspad_tbx.evt_timestamp(
(timestamp.seconds(), timestamp.nanoseconds() / 1e6))
if ts is None:
print "No timestamp, skipping shot"
return
if len(self.params_cache.debug.event_timestamp
) > 0 and ts not in self.params_cache.debug.event_timestamp:
return
if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events:
if ts in self.known_events:
if self.known_events[ts] not in ["stop", "done", "fail"]:
if self.params_cache.debug.skip_bad_events:
print "Skipping event %s: possibly caused an unknown exception previously" % ts
return
elif self.params_cache.debug.skip_processed_events:
print "Skipping event %s: processed successfully previously" % ts
return
else:
if self.params_cache.debug.skip_unprocessed_events:
print "Skipping event %s: not processed previously" % ts
return
self.debug_start(ts)
evt = run.event(timestamp)
if evt.get("skip_event") or "skip_event" in [
key.key() for key in evt.keys()
]:
print "Skipping event", ts
self.debug_write("psana_skip", "skip")
return
print "Accepted", ts
self.params = copy.deepcopy(self.params_cache)
# the data needs to have already been processed and put into the event by psana
if self.params.format.file_format == 'cbf':
# get numpy array, 32x185x388
data = cspad_cbf_tbx.get_psana_corrected_data(
self.psana_det,
evt,
use_default=False,
dark=True,
common_mode=self.common_mode,
apply_gain_mask=self.params.format.cbf.gain_mask_value
is not None,
gain_mask_value=self.params.format.cbf.gain_mask_value,
per_pixel_gain=False)
if data is None:
print "No data"
self.debug_write("no_data", "skip")
return
if self.params.format.cbf.override_distance is None:
distance = cspad_tbx.env_distance(
self.params.input.address, run.env(),
self.params.format.cbf.detz_offset)
if distance is None:
print "No distance, skipping shot"
self.debug_write("no_distance", "skip")
return
else:
distance = self.params.format.cbf.override_distance
if self.params.format.cbf.override_energy is None:
wavelength = cspad_tbx.evt_wavelength(evt)
if wavelength is None:
print "No wavelength, skipping shot"
self.debug_write("no_wavelength", "skip")
return
else:
wavelength = 12398.4187 / self.params.format.cbf.override_energy
if self.params.format.file_format == 'pickle':
image_dict = evt.get(self.params.format.pickle.out_key)
data = image_dict['DATA']
timestamp = t = ts
s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[
20:23]
print "Processing shot", s
if self.params.format.file_format == 'cbf':
# stitch together the header, data and metadata into the final dxtbx format object
cspad_img = cspad_cbf_tbx.format_object_from_data(
self.base_dxtbx, data, distance, wavelength, timestamp,
self.params.input.address)
if self.params.input.reference_geometry is not None:
from dxtbx.model import Detector
# copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead.
cspad_img._detector_instance = Detector.from_dict(
self.reference_detector.to_dict())
cspad_img.sync_detector_to_cbf()
elif self.params.format.file_format == 'pickle':
from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory
cspad_img = FormatPYunspecifiedStillInMemory(image_dict)
cspad_img.timestamp = s
if self.params.dispatch.dump_all:
self.save_image(
cspad_img, self.params,
os.path.join(self.params.output.output_dir, "shot-" + s))
self.cache_ranges(cspad_img, self.params)
imgset = MemImageSet([cspad_img])
if self.params.dispatch.estimate_gain_only:
from dials.command_line.estimate_gain import estimate_gain
estimate_gain(imgset)
return
if not self.params.dispatch.find_spots:
self.debug_write("data_loaded", "done")
return
datablock = DataBlockFactory.from_imageset(imgset)[0]
# before calling DIALS for processing, set output paths according to the templates
if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template:
self.params.output.indexed_filename = os.path.join(
self.params.output.output_dir,
self.indexed_filename_template % ("idx-" + s))
if "%s" in self.refined_experiments_filename_template:
self.params.output.refined_experiments_filename = os.path.join(
self.params.output.output_dir,
self.refined_experiments_filename_template % ("idx-" + s))
if "%s" in self.integrated_filename_template:
self.params.output.integrated_filename = os.path.join(
self.params.output.output_dir,
self.integrated_filename_template % ("idx-" + s))
if "%s" in self.reindexedstrong_filename_template:
self.params.output.reindexedstrong_filename = os.path.join(
self.params.output.output_dir,
self.reindexedstrong_filename_template % ("idx-" + s))
# Load a dials mask from the trusted range and psana mask
from dials.util.masking import MaskGenerator
generator = MaskGenerator(self.params.border_mask)
mask = generator.generate(imgset)
if self.params.format.file_format == "cbf":
mask = tuple([a & b for a, b in zip(mask, self.dials_mask)])
if self.spotfinder_mask is None:
self.params.spotfinder.lookup.mask = mask
else:
self.params.spotfinder.lookup.mask = tuple(
[a & b for a, b in zip(mask, self.spotfinder_mask)])
if self.integration_mask is None:
self.params.integration.lookup.mask = mask
else:
self.params.integration.lookup.mask = tuple(
[a & b for a, b in zip(mask, self.integration_mask)])
self.debug_write("spotfind_start")
try:
observed = self.find_spots(datablock)
except Exception, e:
import traceback
traceback.print_exc()
print str(e), "event", timestamp
self.debug_write("spotfinding_exception", "fail")
return
def generate_mask(
experiments, # type: ExperimentList
params, # type: phil.scope_extract
):
# type: (...) -> Tuple[Masks, Optional[ExperimentList]]
"""
Generate a pixel mask for each imageset in an experiment list.
Use the masking parameters :param:`params` and the experiments in the experiment
list :param:`experiments` to define a pixel mask for each of the associated
imagesets. The masks are generated using :mod:`dials.util.masking`.
The masks will be saved to disk at the location specified by
:attr:`params.output.mask`. If the experiment list contains more than one
imageset, multiple mask files will be produced, with filenames differentiated by
an appended number. Optionally, if a path :attr:`params.output.experiments` is
set, a modified copy of :param:`experiments` with the masks applied will be saved
to that location.
Args:
experiments: An experiment list containing only one imageset.
params: Masking parameters, having the structure defined in
:data:`phil_scope`.
Returns:
A list of masks, one for each imageset.
A copy of :param:`experiments` with the masks applied (optional,
only returned if :attr:`params.output.experiments` is set).
"""
imagesets = experiments.imagesets()
masks = []
# Create output mask filenames
num_imagesets = len(imagesets)
if num_imagesets == 1:
filenames = [params.output.mask]
else:
# If there is more than one imageset, append a number to each output filename
name, ext = os.path.splitext(params.output.mask)
pad = len(str(num_imagesets))
filenames = [
"{name}_{num:0{pad}}{ext}".format(name=name,
num=i + 1,
pad=pad,
ext=ext)
for i in range(num_imagesets)
]
# Generate the mask
generator = MaskGenerator(params)
for imageset, filename in zip(imagesets, filenames):
mask = generator.generate(imageset)
masks.append(mask)
# Save the mask to file
log.info("Writing mask to %s", filename)
with open(filename, "wb") as fh:
pickle.dump(mask, fh)
if params.output.experiments:
# Apply the mask to the imageset
imageset.external_lookup.mask.data = ImageBool(mask)
imageset.external_lookup.mask.filename = filename
if params.output.experiments:
# Save the experiment list
log.info("Saving experiments to %s", params.output.experiments)
experiments.as_file(params.output.experiments)
else:
experiments = None
return masks, experiments
