def calculate_parameters(self, experiments=None):
"""Image modification for current cctbx.xfel."""
if not experiments:
# If data are given, apply modifications as specified below
error = "IOTA IMPORT ERROR: Experiment list not found!"
return None, error
else:
error = []
# Calculate auto-threshold
# TODO: Revisit this; I REALLY don't like it.
if self.auto_threshold:
beamX = self.img_object.final["beamX"]
beamY = self.img_object.final["beamY"]
px_size = self.img_object.final["pixel"]
try:
img = dxtbx.load(self.img_object.img_path)
raw_data = img.get_raw_data()
beam_x_px = int(beamX / px_size)
beam_y_px = int(beamY / px_size)
data_array = raw_data.as_numpy_array().astype(float)
self.center_int = np.nanmax(
data_array[
beam_y_px - 20 : beam_y_px + 20,
beam_x_px - 20 : beam_x_px + 20,
]
)
except Exception as e:
error.append(
"IOTA IMPORT ERROR: Auto-threshold failed! {}".format(e)
)
# Estimate gain (or set gain to 1.00 if cannot calculate)
if self.estimate_gain:
with util.Capturing() as junk_output:
try:
assert self.img_object.experiments # Must have experiments here
imageset = self.img_object.experiments.extract_imagesets()[0]
self.img_object.gain = estimate_gain(imageset)
except Exception as e:
error.append(
"IOTA IMPORT ERROR: Estimate gain failed! ".format(e)
)
# Collect error messages for logging
if error:
error_message = "\n".join(error)
else:
error_message = None
return experiments, error_message
def calculate_parameters(self, datablock=None):
''' Image modification for current cctbx.xfel '''
if not datablock:
# If data are given, apply modifications as specified below
error = 'IOTA IMPORT ERROR: Datablock not found!'
return None, error
else:
error = []
# Calculate auto-threshold
# TODO: Revisit this; I REALLY don't like it.
if self.iparams.cctbx_xfel.auto_threshold:
beamX = self.img_object.final['beamX']
beamY = self.img_object.final['beamY']
px_size = self.img_object.final['pixel']
try:
img = dxtbx.load(self.img_object.img_path)
raw_data = img.get_raw_data()
beam_x_px = int(beamX / px_size)
beam_y_px = int(beamY / px_size)
data_array = raw_data.as_numpy_array().astype(float)
self.center_int = np.nanmax(
data_array[beam_y_px - 20:beam_y_px + 20,
beam_x_px - 20:beam_x_px + 20])
except Exception as e:
error.append(
'IOTA IMPORT ERROR: Auto-threshold failed! {}'.format(
e))
# Estimate gain (or set gain to 1.00 if cannot calculate)
if self.iparams.advanced.estimate_gain:
with util.Capturing() as junk_output:
try:
assert self.img_object.datablock # Must have datablock here
imageset = self.img_object.datablock.extract_imagesets(
)[0]
self.img_object.gain = estimate_gain(imageset)
except Exception as e:
error.append(
'IOTA IMPORT ERROR: Estimate gain failed! '.format(
e))
# Collect error messages for logging
if error:
error_message = '\n'.join(error)
else:
error_message = None
return datablock, error_message
def import_and_process(self, input_entry):
img_object = self.importer.run(input_entry)
if img_object.status == 'imported':
with util.Capturing() as junk_output:
img_object = self.integrator.run(img_object)
# Update main log
if hasattr(self.info, 'logfile'):
main_log_entry = "\n".join(img_object.log_info)
util.main_log(self.info.logfile, main_log_entry)
util.main_log(self.info.logfile, '\n{:-^100}\n'.format(''))
# Set status to final
img_object.status = 'final'
return img_object
def get_hkl_slice(self, sg='P1', axis='l'):
try:
all_obs = ep.load(self.idx_file)
except IOError:
return None
with util.Capturing():
ext_sg = str(all_obs.space_group_info()).replace(' ', '').lower()
sg = sg.replace(' ', '').lower()
if ext_sg != sg:
all_obs = all_obs.change_symmetry(sg, merge_non_unique=False)
# Recalculate redundancies and slices from updated indices
red = all_obs.merge_equivalents().redundancies()
return red.slice(axis=axis, slice_start=0, slice_end=0)
def run_distl(self, params):
""" Performs a quick DISTL spotfinding and returns Bragg spots information.
"""
from spotfinder.applications import signal_strength
# run DISTL spotfinder
try:
with util.Capturing() as distl_output:
Org = signal_strength.run_signal_strength(params)
except NotImplementedError as e:
print("NOT IMPLEMENTED ERROR FOR {}".format(self.img))
# Extract relevant spotfinding info
for frame in Org.S.images.keys():
saturation = Org.Files.imageindex(frame).saturation
Bragg_spots = [
flex.sum(spot.wts)
for spot in Org.S.images[frame]['inlier_spots']
]
return Bragg_spots
def get_results(self, finished_objects=None):
if not finished_objects:
finished_objects = self.info.get_finished_objects()
if not finished_objects:
return False
final_objects = []
self.info.unplotted_stats = {}
for key in self.info.stats:
self.info.unplotted_stats[key] = dict(lst=[])
for obj in finished_objects:
item = [obj.input_index, obj.img_path, obj.img_index]
if len(self.info.unprocessed
) > 0 and item in self.info.unprocessed:
self.info.unprocessed.remove(item)
if (len(self.info.categories['not_processed'][0]) > 0
and item in self.info.categories['not_processed'][0]):
self.info.categories['not_processed'][0].remove(item)
if obj.fail:
key = obj.fail.replace(' ', '_')
if key in self.info.categories:
self.info.categories[key][0].append(item)
else:
self.info.categories['integrated'][0].append(
obj.final['final'])
self.info.final_objects.append(obj.obj_file)
final_objects.append(obj)
if not obj.fail or 'triage' not in obj.fail:
self.info.categories['have_diffraction'][0].append(
obj.img_path)
# Calculate processing stats from final objects
if final_objects:
self.info.pixel_size = final_objects[0].final['pixel_size']
# Get observations from file
try:
all_obs = ep.load(self.info.idx_file)
except Exception:
all_obs = None
# Collect image processing stats
for obj in final_objects:
for key in self.info.stats:
if key in obj.final:
stat_tuple = (obj.input_index, obj.img_path,
obj.img_index, obj.final[key])
self.info.stats[key]['lst'].append(stat_tuple)
if key not in self.info.unplotted_stats:
self.info.unplotted_stats[key] = dict(lst=[])
self.info.unplotted_stats[key]['lst'].append(
stat_tuple)
# Unit cells and space groups (i.e. cluster iterable)
self.info.cluster_iterable.append([
float(obj.final['a']),
float(obj.final['b']),
float(obj.final['c']),
float(obj.final['alpha']),
float(obj.final['beta']),
float(obj.final['gamma']),
str(obj.final['sg'])
])
# Get observations from this image
obs = None
if 'observations' in obj.final:
obs = obj.final['observations'].as_non_anomalous_array()
else:
pickle_path = obj.final['final']
if os.path.isfile(pickle_path):
try:
pickle = ep.load(pickle_path)
obs = pickle['observations'][
0].as_non_anomalous_array()
except Exception as e:
print('IMAGE_PICKLE_ERROR for {}: {}'.format(
pickle_path, e))
with util.Capturing():
if obs:
# Append observations to combined miller array
obs = obs.expand_to_p1()
if all_obs:
all_obs = all_obs.concatenate(
obs, assert_is_similar_symmetry=False)
else:
all_obs = obs
# Get B-factor from this image
try:
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(500)
observations_as_f = obs.as_amplitude_array()
observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(observations_as_f,
asu_contents,
e_statistics=True)
b_factor = wp.wilson_b
except RuntimeError as e:
b_factor = 0
print('B_FACTOR_ERROR: ', e)
self.info.b_factors.append(b_factor)
# Save collected observations to file
if all_obs:
ep.dump(self.info.idx_file, all_obs)
# Calculate dataset stats
for k in self.info.stats:
stat_list = list(zip(*self.info.stats[k]['lst']))[2]
stats = dict(lst=self.info.stats[k]['lst'],
median=np.median(stat_list),
mean=np.mean(stat_list),
std=np.std(stat_list),
max=np.max(stat_list),
min=np.min(stat_list),
cons=Counter(stat_list).most_common(1)[0][0])
self.info.stats[k].update(stats)
return True
else:
return False
def generate_phil_string(self, current_phil):
with util.Capturing() as txt_output:
current_phil.show()
self.phil_string = ''
for one_output in txt_output:
self.phil_string += one_output + '\n'
def process(self, img_object):
# write out DIALS info (tied to self.write_pickle)
if self.write_pickle:
self.params.output.indexed_filename = img_object.ridx_path
self.params.output.strong_filename = img_object.rspf_path
self.params.output.refined_experiments_filename = img_object.eref_path
self.params.output.integrated_experiments_filename = img_object.eint_path
self.params.output.integrated_filename = img_object.rint_path
# Set up integration pickle path and logfile
self.params.output.integration_pickle = img_object.int_file
self.int_log = img_object.int_log
# configure DIALS logging
self.dials_log = getattr(img_object, 'dials_log', None)
if self.dials_log:
log.config(verbosity=1, logfile=self.dials_log)
# Create output folder if one does not exist
if self.write_pickle:
if not os.path.isdir(img_object.int_path):
os.makedirs(img_object.int_path)
# Auto-set threshold and gain (not saved for target.phil)
if self.iparams.cctbx_xfel.auto_threshold:
center_int = img_object.center_int if img_object.center_int else 0
threshold = int(center_int)
self.params.spotfinder.threshold.dispersion.global_threshold = threshold
if self.iparams.image_import.estimate_gain:
self.params.spotfinder.threshold.dispersion.gain = img_object.gain
# Update geometry if reference geometry was applied
from dials.command_line.dials_import import ManualGeometryUpdater
update_geometry = ManualGeometryUpdater(self.params)
try:
imagesets = img_object.experiments.imagesets()
update_geometry(imagesets[0])
experiment = img_object.experiments[0]
experiment.beam = imagesets[0].get_beam()
experiment.detector = imagesets[0].get_detector()
except RuntimeError as e:
print("DEBUG: Error updating geometry on {}, {}".format(
img_object.img_path, e))
# Set detector if reference geometry was applied
if self.reference_detector is not None:
try:
from dxtbx.model import Detector
imageset = img_object.experiments[0].imageset
imageset.set_detector(
Detector.from_dict(self.reference_detector.to_dict()))
img_object.experiments[0].detector = imageset.get_detector()
except Exception as e:
print('DEBUG: cannot set detector! ', e)
# Write full params to file (DEBUG)
if self.write_logs:
param_string = phil_scope.format(
python_object=self.params).as_str()
full_param_dir = os.path.dirname(self.iparams.cctbx_xfel.target)
full_param_fn = 'full_' + os.path.basename(
self.iparams.cctbx_xfel.target)
full_param_file = os.path.join(full_param_dir, full_param_fn)
with open(full_param_file, 'w') as ftarg:
ftarg.write(param_string)
# **** SPOTFINDING **** #
with util.Capturing() as output:
try:
print("{:-^100}\n".format(" SPOTFINDING: "))
print('<--->')
observed = self.find_spots(img_object.experiments)
img_object.final['spots'] = len(observed)
except Exception as e:
e_spf = str(e)
observed = None
else:
if (self.iparams.data_selection.image_triage
and len(observed) >= self.iparams.data_selection.
image_triage.minimum_Bragg_peaks):
msg = " FOUND {} SPOTS - IMAGE ACCEPTED!".format(
len(observed))
print("{:-^100}\n\n".format(msg))
else:
msg = " FOUND {} SPOTS - IMAGE REJECTED!".format(
len(observed))
print("{:-^100}\n\n".format(msg))
e = 'Insufficient spots found ({})!'.format(len(observed))
return self.error_handler(e, 'triage', img_object, output)
if not observed:
return self.error_handler(e_spf, 'spotfinding', img_object, output)
if self.write_logs:
self.write_int_log(path=img_object.int_log,
output=output,
dials_log=self.dials_log)
# Finish if spotfinding is the last processing stage
if 'spotfind' in self.last_stage:
try:
detector = img_object.experiments.unique_detectors()[0]
beam = img_object.experiments.unique_beams()[0]
except AttributeError:
detector = img_object.experiments.imagesets()[0].get_detector()
beam = img_object.experiments.imagesets()[0].get_beam()
s1 = flex.vec3_double()
for i in range(len(observed)):
s1.append(detector[observed['panel'][i]].get_pixel_lab_coord(
observed['xyzobs.px.value'][i][0:2]))
two_theta = s1.angle(beam.get_s0())
d = beam.get_wavelength() / (2 * flex.asin(two_theta / 2))
img_object.final['res'] = np.max(d)
img_object.final['lres'] = np.min(d)
return img_object
# **** INDEXING **** #
with util.Capturing() as output:
try:
print("{:-^100}\n".format(" INDEXING"))
print('<--->')
experiments, indexed = self.index(img_object.experiments,
observed)
except Exception as e:
e_idx = str(e)
indexed = None
else:
if indexed:
img_object.final['indexed'] = len(indexed)
print("{:-^100}\n\n".format(" USED {} INDEXED REFLECTIONS "
"".format(len(indexed))))
else:
e_idx = "Not indexed for unspecified reason(s)"
img_object.fail = 'failed indexing'
if indexed:
if self.write_logs:
self.write_int_log(path=img_object.int_log,
output=output,
dials_log=self.dials_log)
else:
return self.error_handler(e_idx, 'indexing', img_object, output)
with util.Capturing() as output:
# Bravais lattice and reindex
if self.iparams.cctbx_xfel.determine_sg_and_reindex:
try:
print("{:-^100}\n".format(" DETERMINING SPACE GROUP"))
print('<--->')
experiments, indexed = self.pg_and_reindex(
indexed, experiments)
img_object.final['indexed'] = len(indexed)
lat = experiments[0].crystal.get_space_group().info()
sg = str(lat).replace(' ', '')
if sg != 'P1':
print("{:-^100}\n".format(
" REINDEXED TO SPACE GROUP {} ".format(sg)))
else:
print("{:-^100}\n".format(
" RETAINED TRICLINIC (P1) SYMMETRY "))
reindex_success = True
except Exception as e:
e_ridx = str(e)
reindex_success = False
if reindex_success:
if self.write_logs:
self.write_int_log(path=img_object.int_log,
output=output,
dials_log=self.dials_log)
else:
return self.error_handler(e_ridx, 'indexing', img_object,
output)
# **** REFINEMENT **** #
with util.Capturing() as output:
try:
experiments, indexed = self.refine(experiments, indexed)
refined = True
except Exception as e:
e_ref = str(e)
refined = False
if refined:
if self.write_logs:
self.write_int_log(path=img_object.int_log,
output=output,
dials_log=self.dials_log)
else:
return self.error_handler(e_ref, 'refinement', img_object, output)
# **** INTEGRATION **** #
with util.Capturing() as output:
try:
print("{:-^100}\n".format(" INTEGRATING "))
print('<--->')
integrated = self.integrate(experiments, indexed)
except Exception as e:
e_int = str(e)
integrated = None
else:
if integrated:
img_object.final['integrated'] = len(integrated)
print("{:-^100}\n\n".format(
" FINAL {} INTEGRATED REFLECTIONS "
"".format(len(integrated))))
if integrated:
if self.write_logs:
self.write_int_log(path=img_object.int_log,
output=output,
dials_log=self.dials_log)
else:
return self.error_handler(e_int, 'integration', img_object, output)
# Filter
if self.iparams.cctbx_xfel.filter.flag_on:
self.selector = Selector(
frame=self.frame,
uc_tol=self.iparams.cctbx_xfel.filter.uc_tolerance,
xsys=self.iparams.cctbx_xfel.filter.crystal_system,
pg=self.iparams.cctbx_xfel.filter.pointgroup,
uc=self.iparams.cctbx_xfel.filter.unit_cell,
min_ref=self.iparams.cctbx_xfel.filter.min_reflections,
min_res=self.iparams.cctbx_xfel.filter.min_resolution)
fail, e = self.selector.result_filter()
if fail:
return self.error_handler(e, 'filter', img_object, output)
int_results, log_entry = self.collect_information(
img_object=img_object)
# Update final entry with integration results
img_object.final.update(int_results)
# Update image log
log_entry = "\n".join(log_entry)
img_object.log_info.append(log_entry)
if self.write_logs:
self.write_int_log(path=img_object.int_log, log_entry=log_entry)
return img_object
def process(self, img_object):
# write out DIALS info
pfx = os.path.splitext(img_object.obj_file)[0]
self.params.output.experiments_filename = pfx + '_experiments.json'
self.params.output.indexed_filename = pfx + '_indexed.pickle'
self.params.output.strong_filename = pfx + '_strong.pickle'
self.params.output.refined_experiments_filename = pfx + '_refined_experiments.json'
self.params.output.integrated_experiments_filename = pfx + '_integrated_experiments.json'
self.params.output.integrated_filename = pfx + '_integrated.pickle'
# Set up integration pickle path and logfile
self.params.verbosity = 10
self.params.output.integration_pickle = img_object.int_file
self.int_log = img_object.int_log
# Create output folder if one does not exist
if self.write_pickle:
if not os.path.isdir(img_object.int_path):
os.makedirs(img_object.int_path)
if not img_object.experiments:
from dxtbx.model.experiment_list import ExperimentListFactory as exp
img_object.experiments = exp.from_filenames([img_object.img_path])[0]
# Auto-set threshold and gain (not saved for target.phil)
if self.iparams.cctbx_xfel.auto_threshold:
threshold = int(img_object.center_int)
self.params.spotfinder.threshold.dispersion.global_threshold = threshold
if self.iparams.image_import.estimate_gain:
self.params.spotfinder.threshold.dispersion.gain = img_object.gain
# Update geometry if reference geometry was applied
from dials.command_line.dials_import import ManualGeometryUpdater
update_geometry = ManualGeometryUpdater(self.params)
try:
imagesets = img_object.experiments.imagesets()
update_geometry(imagesets[0])
experiment = img_object.experiments[0]
experiment.beam = imagesets[0].get_beam()
experiment.detector = imagesets[0].get_detector()
except RuntimeError as e:
print("DEBUG: Error updating geometry on {}, {}".format(
img_object.img_path, e))
# Set detector if reference geometry was applied
if self.reference_detector is not None:
try:
from dxtbx.model import Detector
imageset = img_object.experiments[0].imageset
imageset.set_detector(
Detector.from_dict(self.reference_detector.to_dict())
)
img_object.experiments[0].detector = imageset.get_detector()
except Exception as e:
print ('DEBUG: cannot set detector! ', e)
proc_output = []
# **** SPOTFINDING **** #
with util.Capturing() as output:
try:
print ("{:-^100}\n".format(" SPOTFINDING: "))
observed = self.find_spots(img_object.experiments)
img_object.final['spots'] = len(observed)
except Exception as e:
return self.error_handler(e, 'spotfinding', img_object, output)
else:
if (
self.iparams.image_import.image_triage and
len(observed) >= self.iparams.image_import.minimum_Bragg_peaks
):
msg = " FOUND {} SPOTS - IMAGE ACCEPTED!".format(len(observed))
print("{:-^100}\n\n".format(msg))
else:
msg = " FOUND {} SPOTS - IMAGE REJECTED!".format(len(observed))
print("{:-^100}\n\n".format(msg))
e = 'Insufficient spots found ({})!'.format(len(observed))
return self.error_handler(e, 'triage', img_object, output)
proc_output.extend(output)
# Finish if spotfinding is the last processing stage
if 'spotfind' in self.last_stage:
detector = img_object.experiments.unique_detectors()[0]
beam = img_object.experiments.unique_beams()[0]
s1 = flex.vec3_double()
for i in range(len(observed)):
s1.append(detector[observed['panel'][i]].get_pixel_lab_coord(
observed['xyzobs.px.value'][i][0:2]))
two_theta = s1.angle(beam.get_s0())
d = beam.get_wavelength() / (2 * flex.asin(two_theta / 2))
img_object.final['res'] = np.max(d)
img_object.final['lres'] = np.min(d)
return img_object
# **** INDEXING **** #
with util.Capturing() as output:
try:
print ("{:-^100}\n".format(" INDEXING "))
experiments, indexed = self.index(img_object.experiments, observed)
except Exception as e:
return self.error_handler(e, 'indexing', img_object, output)
else:
if indexed:
img_object.final['indexed'] = len(indexed)
print ("{:-^100}\n\n".format(" USED {} INDEXED REFLECTIONS "
"".format(len(indexed))))
else:
img_object.fail = 'failed indexing'
return img_object
# Bravais lattice and reindex
if self.iparams.cctbx_xfel.determine_sg_and_reindex:
try:
print ("{:-^100}\n".format(" DETERMINING SPACE GROUP "))
experiments, indexed = self.pg_and_reindex(indexed, experiments)
img_object.final['indexed'] = len(indexed)
lat = experiments[0].crystal.get_space_group().info()
sg = str(lat).replace(' ', '')
if sg != 'P1':
print ("{:-^100}\n".format(" REINDEXED TO SPACE GROUP {} ".format(sg)))
else:
print ("{:-^100}\n".format(" RETAINED TRICLINIC (P1) SYMMETRY "))
except Exception as e:
return self.error_handler(e, 'indexing', img_object, output)
proc_output.extend(output)
# **** INTEGRATION **** #
with util.Capturing() as output:
try:
experiments, indexed = self.refine(experiments, indexed)
print ("{:-^100}\n".format(" INTEGRATING "))
integrated = self.integrate(experiments, indexed)
except Exception as e:
return self.error_handler(e, 'integration', img_object, output)
else:
if integrated:
img_object.final['integrated'] = len(integrated)
print ("{:-^100}\n\n".format(" FINAL {} INTEGRATED REFLECTIONS "
"".format(len(integrated))))
proc_output.extend(output)
# Filter
if self.iparams.cctbx_xfel.filter.flag_on:
self.selector = Selector(frame=self.frame,
uc_tol=self.iparams.cctbx_xfel.filter.uc_tolerance,
xsys=self.iparams.cctbx_xfel.filter.crystal_system,
pg=self.iparams.cctbx_xfel.filter.pointgroup,
uc=self.iparams.cctbx_xfel.filter.unit_cell,
min_ref=self.iparams.cctbx_xfel.filter.min_reflections,
min_res=self.iparams.cctbx_xfel.filter.min_resolution)
fail, e = self.selector.result_filter()
if fail:
return self.error_handler(e, 'filter', img_object, proc_output)
int_results, log_entry = self.collect_information(img_object=img_object)
# Update final entry with integration results
img_object.final.update(int_results)
# Update image log
log_entry = "\n".join(log_entry)
img_object.log_info.append(log_entry)
if self.write_logs:
with open(img_object.int_log, 'w') as tf:
for i in proc_output:
if 'cxi_version' not in i:
tf.write('\n{}'.format(i))
tf.write('\n{}'.format(log_entry))
return img_object
def get_results(self, finished_objects=None):
if not finished_objects:
finished_objects = self.info.get_finished_objects()
if not finished_objects:
return False
final_objects = []
self.info.unplotted_stats = {}
for key in self.info.stats:
self.info.unplotted_stats[key] = dict(lst=[])
for obj in finished_objects:
item = [obj.input_index, obj.img_path, obj.img_index]
if len(self.info.unprocessed) > 0 and item in self.info.unprocessed:
self.info.unprocessed.remove(item)
if (
len(self.info.categories["not_processed"][0]) > 0
and item in self.info.categories["not_processed"][0]
):
self.info.categories["not_processed"][0].remove(item)
if obj.fail:
key = obj.fail.replace(" ", "_")
if key in self.info.categories:
self.info.categories[key][0].append(item)
else:
self.info.categories["integrated"][0].append(obj.final["final"])
self.info.final_objects.append(obj.obj_file)
final_objects.append(obj)
if not obj.fail or "triage" not in obj.fail:
self.info.categories["have_diffraction"][0].append(obj.img_path)
# Calculate processing stats from final objects
if final_objects:
self.info.pixel_size = final_objects[0].final["pixel_size"]
# Get observations from file
try:
all_obs = ep.load(self.info.idx_file)
except Exception:
all_obs = None
# Collect image processing stats
for obj in final_objects:
for key in self.info.stats:
if key in obj.final:
stat_tuple = (
obj.input_index,
obj.img_path,
obj.img_index,
obj.final[key],
)
self.info.stats[key]["lst"].append(stat_tuple)
# add proc filepath info to 'pointers'
pointer_dict = {
"img_file": obj.img_path,
"obj_file": obj.obj_file,
"img_index": obj.img_index,
"experiments": obj.eint_path,
"reflections": obj.rint_path,
}
self.info.pointers[str(obj.input_index)] = pointer_dict
if key not in self.info.unplotted_stats:
self.info.unplotted_stats[key] = dict(lst=[])
self.info.unplotted_stats[key]["lst"].append(stat_tuple)
# Unit cells and space groups (i.e. cluster iterable)
self.info.cluster_iterable.append(
[
float(obj.final["a"]),
float(obj.final["b"]),
float(obj.final["c"]),
float(obj.final["alpha"]),
float(obj.final["beta"]),
float(obj.final["gamma"]),
str(obj.final["sg"]),
]
)
# Get observations from this image
obs = None
if "observations" in obj.final:
obs = obj.final["observations"].as_non_anomalous_array()
else:
pickle_path = obj.final["final"]
if os.path.isfile(pickle_path):
try:
pickle = ep.load(pickle_path)
obs = pickle["observations"][0].as_non_anomalous_array()
except Exception as e:
print(
"IMAGE_PICKLE_ERROR for {}: {}".format(pickle_path, e)
)
with util.Capturing():
if obs:
# Append observations to combined miller array
obs = obs.expand_to_p1()
if all_obs:
all_obs = all_obs.concatenate(
obs, assert_is_similar_symmetry=False
)
else:
all_obs = obs
# Get B-factor from this image
try:
mxh = mx_handler()
asu_contents = mxh.get_asu_contents(500)
observations_as_f = obs.as_amplitude_array()
observations_as_f.setup_binner(auto_binning=True)
wp = statistics.wilson_plot(
observations_as_f, asu_contents, e_statistics=True
)
b_factor = wp.wilson_b
except RuntimeError as e:
b_factor = 0
print("B_FACTOR_ERROR: ", e)
self.info.b_factors.append(b_factor)
# Save collected observations to file
if all_obs:
ep.dump(self.info.idx_file, all_obs)
# Calculate dataset stats
for k in self.info.stats:
stat_list = list(zip(*self.info.stats[k]["lst"]))[3]
stats = dict(
lst=self.info.stats[k]["lst"],
median=np.median(stat_list).item(),
mean=np.mean(stat_list).item(),
std=np.std(stat_list).item(),
max=np.max(stat_list).item(),
min=np.min(stat_list).item(),
cons=Counter(stat_list).most_common(1)[0][0],
)
self.info.stats[k].update(stats)
return True
else:
return False
def load_image_file(self, filepath):
""" Reads a single image file (e.g. a CBF or MCCD) and returns raw data,
experiment information, and an image type (e.g. pickle, raw image, or none)
:param img: path to image file
:return: data: raw image data and experiment info
img_type: which type of image this was, or None if not loaded
"""
error = None
try:
with util.Capturing() as junk_output:
loaded_img = dxtbx.load(filepath)
except Exception as e:
error = 'IOTA IMPORTER ERROR: DXTBX failed! {}'.format(e)
print(e)
loaded_img = None
# Extract image information
if loaded_img is not None:
raw_data = loaded_img.get_raw_data()
detector = loaded_img.get_detector()[0]
beam = loaded_img.get_beam()
scan = loaded_img.get_scan()
distance = detector.get_distance()
pixel_size = detector.get_pixel_size()[0]
overload = detector.get_trusted_range()[1]
wavelength = beam.get_wavelength()
beam_x = detector.get_beam_centre(beam.get_s0())[0]
beam_y = detector.get_beam_centre(beam.get_s0())[1]
if scan is None:
timestamp = None
else:
msec, sec = math.modf(scan.get_epochs()[0])
timestamp = evt_timestamp((sec, msec))
# Assemble datapack
data = dpack(data=raw_data,
distance=distance,
pixel_size=pixel_size,
wavelength=wavelength,
beam_center_x=beam_x,
beam_center_y=beam_y,
ccd_image_saturation=overload,
saturated_value=overload,
timestamp=timestamp)
if scan is not None:
osc_start, osc_range = scan.get_oscillation()
if osc_start != osc_range:
data['OSC_START'] = 0 #osc_start
data['OSC_RANGE'] = 0 #osc_start
data['TIME'] = scan.get_exposure_times()[0]
# Populate image object information
self.img_object.final['pixel_size'] = pixel_size
self.img_object.final['img_size'] = (data['SIZE1'], data['SIZE2'])
self.img_object.final['beamX'] = beam_x
self.img_object.final['beamY'] = beam_y
self.img_object.final['gain'] = detector.get_gain()
self.img_object.final['distance'] = distance
self.img_object.final['wavelength'] = wavelength
else:
data = None
return data, error
def integrate(self, grid_point):
""" Runs the integration module in cctbx.xfel; used by either grid-search or
final integration function. """
self.s = grid_point['sih']
self.h = grid_point['sph']
self.a = grid_point['spa']
args = self.args
# Generate advanced arguments (and PDF subfolder)
if self.charts and self.tag == 'grid search':
filename = os.path.basename(self.img).split('.')[0]
pdf_folder = os.path.join(self.viz, 'pdf_{}/s{}_h{}_a{}'\
''.format(filename, self.s, self.h, self.a))
if not os.path.exists(pdf_folder):
os.makedirs(pdf_folder)
self.args.extend([
"integration.enable_residual_map=True",
"integration.enable_residual_scatter=True",
"integration.mosaic.enable_AD14F7B=True",
"integration.graphics_backend=pdf",
"integration.pdf_output_dir={}".format(pdf_folder)
])
if self.tag == 'integrate':
args.append("indexing.completeness_pickle={}".format(self.out_img))
#Actually run integration using iota.bulletproof
error_message = ''
with util.Capturing() as index_log:
arguments = [
"distl.minimum_signal_height={}".format(
str(self.s)), "distl.minimum_spot_height={}".format(
str(self.h)), "distl.minimum_spot_area={}".format(
str(self.a)), "indexing.open_wx_viewer=False"
] + list(args[1:])
tmppath = os.path.join(self.tmp_base,
str(uuid.uuid4()) + ".pickle")
assert not os.path.exists(tmppath)
# invoke the indexer in a way that will protect iota from any crashes
command = "iota.bulletproof {} {} {}" \
"".format(tmppath, self.target, " ".join(arguments))
try:
easy_run.fully_buffered(command,
join_stdout_stderr=True).show_stdout()
if not os.path.exists(tmppath):
print(tmppath)
print(command)
raise Exception("Indexing failed for an unknown reason")
# iota.bulletproof saves the needed results from indexing in a tmp file
result = easy_pickle.load(tmppath)
os.remove(tmppath)
if isinstance(result, str):
raise Exception(result)
else:
int_final = result
except Exception, e:
int_final = None
if hasattr(e, "classname"):
print(
e.classname,
"for %s:" % self.img,
)
error_message = "{}: {}".format(
e.classname, e[0].replace('\n', ' ')[:50])
else:
print("Integration error for %s:" % self.img, )
error_message = "{}".format(str(e).replace('\n', ' ')[:50])
print(e)
