def run(self):
''' Parse the options. '''
from dials.util.options import flatten_experiments, flatten_reflections
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
assert len(reflections) == 1
reflections = reflections[0]
nvparams = dsp_phil.extract()
domain_size = flex.double()
mosaic_angle = flex.double()
filtered_reflections = flex.reflection_table()
for i in range(len(experiments)):
refls = reflections.select(reflections['id'] == i)
tmp = refls['id']
refls['id'] = flex.int(len(refls), 0)
try:
nv = NaveParameters(params = nvparams, experiments=experiments[i:i+1], reflections=refls, refinery=None, graph_verbose=False)
crystal_model_nv = nv()[0]
except Exception as e:
print("Error recomputing mosaicity for experiment %d: %s"%(i, str(e)))
continue
domain_size.append(experiments[i].crystal.get_domain_size_ang() - crystal_model_nv.get_domain_size_ang())
mosaic_angle.append(experiments[i].crystal.get_half_mosaicity_deg() - crystal_model_nv.get_half_mosaicity_deg())
experiments[i].crystal = crystal_model_nv
refls['id'] = tmp
refls = refls.select(nv.nv_acceptance_flags)
filtered_reflections.extend(refls)
print("Saving new experiments as %s"%params.output.experiments)
experiments.as_file(params.output.experiments)
print("Removed %d out of %d reflections as outliers"%(len(reflections) - len(filtered_reflections), len(reflections)))
print("Saving filtered reflections as %s"%params.output.experiments)
filtered_reflections.as_pickle(params.output.reflections)
if params.plot_changes:
from matplotlib import pyplot as plt
domain_size = domain_size.select((domain_size >= -10) & (domain_size <= 10))
mosaic_angle = mosaic_angle.select((mosaic_angle >= -0.1) & (mosaic_angle <= 0.1))
for d in [domain_size, mosaic_angle]:
f = plt.figure()
plt.hist(d, bins=30)
plt.show()
def __init__(self, img, gain, params, center_intensity=0):
""" Initialization and data read-in
"""
self.gain = gain
self.params = params
# Read settings from the DIALS target (.phil) file
# If none is provided, use default settings (and may God have mercy)
if self.params.dials.target != None:
with open(self.params.dials.target, 'r') as settings_file:
settings_file_contents = settings_file.read()
settings = parse(settings_file_contents)
current_phil = phil_scope.fetch(sources=[settings])
self.phil = current_phil.extract()
else:
self.phil = phil_scope.extract()
# Modify settings
self.phil.output.strong_filename = None
self.processor = IOTADialsProcessor(params=self.phil)
# Set customized parameters
beamX = self.params.image_conversion.beam_center.x
beamY = self.params.image_conversion.beam_center.y
if beamX != 0 or beamY != 0:
self.phil.geometry.detector.slow_fast_beam_centre = '{} {}'.format(
beamY, beamX)
if self.params.image_conversion.distance != 0:
self.phil.geometry.detector.distance = self.params.image_conversion.distance
if self.params.advanced.estimate_gain:
self.phil.spotfinder.threshold.dispersion.gain = gain
if self.params.image_conversion.mask is not None:
self.phil.spotfinder.lookup.mask = self.params.image_conversion.mask
self.phil.integration.lookup.mask = self.params.image_conversion.mask
if self.params.dials.auto_threshold:
threshold = int(center_intensity)
self.phil.spotfinder.threshold.dispersion.global_threshold = threshold
# Convert raw image into single-image datablock
with misc.Capturing() as junk_output:
self.datablock = DataBlockFactory.from_filenames([img])[0]
def msi(jid):
"""
msi stands for mad --> spot --> index
Here we load a data image that has crystal diffraction
then we spot peaks and index the crystal image
"""
#time.sleep(jid)
import sys
import numpy as np
import glob
import os
from copy import deepcopy
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dials.command_line.stills_process import phil_scope\
as indexer_phil_scope
from dials.algorithms.indexing.lattice_search import basis_vector_search_phil_scope
from dials.array_family import flex
import dxtbx
from dxtbx.datablock import DataBlockFactory
from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentList, Experiment
from libtbx.utils import Sorry as Sorry
from libtbx.phil import parse
from cxi_xdr_xes.command_line.two_color_process import two_color_phil_scope
from cxi_xdr_xes.two_color import two_color_indexer
indexer_two_color = two_color_indexer.indexer_two_color
from cctbx import crystal
from cxid9114 import utils
from cxid9114 import parameters
indexer_phil_scope.adopt_scope(two_color_phil_scope)
indexer_phil_scope.adopt_scope(basis_vector_search_phil_scope)
mad_index_params = indexer_phil_scope.extract()
fnames = glob.glob(glob_str)
out_dir = os.path.join( args.out_dir , "job%d" % jid)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
refls_dir = "refls_%s" % tag
Els_dir = "Els_%s" % tag
dumps_dir = "dumps_%s" % tag
for o in [refls_dir, Els_dir, dumps_dir]:
sub_o = os.path.join(out_dir, o)
if not os.path.exists(sub_o):
os.makedirs(sub_o)
# track shots that indexed, or shots that
# had too few spots to index, so can change parameters and try again
def load_tracker_f(fname):
data = []
if os.path.exists(fname):
data = np.loadtxt(fname, str)
if data.size and not data.shape:
data = list(set(data[None].astype(int)))
else:
data = list(set(data.astype(int)))
return data
skip_weak = False
skip_failed = True #False
skip_indexed = False
weak_shots_f = os.path.join(out_dir, "weak_shots.txt")
failed_idx_f = os.path.join(out_dir, "failed_shots.txt")
indexed_f = os.path.join(out_dir, "indexed_shots.txt")
weak_shots = load_tracker_f(weak_shots_f)
failed_shots = load_tracker_f(failed_idx_f)
indexed_shots = load_tracker_f(indexed_f)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# ------ spotting parameters
spot_par = find_spots_phil_scope.fetch(source=parse("")).extract()
spot_par.spotfinder.threshold.dispersion.global_threshold = 40
spot_par.spotfinder.threshold.dispersion.gain = 28
spot_par.spotfinder.threshold.dispersion.kernel_size = [2,2]
spot_par.spotfinder.threshold.dispersion.sigma_strong = 1
spot_par.spotfinder.threshold.dispersion.sigma_background = 6
spot_par.spotfinder.filter.min_spot_size = 3
spot_par.spotfinder.force_2d = True
# ------ indexing parameters
KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P1")
#KNOWN_SYMMETRY = crystal.symmetry("79,79,38,90,90,90", "P43212")
mad_index_params.refinement.parameterisation.beam.fix = "all"
mad_index_params.refinement.parameterisation.detector.fix = "all"
#mad_index_params.refinement.verbosity = 3
#mad_index_params.refinement.reflections.outlier.algorithm = "null"
mad_index_params.indexing.stills.refine_all_candidates = do_stills_refine
mad_index_params.indexing.optimise_initial_basis_vectors = do_basis_refine
#mad_index_params.indexing.stills.refine_candidates_with_known_symmetry = False
mad_index_params.indexing.known_symmetry.space_group = KNOWN_SYMMETRY.space_group_info()
mad_index_params.indexing.known_symmetry.unit_cell = KNOWN_SYMMETRY.unit_cell()
mad_index_params.indexing.refinement_protocol.d_min_start = None
mad_index_params.indexing.debug = True
mad_index_params.indexing.real_space_grid_search.characteristic_grid = 0.02
mad_index_params.indexing.known_symmetry.absolute_angle_tolerance = 5.0
mad_index_params.indexing.known_symmetry.relative_length_tolerance = 0.3
mad_index_params.indexing.stills.rmsd_min_px = 20000
mad_index_params.indexing.refinement_protocol.n_macro_cycles = 1
mad_index_params.indexing.multiple_lattice_search.max_lattices = 1
mad_index_params.indexing.basis_vector_combinations.max_refine = 10000000000
#mad_index_params.indexing.basis_vector_combinations.max_combinations = 150
#mad_index_params.indexing.stills.candidate_outlier_rejection = False
mad_index_params.indexing.refinement_protocol.mode = None #$"repredict_only"
mad_index_params.indexing.two_color.high_energy = parameters.ENERGY_HIGH
mad_index_params.indexing.two_color.low_energy = parameters.ENERGY_LOW
mad_index_params.indexing.two_color.avg_energy = parameters.ENERGY_LOW * .5 + parameters.ENERGY_HIGH * .5
#mad_index_params.indexing.two_color.spiral_method = (1., 100000) # 1000000)
mad_index_params.indexing.two_color.n_unique_v = 22
#mad_index_params.indexing.two_color.block_size = 25
#mad_index_params.indexing.two_color.filter_by_mag = (10,3)
# ------
N = len(fnames)
idx_split = np.array_split(np.arange(N), n_jobs)
n_idx = 0 # number indexed
for idx in idx_split[jid]:
if args.Nmax is not None:
if idx == args.Nmax:
sys.exit()
img_f = fnames[idx]
loader = dxtbx.load(img_f)
iset = loader.get_imageset(filenames=[loader.get_image_file()] )
DET = loader.get_detector()
BEAM = loader.get_beam()
El = ExperimentListFactory.from_imageset_and_crystal(iset, crystal=None)
E = El[0]
from cxid9114.geom.multi_panel import CSPAD_refined, CSPAD
El[0].detector = CSPAD_refined
if idx in weak_shots and skip_weak:
print("Skipping weak shots %d" % idx)
continue
if idx in failed_shots and skip_failed:
print("Skipping failed idx shots %d" % idx)
continue
if idx in indexed_shots and skip_indexed:
print("Skipping already idx shots %d" % idx)
continue
refls_strong = flex.reflection_table.from_observations(El, spot_par)
if len(refls_strong) < min_spot_per_pattern:
print("Not enough spots shot %d, continuing!" % idx)
weak_shots.append(idx)
try:
np.savetxt(weak_shots_f, weak_shots, fmt="%d")
except:
pass
continue
# ==================================
# 2 color indexer of 2 color pattern
# ==================================
try:
orientAB = indexer_two_color(
reflections=refls_strong,
experiments=El,
params=mad_index_params)
orientAB.index()
except (Sorry, RuntimeError, AssertionError) as error:
print("####\nIndexingFailed T_T \n####")
print (error)
failed_shots.append(idx)
try:
np.savetxt(failed_idx_f, failed_shots, fmt="%d")
except:
pass
continue
print("####\n *_* IndexingWORKED %d *_* \n####" % idx)
n_idx += 1
indexed_shots.append(idx)
try:
np.savetxt(indexed_f, indexed_shots, fmt="%d")
except:
pass
crystalAB = orientAB.refined_experiments.crystals()[0]
refl_pkl = os.path.join(out_dir,refls_dir, "refl_%d_%s.pkl" % (idx, tag))
utils.save_flex(refls_strong, refl_pkl)
El[0].crystal = crystalAB
El_json = os.path.join(out_dir, Els_dir, "El_%d_%s.json" % (idx, tag))
El.as_json(filename=El_json)
def __init__(self,
iparams,
write_pickle=True,
write_logs=True,
last_stage='integrate'):
''' Constructor
:param iparams: IOTA params
:param write_pickle: Set to True to write out an integration pickle
'''
self.iparams = iparams
self.write_pickle = write_pickle
self.write_logs = write_logs
self.last_stage = last_stage
self.dlog_bookmark = 0
# Get Processor PHIL and initialize Processor
if self.iparams.cctbx_xfel.target:
with open(self.iparams.cctbx_xfel.target, 'r') as tf:
tphil_string = tf.read()
tparams = phil_scope.fetch(source=parse(tphil_string)).extract()
else:
tparams = phil_scope.extract()
Processor.__init__(self, params=tparams)
# IOTA-specific settings from here
# Turn off all peripheral output
self.params.output.experiments_filename = None
self.params.output.indexed_filename = None
self.params.output.strong_filename = None
self.params.output.refined_experiments_filename = None
self.params.output.integrated_experiments_filename = None
self.params.output.integrated_filename = None
self.params.output.profile_filename = None
# Set customized parameters
beamX = self.iparams.image_import.beam_center.x
beamY = self.iparams.image_import.beam_center.y
if beamX != 0 or beamY != 0:
self.params.geometry.detector.slow_fast_beam_centre = '{} {}'.format(
beamY, beamX)
if self.iparams.image_import.distance != 0:
self.params.geometry.detector.distance = self.iparams.image_import.distance
if self.iparams.image_import.mask is not None:
self.params.spotfinder.lookup.mask = self.iparams.image_import.mask
self.params.integration.lookup.mask = self.iparams.image_import.mask
if self.iparams.cctbx_xfel.target_space_group is not None:
sg = self.iparams.cctbx_xfel.target_space_group
self.params.indexing.known_symmetry.space_group = sg
if self.iparams.cctbx_xfel.target_unit_cell is not None:
uc = self.iparams.cctbx_xfel.target_unit_cell
self.params.indexing.known_symmetry.unit_cell = uc
if not self.params.indexing.stills.method_list:
self.params.indexing.stills.method_list = [
'fft1d', 'real_space_grid_search'
]
if self.iparams.cctbx_xfel.use_fft3d:
self.params.indexing.stills.method_list.insert(2, 'fft3d')
if self.iparams.cctbx_xfel.significance_filter.flag_on:
sigma = self.iparams.cctbx_xfel.significance_filter.sigma
sigma = sigma if sigma else 1
self.params.significance_filter.enable = True
self.params.significance_filter.isigi_cutoff = sigma
# Load reference geometry
self.reference_detector = None
if self.iparams.advanced.reference_geometry:
from dxtbx.model.experiment_list import ExperimentListFactory
try:
ref_experiments = ExperimentListFactory.from_json_file(
str(self.iparams.advanced.reference_geometry),
check_format=False)
except Exception as e:
print('DEBUG: Could not make exp. list because: ', e)
try:
import dxtbx
img = dxtbx.load(
str(self.iparams.advanced.reference_geometry))
except Exception:
print("DEBUG: Couldn't load geometry file {}"
"".format(self.iparams.advanced.reference_geometry))
else:
self.reference_detector = img.get_detector()
else:
assert len(ref_experiments.detectors()) == 1
self.reference_detector = ref_experiments.detectors()[0]
def test_stills_indexer_multi_lattice_bug_MosaicSauter2014(
dials_regression, tmpdir):
"""Problem: In stills_indexer, before calling the refine function, the
experiment list contains a list of dxtbx crystal models (that are not
MosaicSauter2014 models). The conversion to MosaicSauter2014 is made
during the refine step when functions from nave_parameters is called.
If the experiment list contains more than 1 experiment, for eg.
multiple lattices, only the first crystal gets assigned mosaicity. In
actuality, all crystal models should be assigned mosaicity. This test
only compares whether or not all crystal models have been assigned a
MosaicSauter2014 model."""
import dxtbx.model
from dxtbx.model import Crystal
from dxtbx.model.experiment_list import (
Experiment,
ExperimentList,
ExperimentListFactory,
)
from dials.algorithms.indexing.stills_indexer import StillsIndexer
from dials.array_family import flex
from dials.command_line.stills_process import (
phil_scope as stills_process_phil_scope, )
experiment_data = os.path.join(
dials_regression,
"refinement_test_data",
"cspad_refinement",
"cspad_refined_experiments_step6_level2_300.json",
)
reflection_data = os.path.join(
dials_regression,
"refinement_test_data",
"cspad_refinement",
"cspad_reflections_step7_300.pickle",
)
refl = flex.reflection_table.from_file(reflection_data)
explist = ExperimentListFactory.from_json_file(experiment_data,
check_format=False)[0:2]
reflist = refl.select(
refl["id"] < 2) # Only use the first 2 for convenience
# Construct crystal models that don't have mosaicity. These A,B,C values are the same
# as read in from the dials_regression folder
# Crystal-0
cs0 = Crystal(explist[0].crystal)
exp0 = Experiment(
imageset=explist[0].imageset,
beam=explist[0].beam,
detector=explist[0].detector,
goniometer=None,
scan=None,
crystal=cs0,
)
# Crystal-1
cs1 = Crystal(explist[1].crystal)
exp1 = Experiment(
imageset=explist[1].imageset,
beam=explist[1].beam,
detector=explist[1].detector,
goniometer=None,
scan=None,
crystal=cs1,
)
# Construct a new experiment_list that will be passed on for refinement
unrefined_explist = ExperimentList([exp0, exp1])
# Get default params from stills_process and construct StillsIndexer, then run refinement
params = stills_process_phil_scope.extract()
SI = StillsIndexer(reflist, unrefined_explist, params=params)
refined_explist, new_reflist = SI.refine(unrefined_explist, reflist)
# Now check whether the models have mosaicity after stills_indexer refinement
# Also check that mosaicity values are within expected limits
for ii, crys in enumerate(refined_explist.crystals()):
assert isinstance(crys, dxtbx.model.MosaicCrystalSauter2014)
if ii == 0:
assert crys.get_domain_size_ang() == pytest.approx(2242.0, rel=0.1)
if ii == 1:
assert crys.get_domain_size_ang() == pytest.approx(2689.0, rel=0.1)
def test_stills_indexer_multi_lattice_bug_MosaicSauter2014(
dials_regression, tmpdir):
""" Problem: In stills_indexer, before calling the refine function, the experiment list contains a list of
dxtbx crystal models (that are not MosaicSauter2014 models). The conversion to MosaicSauter2014 is made
during the refine step when functions from nave_parameters is called. If the experiment list contains
more than 1 experiment, for eg. multiple lattices, only the first crystal gets assigned mosaicity. In
actuality, all crystal models should be assigned mosaicity. This test only compares whether or not all crystal models
have been assigned a MosaicSauter2014 model. """
import libtbx
from dxtbx.model.experiment_list import ExperimentListFactory
from dxtbx.model.experiment_list import Experiment, ExperimentList
from dials.array_family import flex
from dxtbx.model import Crystal
from scitbx.matrix import col
from dials.algorithms.indexing.stills_indexer import StillsIndexer
from dials.command_line.stills_process import (
phil_scope as stills_process_phil_scope, )
import dxtbx_model_ext # needed for comparison of types
xfel_regression = libtbx.env.find_in_repositories(
relative_path="xfel_regression", test=os.path.isdir)
if not xfel_regression:
pytest.skip("test requires xfel_regression")
experiment_data = os.path.join(xfel_regression, "cspad_cbf_metrology",
"input_combined_experiments_300.json")
reflection_data = os.path.join(xfel_regression, "cspad_cbf_metrology",
"input_combined_reflections_300.pickle")
refl = flex.reflection_table.from_file(reflection_data)
explist = ExperimentListFactory.from_json_file(experiment_data,
check_format=False)[0:2]
reflist = refl.select(
refl["id"] < 2) # Only use the first 2 for convenience
# Construct crystal models that don't have mosaicity. These A,B,C values are the same
# as read in from the xfel_regression folder
# Crystal-0
sg = "P6122"
c = col((-6.273802315592485, 130.0982158871206, -5.79093135302508))
b = col((12.889379382910931, -3.488465493571038, -92.33550227410447))
a = col((73.48274403946694, 6.087949131831335, 57.16094537694445))
cs0 = Crystal(a, b, c, sg)
exp0 = Experiment(
imageset=explist[0].imageset,
beam=explist[0].beam,
detector=explist[0].detector,
goniometer=None,
scan=None,
crystal=cs0,
)
# Crystal-1
a1 = col((-9.848734136977392, -91.47863873838793, 15.37304224517648))
b1 = col((-61.88040624461757, 60.16978337475523, 35.38476831968059))
c1 = col((-72.13672676543942, -10.447921098609202, -108.38564134527415))
cs1 = Crystal(a1, b1, c1, sg)
exp1 = Experiment(
imageset=explist[1].imageset,
beam=explist[1].beam,
detector=explist[1].detector,
goniometer=None,
scan=None,
crystal=cs1,
)
# Construct a new experiment_list that will be passed on for refinement
unrefined_explist = ExperimentList([exp0, exp1])
# Get default params from stills_process and construct StillsIndexer, then run refinement
params = stills_process_phil_scope.extract()
SI = StillsIndexer(reflist, unrefined_explist, params=params)
refined_explist, new_reflist = SI.refine(unrefined_explist, reflist)
# Now check whether the models have mosaicity after stills_indexer refinement
for crys in refined_explist.crystals():
assert isinstance(crys, dxtbx_model_ext.MosaicCrystalSauter2014)
from cctbx import crystal
import dxtbx
from cxid9114.spots import spot_utils
from cxid9114 import parameters
from libtbx.utils import Sorry
#import stackimpact
#agent = stackimpact.start(
# agent_key = 'agent key here',
# app_name = 'MyPythonApp')
#agent.start_allocation_profiler()
if HAS_TWO_COLOR:
indexer_phil_scope.adopt_scope(two_color_phil_scope)
params = indexer_phil_scope.extract()
MIN_SPOT_PER_HIT = 5
# =====================================
# Parameters
show_hits = False
INDEXER = two_color_indexer.indexer_two_color
KNOWN_SYMMETRY = crystal.symmetry("78.95,78.95,38.13,90,90,90", "P43212")
params.refinement.parameterisation.beam.fix = "all"
params.refinement.parameterisation.detector.fix = "all"
params.indexing.known_symmetry.space_group = KNOWN_SYMMETRY.space_group_info(
)
params.refinement.verbosity = 3
params.indexing.refinement_protocol.d_min_start = None
params.indexing.known_symmetry.unit_cell = KNOWN_SYMMETRY.unit_cell()
