def index(self, datablock, reflections):
from time import time
from logging import info
import copy
st = time()
info('*' * 80)
info('Indexing Strong Spots')
info('*' * 80)
imagesets = datablock.extract_imagesets()
params = copy.deepcopy(self.params)
# don't do scan-varying refinement during indexing
params.refinement.parameterisation.crystal.scan_varying = False
from dials.algorithms.indexing.indexer import indexer_base
idxr = indexer_base.from_parameters(
reflections, imagesets,
params=params)
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
if self.params.output.indexed_filename:
self.save_reflections(indexed, self.params.output.indexed_filename)
info('')
info('Time Taken = %f seconds' % (time() - st))
return experiments, indexed
def index(self, datablock, reflections):
from time import time
import copy
st = time()
logger.info('*' * 80)
logger.info('Indexing Strong Spots')
logger.info('*' * 80)
imagesets = datablock.extract_imagesets()
params = copy.deepcopy(self.params)
# don't do scan-varying refinement during indexing
params.refinement.parameterisation.scan_varying = False
from dials.algorithms.indexing.indexer import indexer_base
idxr = indexer_base.from_parameters(reflections,
imagesets,
params=params)
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
if self.params.output.indexed_filename:
self.save_reflections(indexed, self.params.output.indexed_filename)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return experiments, indexed
def index(self, datablock, reflections):
from dials.algorithms.indexing.indexer import indexer_base
from time import time
import copy
st = time()
logger.info('*' * 80)
logger.info('Indexing Strong Spots')
logger.info('*' * 80)
imagesets = datablock.extract_imagesets()
params = copy.deepcopy(self.params)
# don't do scan-varying refinement during indexing
params.refinement.parameterisation.scan_varying = False
if hasattr(self, 'known_crystal_models'):
known_crystal_models = self.known_crystal_models
else:
known_crystal_models = None
if params.indexing.stills.method_list is None:
idxr = indexer_base.from_parameters(
reflections, imagesets, known_crystal_models=known_crystal_models,
params=params)
idxr.index()
else:
indexing_error = None
for method in params.indexing.stills.method_list:
params.indexing.method = method
try:
idxr = indexer_base.from_parameters(
reflections, imagesets,
params=params)
idxr.index()
except Exception, e:
logger.info("Couldn't index using method %s"%method)
if indexing_error is None:
if e is None:
e = Exception("Couldn't index using method %s"%method)
indexing_error = e
else:
indexing_error = None
break
if indexing_error is not None:
raise indexing_error
def index(self, datablock, reflections):
from dials.algorithms.indexing.indexer import indexer_base
from time import time
import copy
st = time()
logger.info('*' * 80)
logger.info('Indexing Strong Spots')
logger.info('*' * 80)
imagesets = datablock.extract_imagesets()
params = copy.deepcopy(self.params)
# don't do scan-varying refinement during indexing
params.refinement.parameterisation.scan_varying = False
if params.indexing.stills.method_list is None:
idxr = indexer_base.from_parameters(
reflections, imagesets,
params=params)
else:
indexing_error = None
for method in params.indexing.stills.method_list:
params.indexing.method = method
try:
idxr = indexer_base.from_parameters(
reflections, imagesets,
params=params)
except Exception, e:
logger.info("Couldn't index using method %s"%method)
if indexing_error is None:
if e is None:
e = Exception("Couldn't index using method %s"%method)
indexing_error = e
else:
indexing_error = None
break
if indexing_error is not None:
raise indexing_error
def index(self):
from dials.algorithms.indexing.indexer import indexer_base
# Generate imagesets
imagesets = self.datablock.extract_imagesets()
# Necessary settings for stills processing
#self.phil.refinement.parameterisation.crystal.scan_varying = False
self.phil.indexing.scan_range=[]
# Check if unit cell / space group have been provided
if self.phil.indexing.known_symmetry.space_group == None or \
self.phil.indexing.known_symmetry.unit_cell == None:
self.phil.indexing.method = 'fft1d'
else:
self.phil.indexing.method == 'real_space_grid_search'
# Run indexing
idxr = indexer_base.from_parameters(self.observed, imagesets, params=self.phil)
self.indexed = idxr.refined_reflections
self.experiments = idxr.refined_experiments
def index(self):
from dials.algorithms.indexing.indexer import indexer_base
# Generate imagesets
imagesets = self.datablock.extract_imagesets()
# Necessary settings for stills processing
#self.phil.refinement.parameterisation.crystal.scan_varying = False
self.phil.indexing.scan_range = []
# Check if unit cell / space group have been provided
if self.phil.indexing.known_symmetry.space_group == None or \
self.phil.indexing.known_symmetry.unit_cell == None:
self.phil.indexing.method = 'fft1d'
else:
self.phil.indexing.method == 'real_space_grid_search'
# Run indexing
idxr = indexer_base.from_parameters(self.observed,
imagesets,
params=self.phil)
self.indexed = idxr.refined_reflections
self.experiments = idxr.refined_experiments
waveB = parameters.ENERGY_CONV / en[1]
BeamA = deepcopy(iset.get_beam())
BeamB = deepcopy(iset.get_beam())
BeamA.set_wavelength(waveA)
BeamB.set_wavelength(waveB)
isetA = deepcopy(iset)
isetB = deepcopy(iset)
isetA.set_beam(BeamA)
isetB.set_beam(BeamB)
# index two color pattern using fft1d
orientA = indexer_base.from_parameters(
reflections=spot_utils.as_single_shot_reflections(refls, inplace=False),
imagesets=[isetA],
params=sad_index_params)
orientA.index()
crystalA = orientA.refined_experiments.crystals()[0]
# try with other color
orientB = indexer_base.from_parameters(
reflections=spot_utils.as_single_shot_reflections(refls, inplace=False),
imagesets=[isetB],
params=sad_index_params)
orientB.index()
crystalB = orientB.refined_experiments.crystals()[0]
# ==================================
# 2 color indexer of 2 color pattern
# ==================================
n_indexed = 0
idx_indexed = []
print('Iterating over {:d} hits'.format(Nhits))
for i_hit in range(Nhits):
shot_idx = idx[where_hits[i_hit]]
if show_hits:
loader.show_data(shot_idx)
hit_imgset = imgset[shot_idx:shot_idx + 1]
hit_refl = refl_select.select(shot_idx)
print '\rIndexing shot {:d} (Hit {:d}/{:d}) using {:d} spots' \
.format(shot_idx, i_hit + 1, Nhits, len(hit_refl)),
sys.stdout.flush()
# orient = INDEXER(reflections=hit_refl, imagesets=[hit_imgset], params=params)
orient = indexer_base.from_parameters(reflections=hit_refl,
imagesets=[hit_imgset],
params=params)
try:
orient.index()
n_indexed += 1
idx_indexed.append(i_hit)
except Sorry, RunTimeError:
print("Could not index")
pass
print("Indexed %d / %d hits, out of %d total hits" %
(n_indexed, i_hit + 1, Nhits))
print("Indexed %d / %d hits" % (n_indexed, Nhits))
print idx_indexed
def run(args):
import libtbx.load_env
from libtbx.utils import Sorry
from dials.util import log
usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_reflections=True,
read_datablocks=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0:
if len(experiments) > 0:
imagesets = experiments.imagesets()
else:
parser.print_help()
return
#elif len(datablocks) > 1:
#raise Sorry("Only one DataBlock can be processed at a time")
else:
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
if len(experiments):
known_crystal_models = experiments.crystals()
else:
known_crystal_models = None
if len(reflections) == 0:
raise Sorry("No reflection lists found in input")
if len(reflections) > 1:
#raise Sorry("Multiple reflections lists provided in input")
assert len(reflections) == len(imagesets)
from scitbx.array_family import flex
for i in range(len(reflections)):
reflections[i]['imageset_id'] = flex.int(len(reflections[i]), i)
if i > 0:
reflections[0].extend(reflections[i])
#assert(len(reflections) == 1)
reflections = reflections[0]
for imageset in imagesets:
if (imageset.get_goniometer() is not None
and imageset.get_scan() is not None
and imageset.get_scan().get_oscillation()[1] == 0):
imageset.set_goniometer(None)
imageset.set_scan(None)
from dials.algorithms.indexing.indexer import indexer_base
idxr = indexer_base.from_parameters(
reflections,
imagesets,
known_crystal_models=known_crystal_models,
params=params)
refined_experiments = idxr.refined_experiments
reflections = copy.deepcopy(idxr.refined_reflections)
reflections.extend(idxr.unindexed_reflections)
if len(refined_experiments):
logger.info("Saving refined experiments to %s" %
params.output.experiments)
idxr.export_as_json(refined_experiments,
file_name=params.output.experiments)
logger.info("Saving refined reflections to %s" %
params.output.reflections)
idxr.export_reflections(reflections,
file_name=params.output.reflections)
if params.output.unindexed_reflections is not None:
logger.info("Saving unindexed reflections to %s" %
params.output.unindexed_reflections)
idxr.export_reflections(
idxr.unindexed_reflections,
file_name=params.output.unindexed_reflections)
return
# from dials.util.options import OptionParser
parser = OptionParser(phil=phil_scope)
params, options = parser.parse_args(args=[], show_diff_phil=True)
params.refinement.parameterisation.scan_varying = False
params.refinement.reflections.outlier.algorithm = 'sauter_poon'
params.indexing.method='real_space_grid_search'
filenames=glob.glob(image_glob)
datablock = DataBlockFactory.from_filenames(filenames)[0]
observed = flex.reflection_table.from_observations(datablock, params)
working_params = copy.deepcopy(params)
imagesets = datablock.extract_imagesets()
idxr = indexer_base.from_parameters(observed, imagesets, params=params)
idxr.index()
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
crystal = copy.deepcopy(experiments.crystals()[0])
print crystal.get_unit_cell()
observed.as_pickle("lightweight_index_strong.pickle")
idxr.export_as_json(experiments,file_name="lightweight_index_experiments.json")
idxr.export_reflections(indexed,file_name="lightweight_index_reflections.pickle")
def index(self, datablock, reflections):
from dials.algorithms.indexing.indexer import indexer_base
from time import time
import copy
st = time()
logger.info('*' * 80)
logger.info('Indexing Strong Spots')
logger.info('*' * 80)
imagesets = datablock.extract_imagesets()
params = copy.deepcopy(self.params)
# don't do scan-varying refinement during indexing
params.refinement.parameterisation.scan_varying = False
if hasattr(self, 'known_crystal_models'):
known_crystal_models = self.known_crystal_models
else:
known_crystal_models = None
if params.indexing.stills.method_list is None:
idxr = indexer_base.from_parameters(
reflections,
imagesets,
known_crystal_models=known_crystal_models,
params=params)
idxr.index()
else:
indexing_error = None
for method in params.indexing.stills.method_list:
params.indexing.method = method
try:
idxr = indexer_base.from_parameters(reflections,
imagesets,
params=params)
idxr.index()
except Exception as e:
logger.info("Couldn't index using method %s" % method)
if indexing_error is None:
if e is None:
e = Exception("Couldn't index using method %s" %
method)
indexing_error = e
else:
indexing_error = None
break
if indexing_error is not None:
raise indexing_error
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
if known_crystal_models is not None:
from dials.array_family import flex
filtered = flex.reflection_table()
for idx in set(indexed['miller_index']):
sel = indexed['miller_index'] == idx
if sel.count(True) == 1:
filtered.extend(indexed.select(sel))
logger.info(
"Filtered duplicate reflections, %d out of %d remaining" %
(len(filtered), len(indexed)))
print("Filtered duplicate reflections, %d out of %d remaining" %
(len(filtered), len(indexed)))
indexed = filtered
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return experiments, indexed
sad_index_params.refinement.parameterisation.crystal.fix = "all"
sad_index_params.refinement.parameterisation.detector.fix = None
sad_index_params.refinement.parameterisation.detector.panels = "hierarchical"
sad_index_params.refinement.parameterisation.detector.hierarchy_level = 1
sad_index_params.refinement.parameterisation.beam.fix = "all"
sad_index_params.indexing.stills.refine_candidates_with_known_symmetry = True
sad_index_params.indexing.stills.candidate_outlier_rejection = False
sad_index_params.indexing.stills.rmsd_min_px = 8
sad_index_params.indexing.debug = True
sad_index_params.indexing.fft1d.characteristic_grid = 0.029
#sad_index_params.indexing.refinement_protocol.mode = ""
# index two color pattern using fft1d
orient = indexer_base.from_parameters(
reflections=count_spots.as_single_shot_reflections(refls_strong,
inplace=False),
imagesets=[iset],
params=sad_index_params)
try:
orient.index()
except (Sorry, RuntimeError):
print("\n\n\t INDEXING FAILED\n")
failed_shots.append(idx)
np.savetxt(failed_idx_f, failed_shots, fmt="%d")
continue
indexed_shots.append(idx)
np.savetxt(indexed_f, indexed_shots, fmt="%d")
exp_name = os.path.join(outdir, "exp_%d.json" % Nprocessed)
refl_name = os.path.join(outdir, "refl_%d.pkl" % Nprocessed)
def run(args):
import libtbx.load_env
from libtbx.utils import Sorry
from dials.util import log
from logging import info
usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_reflections=True,
read_datablocks=True,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(
params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
info(dials_version())
# Log the diff phil
diff_phil = parser.diff_phil.as_str()
if diff_phil is not '':
info('The following parameters have been modified:\n')
info(diff_phil)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(datablocks) == 0:
if len(experiments) > 0:
imagesets = experiments.imagesets()
else:
parser.print_help()
return
#elif len(datablocks) > 1:
#raise Sorry("Only one DataBlock can be processed at a time")
else:
imagesets = []
for datablock in datablocks:
imagesets.extend(datablock.extract_imagesets())
if len(experiments):
known_crystal_models = experiments.crystals()
else:
known_crystal_models = None
if len(reflections) == 0:
raise Sorry("No reflection lists found in input")
if len(reflections) > 1:
#raise Sorry("Multiple reflections lists provided in input")
assert len(reflections) == len(imagesets)
from scitbx.array_family import flex
for i in range(len(reflections)):
reflections[i]['imageset_id'] = flex.int(len(reflections[i]), i)
if i > 0:
reflections[0].extend(reflections[i])
#assert(len(reflections) == 1)
reflections = reflections[0]
for imageset in imagesets:
if (imageset.get_goniometer() is not None and
imageset.get_scan() is not None and
imageset.get_scan().get_oscillation()[1] == 0):
imageset.set_goniometer(None)
imageset.set_scan(None)
from dials.algorithms.indexing.indexer import indexer_base
idxr = indexer_base.from_parameters(
reflections, imagesets,
known_crystal_models=known_crystal_models,
params=params)
refined_experiments = idxr.refined_experiments
reflections = copy.deepcopy(idxr.refined_reflections)
reflections.extend(idxr.unindexed_reflections)
if len(refined_experiments):
info("Saving refined experiments to %s" %params.output.experiments)
idxr.export_as_json(refined_experiments,
file_name=params.output.experiments)
info("Saving refined reflections to %s" %params.output.reflections)
idxr.export_reflections(
reflections, file_name=params.output.reflections)
if params.output.unindexed_reflections is not None:
info("Saving unindexed reflections to %s"
%params.output.unindexed_reflections)
idxr.export_reflections(idxr.unindexed_reflections,
file_name=params.output.unindexed_reflections)
return
filenames.append(path)
datablock = DataBlockFactory.from_filenames(filenames)[0]
observed = flex.reflection_table.from_observations(datablock, params)
# observed = flex.reflection_table.from_pickle("/lustre/ttscratch1/mewall/exafel/YSP_CSB08_13_PY-EVN/raw/strong.pickle")
observed.as_pickle("strong.pickle")
print "Number of observed reflections:", len(observed)
working_params = copy.deepcopy(params)
imagesets = datablock.extract_imagesets()
print "indexing..."
t0 = time()
# new dials, fix by Aaron
idxr = indexer_base.from_parameters(observed, imagesets, params=params)
idxr.index()
tel = time() - t0
print "done indexing (", tel, " sec)"
# new dials
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
# print experiments.crystals()[0]
# read input file with list of diffraction images and scale factors
f = open(ifname, "r")
lines = []
linenum = 1
framenum = -1
for line in f:
# current reflection table
curr_refls = copy.deepcopy(truth_refls)
curr_det = copy.deepcopy(det)
all_crysts = []
_stdout = sys.stdout
for _ in range(n_sim):
#for i in range(len(curr_refls)):
# x,y,z = curr_refls[i]['xyzobs.px.value']
# curr_refls['xyzobs.px.value'][i] = x-1, y-1,z
iset.set_detector( curr_det)
orient = indexer_base.from_parameters(
reflections=curr_refls,
imagesets=[iset], # there is only one
params=idxpar)
try:
if silence_indexer:
sys.stdout = io.BytesIO()
orient.index()
sys.stdout = _stdout
else:
orient.index()
except (Sorry, RuntimeError):
sys.stdout = _stdout
print("Exiting prematurely because indexing failed")
break
new_cryst = orient.refined_experiments.crystals()[0]
def index_from_files(f_one,f_two,f_three):
import dxtbx
from iotbx.phil import parse
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
from dials.algorithms.indexing.indexer import indexer_base
from dials.util.options import OptionParser
import copy
phil_scope_str='''
output {{
shoeboxes = True
.type = bool
.help = Save the raw pixel values inside the reflection shoeboxes.
}}
include scope dials.algorithms.spot_finding.factory.phil_scope
include scope dials.algorithms.indexing.indexer.index_only_phil_scope
include scope dials.algorithms.refinement.refiner.phil_scope
indexing.known_symmetry.unit_cell={0}
.type = unit_cell
indexing.known_symmetry.space_group={1}
.type = space_group
'''
phil_scope = parse(phil_scope_str.format(target_cell,target_sg), process_includes=True)
# from dials.util.options import OptionParser
parser = OptionParser(phil=phil_scope)
params, options = parser.parse_args(args=[], show_diff_phil=True)
params.refinement.parameterisation.scan_varying = False
params.indexing.method='real_space_grid_search'
# params.indexing.method='fft3d'
# params.indexing.max_cell=800
# params.spotfinder.filter.min_spot_size=3
filenames = [f_one,f_two,f_three]
datablock = DataBlockFactory.from_filenames(filenames)[0]
observed = flex.reflection_table.from_observations(datablock, params)
observed.as_pickle("strong.pickle")
print "Number of observed reflections:", len(observed)
working_params = copy.deepcopy(params)
imagesets = datablock.extract_imagesets()
print "indexing..."
t0 = time()
# new dials, fix by Aaron
idxr = indexer_base.from_parameters(observed, imagesets, params=params)
idxr.index()
tel = time()-t0
print "done indexing (",tel," sec)"
# new dials
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
print experiments.crystals()[0]
crystal_params = experiments.crystals()[0]
with open('crystal.pkl', 'wb') as output:
pickle.dump(crystal_params, output, pickle.HIGHEST_PROTOCOL)
return
# ==================================
# 2 color indexer of 2 color pattern
# ==================================
sad_index_params.indexing.multiple_lattice_search.max_lattices = 1
sad_index_params.indexing.stills.refine_all_candidates = False
sad_index_params.indexing.stills.refine_candidates_with_known_symmetry = False
sad_index_params.indexing.stills.candidate_outlier_rejection = False
sad_index_params.indexing.refinement_protocol.mode = "ignore"
sad_index_params2 = deepcopy(sad_index_params)
#sad_index_params2.indexing.stills.refine_all_candidates = False
#sad_index_params2.indexing.stills.refine_candidates_with_known_symmetry = False
#sad_index_params2.indexing.stills.candidate_outlier_rejection = False
#sad_index_params2.indexing.refinement_protocol.mode = "ignore"
orientAB_fft1d = indexer_base.from_parameters(
reflections=spot_utils.as_single_shot_reflections(reflAB, inplace=False),
imagesets=isetsAB,
params=sad_index_params2)
orientAB_fft1d.index()
crystal_AB_fft1d = orientAB_fft1d.refined_experiments.crystals()[0]
orientAB_2_fft1d = indexer_base.from_parameters(
reflections=spot_utils.as_single_shot_reflections(reflAB, inplace=False),
imagesets=isetsAB_2,
params=sad_index_params2)
orientAB_2_fft1d.index()
crystalAB_2_fft1d = orientAB_2_fft1d.refined_experiments.crystals()[0]
# index the 1 color patterns:
orientA = indexer_base.from_parameters(
