def __init__(self,
img,
index=None,
termfile=None,
paramfile=None,
output_file=None,
output_dir=None,
backend='dials',
action_code='spotfind',
min_bragg=10,
n_processors=1,
verbose=False
):
self.img = img
self.backend = backend
self.paramfile = paramfile
self.termfile = termfile
self.n_processors = n_processors
self.index = index
self.verbose = verbose
self.min_bragg = min_bragg
if output_file is not None:
if output_dir is not None:
self.output = os.path.join(os.path.abspath(output_dir), output_file)
else:
self.output = os.path.abspath(output_file)
else:
self.output = None
Thread.__init__(self)
# Determine which processes will be included
if action_code == 'spotfind':
self.run_indexing = False
self.run_integration = False
elif action_code == 'index':
self.run_indexing = True
self.run_integration = False
elif action_code == 'integrate':
self.run_indexing = True
self.run_integration = True
# Initialize IOTA DIALS Processor
if self.backend.lower() == 'dials':
if self.paramfile is not None:
with open(self.paramfile, 'r') as phil_file:
phil_string = phil_file.read()
user_phil = ip.parse(phil_string)
self.dials_phil = phil_scope.fetch(source=user_phil)
else:
self.dials_phil = phil_scope
self.params = self.dials_phil.extract()
if self.backend == 'dials':
self.processor = IOTADialsProcessor(params=self.params)
def __init__(self,
parent,
data_list=None,
term_file=None,
proc_params=None,
backend='dials'):
Thread.__init__(self)
self.parent = parent
self.data_list = data_list
self.term_file = term_file
self.terminated = False
self.backend = backend
if self.backend == 'dials':
from iota.components.iota_dials import IOTADialsProcessor
self.processor = IOTADialsProcessor(params=proc_params)
def __init__(self,
parent,
data_list=None,
term_file=None,
proc_params=None,
backend='dials',
n_proc=0,
run_indexing=False,
run_integration=False):
Thread.__init__(self)
self.parent = parent
self.data_list = data_list
self.term_file = term_file
self.terminated = False
self.backend = backend
self.run_indexing = run_indexing
self.run_integration = run_integration
if n_proc > 0:
self.n_proc = n_proc
else:
self.n_proc = multiprocessing.cpu_count() - 2
if self.backend == 'dials':
# Modify default DIALS parameters
# These parameters will be set no matter what
proc_params.output.datablock_filename = None
proc_params.output.indexed_filename = None
proc_params.output.strong_filename = None
proc_params.output.refined_experiments_filename = None
proc_params.output.integrated_filename = None
proc_params.output.integrated_experiments_filename = None
proc_params.output.profile_filename = None
proc_params.output.integration_pickle = None
#TODO: These *should* be overridden by a script...
proc_params.indexing.stills.method_list = ['fft3d']
from iota.components.iota_dials import IOTADialsProcessor
self.processor = IOTADialsProcessor(params=proc_params)
class DIALSSpfIdx(Thread):
def __init__(self,
img,
index=None,
termfile=None,
paramfile=None,
output_file=None,
output_dir=None,
backend='dials',
action_code='spotfind',
min_bragg=10,
n_processors=1,
verbose=False
):
self.img = img
self.backend = backend
self.paramfile = paramfile
self.termfile = termfile
self.n_processors = n_processors
self.index = index
self.verbose = verbose
self.min_bragg = min_bragg
if output_file is not None:
if output_dir is not None:
self.output = os.path.join(os.path.abspath(output_dir), output_file)
else:
self.output = os.path.abspath(output_file)
else:
self.output = None
Thread.__init__(self)
# Determine which processes will be included
if action_code == 'spotfind':
self.run_indexing = False
self.run_integration = False
elif action_code == 'index':
self.run_indexing = True
self.run_integration = False
elif action_code == 'integrate':
self.run_indexing = True
self.run_integration = True
# Initialize IOTA DIALS Processor
if self.backend.lower() == 'dials':
if self.paramfile is not None:
with open(self.paramfile, 'r') as phil_file:
phil_string = phil_file.read()
user_phil = ip.parse(phil_string)
self.dials_phil = phil_scope.fetch(source=user_phil)
else:
self.dials_phil = phil_scope
self.params = self.dials_phil.extract()
if self.backend == 'dials':
self.processor = IOTADialsProcessor(params=self.params)
def process_image(self):
if os.path.isfile(self.termfile):
raise IOTATermination('IOTA_TRACKER: Termination signal received!')
else:
with Capturing() as junk_output:
err = []
start = time.time()
fail = False
sg = None
uc = None
obs = None
status = None
res = 99
try:
datablock = DataBlockFactory.from_filenames([self.img])[0]
observed = self.processor.find_spots(datablock=datablock)
status = 'spots found'
except Exception, e:
fail = True
observed = []
err.append(e)
pass
# TODO: Indexing / lattice determination very slow (how to speed up?)
if self.run_indexing:
if not fail:
try:
experiments, indexed = self.processor.index(
datablock=datablock, reflections=observed)
except Exception, e:
fail = True
err.append(e)
pass
if not fail:
try:
solution = self.processor.refine_bravais_settings(
reflections=indexed, experiments=experiments)
# Only reindex if higher-symmetry solution found
if solution is not None:
experiments, indexed = self.processor.reindex(
reflections=indexed,
experiments=experiments,
solution=solution)
obs = experiments
lat = experiments[0].crystal.get_space_group().info()
sg = str(lat).replace(' ', '')
status = 'indexed'
except Exception:
fail = True
err.append(e)
pass
if not fail:
unit_cell = experiments[0].crystal.get_unit_cell().parameters()
uc = ' '.join(['{:.1f}'.format(i) for i in unit_cell])
if self.run_integration:
if not fail:
try:
# Run refinement
experiments, indexed = self.processor.refine(
experiments=experiments,
centroids=indexed)
integrated = self.processor.integrate(experiments=experiments,
indexed=indexed)
status = 'integrated'
except Exception:
fail = True
err.append(e)
pass
if status == 'integrated':
res = self.processor.frame['observations'][0].d_max_min()
else:
detector = datablock.unique_detectors()[0]
beam = datablock.unique_beams()[0]
s1 = flex.vec3_double()
for i in xrange(len(observed)):
s1.append(detector[observed['panel'][i]].get_pixel_lab_coord(
observed['xyzobs.px.value'][i][0:2]))
two_theta = s1.angle(beam.get_s0())
d = beam.get_wavelength() / (2 * flex.asin(two_theta / 2))
res = (np.max(d), np.min(d))
if len(observed) < self.min_bragg:
res = (99, 99)
elapsed = time.time() - start
info = [self.index, len(observed), self.img, sg, uc]
return status, info, res, elapsed, err
class DIALSSpfIdx(Thread):
def __init__(self,
img,
index=None,
termfile=None,
paramfile=None,
output=None,
backend='dials',
action_code='spotfind',
n_processors=1,
verbose=False):
self.img = img
self.backend = backend
self.paramfile = paramfile
self.termfile = termfile
self.n_processors = n_processors
self.index = index
self.verbose = verbose
print 'DEBUG: VERBOSE = ', self.verbose
if output is not None:
self.output = os.path.abspath(output)
else:
self.output = None
Thread.__init__(self)
# Determine which processes will be included
if action_code == 'spotfind':
self.run_indexing = False
self.run_integration = False
elif action_code == 'index':
self.run_indexing = True
self.run_integration = False
elif action_code == 'integrate':
self.run_indexing = True
self.run_integration = True
# Initialize IOTA DIALS Processor
if self.backend.lower() == 'dials':
if self.paramfile is not None:
with open(self.paramfile, 'r') as phil_file:
phil_string = phil_file.read()
user_phil = ip.parse(phil_string)
self.dials_phil = phil_scope.fetch(source=user_phil)
else:
self.dials_phil = phil_scope
self.params = self.dials_phil.extract()
if self.backend == 'dials':
self.processor = IOTADialsProcessor(params=self.params)
def process_image(self):
if os.path.isfile(self.termfile):
raise IOTATermination('IOTA_TRACKER: Termination signal received!')
else:
with Capturing() as junk_output:
start = time.time()
fail = False
sg = None
uc = None
try:
datablock = DataBlockFactory.from_filenames([self.img])[0]
observed = self.processor.find_spots(datablock=datablock)
except Exception:
fail = True
pass
# TODO: Indexing / lattice determination very slow (how to speed up?)
if self.run_indexing:
if not fail:
try:
experiments, indexed = self.processor.index(
datablock=datablock, reflections=observed)
except Exception:
fail = True
pass
if not fail:
try:
solution = self.processor.refine_bravais_settings(
reflections=indexed, experiments=experiments)
# Only reindex if higher-symmetry solution found
if solution is not None:
experiments, indexed = self.processor.reindex(
reflections=indexed,
experiments=experiments,
solution=solution)
lat = experiments[0].crystal.get_space_group(
).info()
sg = str(lat).replace(' ', '')
except Exception:
fail = True
pass
if self.run_integration:
if not fail:
try:
# Run refinement
experiments, indexed = self.processor.refine(
experiments=experiments, centroids=indexed)
integrated = self.processor.integrate(
experiments=experiments, indexed=indexed)
frame = self.processor.frame
unit_cell = frame['observations'][0].unit_cell(
).parameters()
uc = ' '.join(
['{:.1f}'.format(i) for i in unit_cell])
except Exception:
fail = True
pass
elapsed = time.time() - start
return [self.index, len(observed), self.img, sg, uc], elapsed
def run(self):
info, elapsed = self.process_image()
if info is not None:
idx, no_spots, img_path, sg, uc = info
if self.verbose:
print 'RESULT: ', idx, img_path, no_spots, sg, uc, '---> ', elapsed
if self.output is not None:
with open(self.output, 'a') as outf:
info_line = ' '.join([str(i) for i in info])
outf.write('{}\n'.format(info_line))
else:
print 'RESULT: NONE'
def __init__(self,
img,
index=None,
termfile=None,
paramfile=None,
output_file=None,
output_dir=None,
backend='dials',
action_code='spotfind',
min_bragg=10,
n_processors=1,
verbose=False):
self.img = img
self.backend = backend
self.paramfile = paramfile
self.termfile = termfile
self.n_processors = n_processors
self.index = index
self.verbose = verbose
self.min_bragg = min_bragg
if output_file is not None:
if output_dir is not None:
self.output = os.path.join(os.path.abspath(output_dir),
output_file)
else:
self.output = os.path.abspath(output_file)
else:
self.output = None
Thread.__init__(self)
# Determine which processes will be included
if action_code == 'spotfind':
self.run_indexing = False
self.run_integration = False
elif action_code == 'index':
self.run_indexing = True
self.run_integration = False
elif action_code == 'integrate':
self.run_indexing = True
self.run_integration = True
# Initialize IOTA DIALS Processor
if self.backend.lower() == 'dials':
if self.paramfile is not None:
with open(self.paramfile, 'r') as phil_file:
phil_string = phil_file.read()
user_phil = ip.parse(phil_string)
self.dials_phil = phil_scope.fetch(source=user_phil)
else:
default_params, _ = write_defaults(method='dials',
write_target_file=False,
write_param_file=False)
default_phil_string = '\n'.join(default_params)
default_phil = ip.parse(default_phil_string)
self.dials_phil = phil_scope.fetch(source=default_phil)
self.params = self.dials_phil.extract()
# Modify default DIALS parameters
# These parameters will be set no matter what
self.params.output.datablock_filename = None
self.params.output.indexed_filename = None
self.params.output.strong_filename = None
self.params.output.refined_experiments_filename = None
self.params.output.integrated_filename = None
self.params.output.integrated_experiments_filename = None
self.params.output.profile_filename = None
self.params.output.integration_pickle = None
# These parameters will be set only if there's no script
if self.paramfile is None:
self.params.indexing.stills.method_list = ['fft3d']
self.params.spotfinder.threshold.dispersion.global_threshold = 75
if self.backend == 'dials':
self.processor = IOTADialsProcessor(params=self.params,
write_pickle=False)
class DIALSSpfIdx(Thread):
def __init__(self,
img,
index=None,
termfile=None,
paramfile=None,
output_file=None,
output_dir=None,
backend='dials',
action_code='spotfind',
min_bragg=10,
n_processors=1,
verbose=False):
self.img = img
self.backend = backend
self.paramfile = paramfile
self.termfile = termfile
self.n_processors = n_processors
self.index = index
self.verbose = verbose
self.min_bragg = min_bragg
if output_file is not None:
if output_dir is not None:
self.output = os.path.join(os.path.abspath(output_dir),
output_file)
else:
self.output = os.path.abspath(output_file)
else:
self.output = None
Thread.__init__(self)
# Determine which processes will be included
if action_code == 'spotfind':
self.run_indexing = False
self.run_integration = False
elif action_code == 'index':
self.run_indexing = True
self.run_integration = False
elif action_code == 'integrate':
self.run_indexing = True
self.run_integration = True
# Initialize IOTA DIALS Processor
if self.backend.lower() == 'dials':
if self.paramfile is not None:
with open(self.paramfile, 'r') as phil_file:
phil_string = phil_file.read()
user_phil = ip.parse(phil_string)
self.dials_phil = phil_scope.fetch(source=user_phil)
else:
default_params, _ = write_defaults(method='dials',
write_target_file=False,
write_param_file=False)
default_phil_string = '\n'.join(default_params)
default_phil = ip.parse(default_phil_string)
self.dials_phil = phil_scope.fetch(source=default_phil)
self.params = self.dials_phil.extract()
# Modify default DIALS parameters
# These parameters will be set no matter what
self.params.output.datablock_filename = None
self.params.output.indexed_filename = None
self.params.output.strong_filename = None
self.params.output.refined_experiments_filename = None
self.params.output.integrated_filename = None
self.params.output.integrated_experiments_filename = None
self.params.output.profile_filename = None
self.params.output.integration_pickle = None
# These parameters will be set only if there's no script
if self.paramfile is None:
self.params.indexing.stills.method_list = ['fft3d']
self.params.spotfinder.threshold.dispersion.global_threshold = 75
if self.backend == 'dials':
self.processor = IOTADialsProcessor(params=self.params,
write_pickle=False)
def process_image(self):
if os.path.isfile(self.termfile):
raise IOTATermination('IOTA_TRACKER: Termination signal received!')
else:
with Capturing() as junk_output:
# if True:
err = []
start = time.time()
fail = False
sg = None
uc = None
status = None
score = 0
try:
datablock = DataBlockFactory.from_filenames([self.img])[0]
observed = self.processor.find_spots(datablock=datablock)
status = 'spots found'
except Exception, e:
fail = True
observed = []
err.append('SPOTFINDING ERROR: {}'.format(e))
pass
# TODO: Indexing / lattice determination very slow (how to speed up?)
if self.run_indexing:
if not fail:
try:
experiments, indexed = self.processor.index(
datablock=datablock, reflections=observed)
score = len(indexed)
except Exception, e:
fail = True
err.append('INDEXING ERROR: {}'.format(e))
pass
if not fail:
try:
solution = self.processor.refine_bravais_settings(
reflections=indexed, experiments=experiments)
# Only reindex if higher-symmetry solution found
if solution is not None:
experiments, indexed = self.processor.reindex(
reflections=indexed,
experiments=experiments,
solution=solution)
obs = experiments
lat = experiments[0].crystal.get_space_group(
).info()
sg = str(lat).replace(' ', '')
status = 'indexed'
except Exception, e:
fail = True
err.append('LATTICE ERROR: {}'.format(e))
pass
if not fail:
unit_cell = experiments[0].crystal.get_unit_cell(
).parameters()
uc = ' '.join(['{:.4f}'.format(i) for i in unit_cell])
if self.run_integration:
if not fail:
try:
# Run refinement
experiments, indexed = self.processor.refine(
experiments=experiments, centroids=indexed)
except Exception, e:
fail = True
err.append('REFINEMENT ERROR: {}'.format(e))
pass
if not fail:
try:
print experiments
print indexed
integrated = self.processor.integrate(
experiments=experiments, indexed=indexed)
frame = self.processor.frame
status = 'integrated'
except Exception, e:
err.append('INTEGRATION ERROR: {}'.format(e))
pass
class DIALSSpfIdx(Thread):
def __init__(self,
img,
index=None,
termfile=None,
paramfile=None,
output_file=None,
output_dir=None,
backend='dials',
action_code='spotfind',
min_bragg=10,
n_processors=1,
verbose=False):
self.img = img
self.backend = backend
self.paramfile = paramfile
self.termfile = termfile
self.n_processors = n_processors
self.index = index
self.verbose = verbose
self.min_bragg = min_bragg
if output_file is not None:
if output_dir is not None:
self.output = os.path.join(os.path.abspath(output_dir),
output_file)
else:
self.output = os.path.abspath(output_file)
else:
self.output = None
Thread.__init__(self)
# Determine which processes will be included
if action_code == 'spotfind':
self.run_indexing = False
self.run_integration = False
elif action_code == 'index':
self.run_indexing = True
self.run_integration = False
elif action_code == 'integrate':
self.run_indexing = True
self.run_integration = True
# Initialize IOTA DIALS Processor
if self.backend.lower() == 'dials':
if self.paramfile is not None:
with open(self.paramfile, 'r') as phil_file:
phil_string = phil_file.read()
user_phil = ip.parse(phil_string)
self.dials_phil = phil_scope.fetch(source=user_phil)
else:
default_params, _ = write_defaults(method='dials',
write_target_file=False,
write_param_file=False)
default_phil_string = '\n'.join(default_params)
default_phil = ip.parse(default_phil_string)
self.dials_phil = phil_scope.fetch(source=default_phil)
self.params = self.dials_phil.extract()
# Modify default DIALS parameters
# These parameters will be set no matter what
self.params.output.datablock_filename = None
self.params.output.indexed_filename = None
self.params.output.strong_filename = None
self.params.output.refined_experiments_filename = None
self.params.output.integrated_filename = None
self.params.output.integrated_experiments_filename = None
self.params.output.profile_filename = None
self.params.output.integration_pickle = None
# These parameters will be set only if there's no script
if self.paramfile is None:
self.params.indexing.stills.method_list = ['fft3d']
self.params.spotfinder.threshold.dispersion.global_threshold = 75
if self.backend == 'dials':
self.processor = IOTADialsProcessor(params=self.params,
write_pickle=False)
def process_image(self):
if os.path.isfile(self.termfile):
raise IOTATermination('IOTA_TRACKER: Termination signal received!')
else:
with Capturing() as junk_output:
# if True:
err = []
start = time.time()
fail = False
sg = None
uc = None
status = None
score = 0
try:
datablock = DataBlockFactory.from_filenames([self.img])[0]
observed = self.processor.find_spots(datablock=datablock)
status = 'spots found'
except Exception as e:
fail = True
observed = []
err.append('SPOTFINDING ERROR: {}'.format(e))
pass
# TODO: Indexing / lattice determination very slow (how to speed up?)
if self.run_indexing:
if not fail:
try:
experiments, indexed = self.processor.index(
datablock=datablock, reflections=observed)
score = len(indexed)
except Exception as e:
fail = True
err.append('INDEXING ERROR: {}'.format(e))
pass
if not fail:
try:
solution = self.processor.refine_bravais_settings(
reflections=indexed, experiments=experiments)
# Only reindex if higher-symmetry solution found
if solution is not None:
experiments, indexed = self.processor.reindex(
reflections=indexed,
experiments=experiments,
solution=solution)
obs = experiments
lat = experiments[0].crystal.get_space_group(
).info()
sg = str(lat).replace(' ', '')
status = 'indexed'
except Exception as e:
fail = True
err.append('LATTICE ERROR: {}'.format(e))
pass
if not fail:
unit_cell = experiments[0].crystal.get_unit_cell(
).parameters()
uc = ' '.join(['{:.4f}'.format(i) for i in unit_cell])
if self.run_integration:
if not fail:
try:
# Run refinement
experiments, indexed = self.processor.refine(
experiments=experiments, centroids=indexed)
except Exception as e:
fail = True
err.append('REFINEMENT ERROR: {}'.format(e))
pass
if not fail:
try:
print experiments
print indexed
integrated = self.processor.integrate(
experiments=experiments, indexed=indexed)
frame = ConstructFrame(
integrated, experiments[0]).make_frame()
status = 'integrated'
except Exception as e:
err.append('INTEGRATION ERROR: {}'.format(e))
pass
if status == 'integrated':
res = frame['observations'][0].d_max_min()
else:
detector = datablock.unique_detectors()[0]
beam = datablock.unique_beams()[0]
s1 = flex.vec3_double()
for i in xrange(len(observed)):
s1.append(
detector[observed['panel'][i]].get_pixel_lab_coord(
observed['xyzobs.px.value'][i][0:2]))
two_theta = s1.angle(beam.get_s0())
d = beam.get_wavelength() / (2 * flex.asin(two_theta / 2))
res = (np.max(d), np.min(d))
if len(observed) < self.min_bragg:
res = (99, 99)
elapsed = time.time() - start
info = [self.index, len(observed), self.img, sg, uc]
return status, info, res, score, elapsed, err
def run(self):
errors = []
n_spots = 0
n_overloads = 0
res = (99, 99)
n_rings = 0
avg_I = 0
score = 0
err = []
file_wait_start = time.time()
while True:
if time.time() - file_wait_start > 30:
info = None
elapsed = None
errors.append('{} does not exist'.format(self.img))
break
if os.path.isfile(self.img):
status, info, res, score, elapsed, err = self.process_image()
# errors.extend(err)
break
if info is not None:
idx, n_spots, img_path, sg, uc = info
if self.verbose:
print 'IMAGE #{}: {}'.format(idx, img_path)
print 'SPOTS FOUND: {}'.format(n_spots)
print 'INDEXING: {} INDEXED SPOTS'.format(score)
if res[0] != 99:
print 'RESOLUTION: {:.2f} - {:.2f}'.format(res[0], res[1])
if sg is not None and uc is not None:
print 'BRAVAIS LATTICE: {}'.format(sg)
print 'UNIT CELL: {}'.format(uc)
print 'TOTAL PROCESSING TIME: {:.2f} SEC'.format(elapsed)
if err != []:
for e in err:
print e
if self.output is not None:
with open(self.output, 'a') as outf:
info_line = ' '.join([str(i) for i in info])
outf.write('{}\n'.format(info_line))
if self.verbose:
if errors == []:
err = ''
print_errors = False
else:
err = errors[0]
print_errors = True
print '\n__RESULTS__'
print '{} {} {} {:.2f} {} {} {} {} {{{}}}'.format(
n_spots, n_overloads, score, res[1], n_rings, 0, avg_I, 0, err)
if print_errors:
print "__ERRORS__"
for e in errors:
print e