def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir):
# Check the data files for this test exist
image_path = os.path.join(
dials_regression,
"image_examples",
"LCLS_cspad_nexus",
"idx-20130301060858801.cbf",
)
assert os.path.isfile(image_path)
with open("process_lcls.phil", "w") as f:
f.write(
"""
dispatch.squash_errors = False
spotfinder {
filter.min_spot_size=2
threshold.dispersion.gain=25
threshold.dispersion.global_threshold=100
}
indexing {
known_symmetry {
space_group = P6122
unit_cell = 92.9 92.9 130.4 90 90 120
}
refinement_protocol.d_min_start=1.7
stills.refine_candidates_with_known_symmetry=True
}
"""
)
params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract()
params.output.experiments_filename = None
processor = Processor(params)
mem_img = dxtbx.load(image_path)
raw_data = mem_img.get_raw_data() # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(ImageSetData(MemReader([mem_img]), None))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None)
processor.process_experiments(
"20130301060858801", experiments
) # index/integrate the image
result = "idx-20130301060858801_integrated.refl"
n_refls = list(
range(140, 152)
) # large ranges to handle platform-specific differences
table = flex.reflection_table.from_file(result)
assert len(table) in n_refls, len(table)
assert "id" in table
assert (table["id"] == 0).count(False) == 0
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir,
composite_output):
# Check the data files for this test exist
image_path = os.path.join(
dials_regression,
"image_examples",
"LCLS_cspad_nexus",
"idx-20130301060858801.cbf",
)
assert os.path.isfile(image_path)
with open("process_lcls.phil", "w") as f:
f.write(cspad_cbf_in_memory_phil)
params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract()
params.output.experiments_filename = None
params.output.composite_output = composite_output
if composite_output:
processor = Processor(params, composite_tag="memtest")
else:
processor = Processor(params)
mem_img = dxtbx.load(image_path)
raw_data = mem_img.get_raw_data(
) # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(ImageSetData(MemReader([mem_img]), None))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None)
processor.process_experiments("20130301060858801",
experiments) # index/integrate the image
if composite_output:
processor.finalize()
result = "idx-memtest_integrated.refl"
else:
result = "idx-20130301060858801_integrated.refl"
n_refls = list(range(
140, 152)) # large ranges to handle platform-specific differences
table = flex.reflection_table.from_file(result)
assert len(table) in n_refls, len(table)
assert "id" in table
assert (table["id"] == 0).count(False) == 0
def __init__(self, img, gain, params, center_intensity=0):
""" Initialization and data read-in
"""
self.gain = gain
self.params = params
# Read settings from the DIALS target (.phil) file
# If none is provided, use default settings (and may God have mercy)
if self.params.dials.target != None:
with open(self.params.dials.target, 'r') as settings_file:
settings_file_contents = settings_file.read()
settings = parse(settings_file_contents)
current_phil = phil_scope.fetch(sources=[settings])
self.phil = current_phil.extract()
else:
self.phil = phil_scope.extract()
# Modify settings
self.phil.output.strong_filename = None
self.processor = IOTADialsProcessor(params=self.phil)
# Set customized parameters
beamX = self.params.image_conversion.beam_center.x
beamY = self.params.image_conversion.beam_center.y
if beamX != 0 or beamY != 0:
self.phil.geometry.detector.slow_fast_beam_centre = '{} {}'.format(
beamY, beamX)
if self.params.image_conversion.distance != 0:
self.phil.geometry.detector.distance = self.params.image_conversion.distance
if self.params.advanced.estimate_gain:
self.phil.spotfinder.threshold.dispersion.gain = gain
if self.params.image_conversion.mask is not None:
self.phil.spotfinder.lookup.mask = self.params.image_conversion.mask
self.phil.integration.lookup.mask = self.params.image_conversion.mask
if self.params.dials.auto_threshold:
threshold = int(center_intensity)
self.phil.spotfinder.threshold.dispersion.global_threshold = threshold
# Convert raw image into single-image datablock
with misc.Capturing() as junk_output:
self.datablock = DataBlockFactory.from_filenames([img])[0]
def __init__(self, params_filename, output_tag, logfile=None):
"""
@param params_filename cctbx.xfel/DIALS parameter file for processing
@output_tag String that will prefix output files
@logfile File name for logging
"""
self.parsed_params = parse(file_name=params_filename)
dials_params = phil_scope.fetch(self.parsed_params).extract()
super(CctbxPsanaEventProcessor, self).__init__(dials_params,
output_tag)
self.update_geometry = ManualGeometryUpdater(dials_params)
simple_script = SimpleScript(dials_params)
simple_script.load_reference_geometry()
self.reference_detector = getattr(simple_script, 'reference_detector',
None)
self.output_tag = output_tag
self.detector_params = None
if logfile is not None:
log.config(logfile=logfile)
def __init__(self,
iparams,
write_pickle=True,
write_logs=True,
last_stage='integrate'):
''' Constructor
:param iparams: IOTA params
:param write_pickle: Set to True to write out an integration pickle
'''
self.iparams = iparams
self.write_pickle = write_pickle
self.write_logs = write_logs
self.last_stage = last_stage
self.dlog_bookmark = 0
# Get Processor PHIL and initialize Processor
if self.iparams.cctbx_xfel.target:
with open(self.iparams.cctbx_xfel.target, 'r') as tf:
tphil_string = tf.read()
tparams = phil_scope.fetch(source=parse(tphil_string)).extract()
else:
tparams = phil_scope.extract()
Processor.__init__(self, params=tparams)
# IOTA-specific settings from here
# Turn off all peripheral output
self.params.output.experiments_filename = None
self.params.output.indexed_filename = None
self.params.output.strong_filename = None
self.params.output.refined_experiments_filename = None
self.params.output.integrated_experiments_filename = None
self.params.output.integrated_filename = None
self.params.output.profile_filename = None
# Set customized parameters
beamX = self.iparams.image_import.beam_center.x
beamY = self.iparams.image_import.beam_center.y
if beamX != 0 or beamY != 0:
self.params.geometry.detector.slow_fast_beam_centre = '{} {}'.format(
beamY, beamX)
if self.iparams.image_import.distance != 0:
self.params.geometry.detector.distance = self.iparams.image_import.distance
if self.iparams.image_import.mask is not None:
self.params.spotfinder.lookup.mask = self.iparams.image_import.mask
self.params.integration.lookup.mask = self.iparams.image_import.mask
if self.iparams.cctbx_xfel.target_space_group is not None:
sg = self.iparams.cctbx_xfel.target_space_group
self.params.indexing.known_symmetry.space_group = sg
if self.iparams.cctbx_xfel.target_unit_cell is not None:
uc = self.iparams.cctbx_xfel.target_unit_cell
self.params.indexing.known_symmetry.unit_cell = uc
if not self.params.indexing.stills.method_list:
self.params.indexing.stills.method_list = [
'fft1d', 'real_space_grid_search'
]
if self.iparams.cctbx_xfel.use_fft3d:
self.params.indexing.stills.method_list.insert(2, 'fft3d')
if self.iparams.cctbx_xfel.significance_filter.flag_on:
sigma = self.iparams.cctbx_xfel.significance_filter.sigma
sigma = sigma if sigma else 1
self.params.significance_filter.enable = True
self.params.significance_filter.isigi_cutoff = sigma
# Load reference geometry
self.reference_detector = None
if self.iparams.advanced.reference_geometry:
from dxtbx.model.experiment_list import ExperimentListFactory
try:
ref_experiments = ExperimentListFactory.from_json_file(
str(self.iparams.advanced.reference_geometry),
check_format=False)
except Exception as e:
print('DEBUG: Could not make exp. list because: ', e)
try:
import dxtbx
img = dxtbx.load(
str(self.iparams.advanced.reference_geometry))
except Exception:
print("DEBUG: Couldn't load geometry file {}"
"".format(self.iparams.advanced.reference_geometry))
else:
self.reference_detector = img.get_detector()
else:
assert len(ref_experiments.detectors()) == 1
self.reference_detector = ref_experiments.detectors()[0]
from dials.command_line.stills_process import phil_scope, Processor as BaseProcessor
from xfel.small_cell.command_line.small_cell_index import small_cell_phil_str
from iotbx.phil import parse
phil_scope.adopt_scope(parse(small_cell_phil_str))
# Use the center of mass (com) for the centroid definition for small cell.
program_defaults_phil_str = """
dispatch.refine = True
refinement.parameterisation.crystal.fix = cell
profile {
gaussian_rs {
centroid_definition = *com s1
}
}
"""
phil_scope = phil_scope.fetch(parse(program_defaults_phil_str))
class Processor(BaseProcessor):
def index(self, experiments, reflections):
from time import time
import copy
from xfel.small_cell.small_cell import small_cell_index_detail
st = time()
logger.info('*' * 80)
logger.info('Indexing Strong Spots')
logger.info('*' * 80)
params = copy.deepcopy(self.params)
def __init__(self,
img,
index=None,
termfile=None,
paramfile=None,
output_file=None,
output_dir=None,
backend='dials',
action_code='spotfind',
min_bragg=10,
n_processors=1,
verbose=False):
self.img = img
self.backend = backend
self.paramfile = paramfile
self.termfile = termfile
self.n_processors = n_processors
self.index = index
self.verbose = verbose
self.min_bragg = min_bragg
if output_file is not None:
if output_dir is not None:
self.output = os.path.join(os.path.abspath(output_dir),
output_file)
else:
self.output = os.path.abspath(output_file)
else:
self.output = None
Thread.__init__(self)
# Determine which processes will be included
if action_code == 'spotfind':
self.run_indexing = False
self.run_integration = False
elif action_code == 'index':
self.run_indexing = True
self.run_integration = False
elif action_code == 'integrate':
self.run_indexing = True
self.run_integration = True
# Initialize IOTA DIALS Processor
if self.backend.lower() == 'dials':
if self.paramfile is not None:
with open(self.paramfile, 'r') as phil_file:
phil_string = phil_file.read()
user_phil = ip.parse(phil_string)
self.cctbx_xfel_phil = phil_scope.fetch(source=user_phil)
else:
default_params, _ = write_defaults(method='dials',
write_target_file=False,
write_param_file=False)
default_phil_string = '\n'.join(default_params)
default_phil = ip.parse(default_phil_string)
self.cctbx_xfel_phil = phil_scope.fetch(source=default_phil)
self.params = self.cctbx_xfel_phil.extract()
# Modify default DIALS parameters
# These parameters will be set no matter what
self.params.output.datablock_filename = None
self.params.output.indexed_filename = None
self.params.output.strong_filename = None
self.params.output.refined_experiments_filename = None
self.params.output.integrated_filename = None
self.params.output.integrated_experiments_filename = None
self.params.output.profile_filename = None
self.params.output.integration_pickle = None
# These parameters will be set only if there's no script
if self.paramfile is None:
self.params.indexing.stills.method_list = ['fft3d']
self.params.spotfinder.threshold.dispersion.global_threshold = 75
if self.backend == 'dials':
self.processor = IOTADialsProcessor(params=self.params,
write_pickle=False)
def test_cspad_cbf_in_memory(self):
from os.path import join, exists
import os, dxtbx
from uuid import uuid4
from dials.command_line.stills_process import phil_scope, Processor
from libtbx.phil import parse
from dxtbx.imageset import ImageSet, ImageSetData, MemReader, MemMasker
from dxtbx.datablock import DataBlockFactory
from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory
import cPickle as pickle
dirname = 'tmp_%s' % uuid4().hex
os.mkdir(dirname)
os.chdir(dirname)
assert exists(join(self.lcls_path, 'idx-20130301060858801.cbf'))
f = open("process_lcls.phil", 'w')
f.write("""
dispatch.squash_errors = False
spotfinder {
filter.min_spot_size=2
threshold.dispersion.gain=25
threshold.dispersion.global_threshold=100
}
indexing {
known_symmetry {
space_group = P6122
unit_cell = 92.9 92.9 130.4 90 90 120
}
refinement_protocol.d_min_start=1.7
stills.refine_candidates_with_known_symmetry=True
}
""")
f.close()
params = phil_scope.fetch(
parse(file_name="process_lcls.phil")).extract()
params.output.datablock_filename = None
processor = Processor(params)
mem_img = dxtbx.load(join(self.lcls_path, 'idx-20130301060858801.cbf'))
raw_data = mem_img.get_raw_data(
) # cache the raw data to prevent swig errors
mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle)
mem_img._raw_data = raw_data
mem_img._cbf_handle = None # drop the file handle to prevent swig errors
imgset = ImageSet(
ImageSetData(MemReader([mem_img]), MemMasker([mem_img])))
imgset.set_beam(mem_img.get_beam())
imgset.set_detector(mem_img.get_detector())
datablock = DataBlockFactory.from_imageset(imgset)[0]
processor.process_datablock("20130301060858801",
datablock) # index/integrate the image
result = "idx-20130301060858801_integrated.pickle"
#n_refls = range(140,152) # large ranges to handle platform-specific differences
# 09/20/17 Changes to still indexer: refine candidate basis vectors in target symmetry if supplied
#n_refls = range(128,140) # large ranges to handle platform-specific differences
# 09/27/17 Bugfix for refine_candidates_with_known_symmetry
n_refls = range(
140, 152) # large ranges to handle platform-specific differences
table = pickle.load(open(result, 'rb'))
assert len(table) in n_refls, len(table)
assert 'id' in table
assert (table['id'] == 0).count(False) == 0
print 'OK'
def __init__(self,
source_image,
object_folder,
int_folder,
final_filename,
final,
logfile,
gain=0.32,
center_intensity=0,
params=None):
'''Initialise the script.'''
self.params = params
self.int_base = int_folder
# Read settings from the DIALS target (.phil) file
# If none is provided, use default settings (and may God have mercy)
if self.params.dials.target != None:
with open(self.params.dials.target, 'r') as settings_file:
settings_file_contents = settings_file.read()
settings = parse(settings_file_contents)
current_phil = phil_scope.fetch(source=settings)
else:
current_phil = phil_scope
self.phil = current_phil.extract()
# Set general file-handling settings
file_basename = misc.make_filename(source_image)
self.phil.output.datablock_filename = "{}/{}.json".format(
object_folder, file_basename)
self.phil.output.indexed_filename = "{}/{}_indexed.pickle".format(
object_folder, file_basename)
self.phil.output.strong_filename = "{}/{}_strong.pickle".format(
object_folder, file_basename)
self.phil.output.refined_experiments_filename = "{}/{}_refined_experiments.json".format(
object_folder, file_basename)
self.phil.output.integrated_experiments_filename = "{}/{}_integrated_experiments.json".format(
object_folder, file_basename)
self.phil.output.integrated_filename = "{}/{}_integrated.pickle".format(
object_folder, file_basename)
self.phil.output.profile_filename = "{}/{}_profile.phil".format(
object_folder, file_basename)
self.phil.output.integration_pickle = final_filename
self.int_log = logfile
# Set customized parameters
beamX = self.params.image_conversion.beam_center.x
beamY = self.params.image_conversion.beam_center.y
if beamX != 0 or beamY != 0:
self.phil.geometry.detector.slow_fast_beam_centre = '{} {}'.format(
beamY, beamX)
if self.params.image_conversion.distance != 0:
self.phil.geometry.detector.distance = self.params.image_conversion.distance
if self.params.image_conversion.mask is not None:
self.phil.spotfinder.lookup.mask = self.params.image_conversion.mask
self.phil.integration.lookup.mask = self.params.image_conversion.mask
if self.params.dials.target_space_group is not None:
sg = self.params.dials.target_space_group
self.phil.indexing.known_symmetry.space_group = sg
if self.params.dials.target_unit_cell is not None:
uc = self.params.dials.target_unit_cell
self.phil.indexing.known_symmetry.unit_cell = uc
if self.params.dials.use_fft3d:
self.phil.indexing.stills.method_list = [
'fft1d', 'fft3d', 'real_space_grid_search'
]
if self.params.dials.significance_filter.flag_on:
if self.params.dials.significance_filter.sigma is not None:
sigma = self.params.dials.significance_filter.sigma
self.phil.significance_filter.enable = True
self.phil.significance_filter.isigi_cutoff = sigma
# # Write target file for this IOTA run
# with misc.Capturing() as output:
# mod_phil = current_phil.format(python_object=self.phil)
# mod_phil.show()
# txt_out = ''
# for one_output in output:
# txt_out += one_output + '\n'
# local_target_file = os.path.join(self.int_base, 'target.phil')
# with open(local_target_file, 'w') as tf:
# tf.write(txt_out)
self.img = [source_image]
self.obj_base = object_folder
self.fail = None
self.frame = None
self.final = final
self.final['final'] = final_filename
self.datablock = DataBlockFactory.from_filenames(self.img)[0]
self.obj_filename = "int_{}".format(os.path.basename(self.img[0]))
# Auto-set threshold and gain (not saved for target.phil)
if self.params.dials.auto_threshold:
# This is still experimental and I'm not sure if it does anything...
# rad_avg = RadAverageCalculator(datablock=self.datablock)
# means, res = rad_avg.make_radial_average(num_bins=20, lowres=90)
# threshold = int(np.min(means) * 5)
threshold = int(center_intensity)
self.phil.spotfinder.threshold.dispersion.global_threshold = threshold
if self.params.advanced.estimate_gain:
self.phil.spotfinder.threshold.dispersion.gain = gain
