def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir): # Check the data files for this test exist image_path = os.path.join( dials_regression, "image_examples", "LCLS_cspad_nexus", "idx-20130301060858801.cbf", ) assert os.path.isfile(image_path) with open("process_lcls.phil", "w") as f: f.write( """ dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """ ) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.experiments_filename = None processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data() # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), None)) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None) processor.process_experiments( "20130301060858801", experiments ) # index/integrate the image result = "idx-20130301060858801_integrated.refl" n_refls = list( range(140, 152) ) # large ranges to handle platform-specific differences table = flex.reflection_table.from_file(result) assert len(table) in n_refls, len(table) assert "id" in table assert (table["id"] == 0).count(False) == 0
def test_cspad_cbf_in_memory(dials_regression, run_in_tmpdir, composite_output): # Check the data files for this test exist image_path = os.path.join( dials_regression, "image_examples", "LCLS_cspad_nexus", "idx-20130301060858801.cbf", ) assert os.path.isfile(image_path) with open("process_lcls.phil", "w") as f: f.write(cspad_cbf_in_memory_phil) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.experiments_filename = None params.output.composite_output = composite_output if composite_output: processor = Processor(params, composite_tag="memtest") else: processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), None)) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) experiments = ExperimentListFactory.from_imageset_and_crystal(imgset, None) processor.process_experiments("20130301060858801", experiments) # index/integrate the image if composite_output: processor.finalize() result = "idx-memtest_integrated.refl" else: result = "idx-20130301060858801_integrated.refl" n_refls = list(range( 140, 152)) # large ranges to handle platform-specific differences table = flex.reflection_table.from_file(result) assert len(table) in n_refls, len(table) assert "id" in table assert (table["id"] == 0).count(False) == 0
def __init__(self, img, gain, params, center_intensity=0): """ Initialization and data read-in """ self.gain = gain self.params = params # Read settings from the DIALS target (.phil) file # If none is provided, use default settings (and may God have mercy) if self.params.dials.target != None: with open(self.params.dials.target, 'r') as settings_file: settings_file_contents = settings_file.read() settings = parse(settings_file_contents) current_phil = phil_scope.fetch(sources=[settings]) self.phil = current_phil.extract() else: self.phil = phil_scope.extract() # Modify settings self.phil.output.strong_filename = None self.processor = IOTADialsProcessor(params=self.phil) # Set customized parameters beamX = self.params.image_conversion.beam_center.x beamY = self.params.image_conversion.beam_center.y if beamX != 0 or beamY != 0: self.phil.geometry.detector.slow_fast_beam_centre = '{} {}'.format( beamY, beamX) if self.params.image_conversion.distance != 0: self.phil.geometry.detector.distance = self.params.image_conversion.distance if self.params.advanced.estimate_gain: self.phil.spotfinder.threshold.dispersion.gain = gain if self.params.image_conversion.mask is not None: self.phil.spotfinder.lookup.mask = self.params.image_conversion.mask self.phil.integration.lookup.mask = self.params.image_conversion.mask if self.params.dials.auto_threshold: threshold = int(center_intensity) self.phil.spotfinder.threshold.dispersion.global_threshold = threshold # Convert raw image into single-image datablock with misc.Capturing() as junk_output: self.datablock = DataBlockFactory.from_filenames([img])[0]
def __init__(self, params_filename, output_tag, logfile=None): """ @param params_filename cctbx.xfel/DIALS parameter file for processing @output_tag String that will prefix output files @logfile File name for logging """ self.parsed_params = parse(file_name=params_filename) dials_params = phil_scope.fetch(self.parsed_params).extract() super(CctbxPsanaEventProcessor, self).__init__(dials_params, output_tag) self.update_geometry = ManualGeometryUpdater(dials_params) simple_script = SimpleScript(dials_params) simple_script.load_reference_geometry() self.reference_detector = getattr(simple_script, 'reference_detector', None) self.output_tag = output_tag self.detector_params = None if logfile is not None: log.config(logfile=logfile)
def __init__(self, iparams, write_pickle=True, write_logs=True, last_stage='integrate'): ''' Constructor :param iparams: IOTA params :param write_pickle: Set to True to write out an integration pickle ''' self.iparams = iparams self.write_pickle = write_pickle self.write_logs = write_logs self.last_stage = last_stage self.dlog_bookmark = 0 # Get Processor PHIL and initialize Processor if self.iparams.cctbx_xfel.target: with open(self.iparams.cctbx_xfel.target, 'r') as tf: tphil_string = tf.read() tparams = phil_scope.fetch(source=parse(tphil_string)).extract() else: tparams = phil_scope.extract() Processor.__init__(self, params=tparams) # IOTA-specific settings from here # Turn off all peripheral output self.params.output.experiments_filename = None self.params.output.indexed_filename = None self.params.output.strong_filename = None self.params.output.refined_experiments_filename = None self.params.output.integrated_experiments_filename = None self.params.output.integrated_filename = None self.params.output.profile_filename = None # Set customized parameters beamX = self.iparams.image_import.beam_center.x beamY = self.iparams.image_import.beam_center.y if beamX != 0 or beamY != 0: self.params.geometry.detector.slow_fast_beam_centre = '{} {}'.format( beamY, beamX) if self.iparams.image_import.distance != 0: self.params.geometry.detector.distance = self.iparams.image_import.distance if self.iparams.image_import.mask is not None: self.params.spotfinder.lookup.mask = self.iparams.image_import.mask self.params.integration.lookup.mask = self.iparams.image_import.mask if self.iparams.cctbx_xfel.target_space_group is not None: sg = self.iparams.cctbx_xfel.target_space_group self.params.indexing.known_symmetry.space_group = sg if self.iparams.cctbx_xfel.target_unit_cell is not None: uc = self.iparams.cctbx_xfel.target_unit_cell self.params.indexing.known_symmetry.unit_cell = uc if not self.params.indexing.stills.method_list: self.params.indexing.stills.method_list = [ 'fft1d', 'real_space_grid_search' ] if self.iparams.cctbx_xfel.use_fft3d: self.params.indexing.stills.method_list.insert(2, 'fft3d') if self.iparams.cctbx_xfel.significance_filter.flag_on: sigma = self.iparams.cctbx_xfel.significance_filter.sigma sigma = sigma if sigma else 1 self.params.significance_filter.enable = True self.params.significance_filter.isigi_cutoff = sigma # Load reference geometry self.reference_detector = None if self.iparams.advanced.reference_geometry: from dxtbx.model.experiment_list import ExperimentListFactory try: ref_experiments = ExperimentListFactory.from_json_file( str(self.iparams.advanced.reference_geometry), check_format=False) except Exception as e: print('DEBUG: Could not make exp. list because: ', e) try: import dxtbx img = dxtbx.load( str(self.iparams.advanced.reference_geometry)) except Exception: print("DEBUG: Couldn't load geometry file {}" "".format(self.iparams.advanced.reference_geometry)) else: self.reference_detector = img.get_detector() else: assert len(ref_experiments.detectors()) == 1 self.reference_detector = ref_experiments.detectors()[0]
from dials.command_line.stills_process import phil_scope, Processor as BaseProcessor from xfel.small_cell.command_line.small_cell_index import small_cell_phil_str from iotbx.phil import parse phil_scope.adopt_scope(parse(small_cell_phil_str)) # Use the center of mass (com) for the centroid definition for small cell. program_defaults_phil_str = """ dispatch.refine = True refinement.parameterisation.crystal.fix = cell profile { gaussian_rs { centroid_definition = *com s1 } } """ phil_scope = phil_scope.fetch(parse(program_defaults_phil_str)) class Processor(BaseProcessor): def index(self, experiments, reflections): from time import time import copy from xfel.small_cell.small_cell import small_cell_index_detail st = time() logger.info('*' * 80) logger.info('Indexing Strong Spots') logger.info('*' * 80) params = copy.deepcopy(self.params)
def __init__(self, img, index=None, termfile=None, paramfile=None, output_file=None, output_dir=None, backend='dials', action_code='spotfind', min_bragg=10, n_processors=1, verbose=False): self.img = img self.backend = backend self.paramfile = paramfile self.termfile = termfile self.n_processors = n_processors self.index = index self.verbose = verbose self.min_bragg = min_bragg if output_file is not None: if output_dir is not None: self.output = os.path.join(os.path.abspath(output_dir), output_file) else: self.output = os.path.abspath(output_file) else: self.output = None Thread.__init__(self) # Determine which processes will be included if action_code == 'spotfind': self.run_indexing = False self.run_integration = False elif action_code == 'index': self.run_indexing = True self.run_integration = False elif action_code == 'integrate': self.run_indexing = True self.run_integration = True # Initialize IOTA DIALS Processor if self.backend.lower() == 'dials': if self.paramfile is not None: with open(self.paramfile, 'r') as phil_file: phil_string = phil_file.read() user_phil = ip.parse(phil_string) self.cctbx_xfel_phil = phil_scope.fetch(source=user_phil) else: default_params, _ = write_defaults(method='dials', write_target_file=False, write_param_file=False) default_phil_string = '\n'.join(default_params) default_phil = ip.parse(default_phil_string) self.cctbx_xfel_phil = phil_scope.fetch(source=default_phil) self.params = self.cctbx_xfel_phil.extract() # Modify default DIALS parameters # These parameters will be set no matter what self.params.output.datablock_filename = None self.params.output.indexed_filename = None self.params.output.strong_filename = None self.params.output.refined_experiments_filename = None self.params.output.integrated_filename = None self.params.output.integrated_experiments_filename = None self.params.output.profile_filename = None self.params.output.integration_pickle = None # These parameters will be set only if there's no script if self.paramfile is None: self.params.indexing.stills.method_list = ['fft3d'] self.params.spotfinder.threshold.dispersion.global_threshold = 75 if self.backend == 'dials': self.processor = IOTADialsProcessor(params=self.params, write_pickle=False)
def test_cspad_cbf_in_memory(self): from os.path import join, exists import os, dxtbx from uuid import uuid4 from dials.command_line.stills_process import phil_scope, Processor from libtbx.phil import parse from dxtbx.imageset import ImageSet, ImageSetData, MemReader, MemMasker from dxtbx.datablock import DataBlockFactory from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory import cPickle as pickle dirname = 'tmp_%s' % uuid4().hex os.mkdir(dirname) os.chdir(dirname) assert exists(join(self.lcls_path, 'idx-20130301060858801.cbf')) f = open("process_lcls.phil", 'w') f.write(""" dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """) f.close() params = phil_scope.fetch( parse(file_name="process_lcls.phil")).extract() params.output.datablock_filename = None processor = Processor(params) mem_img = dxtbx.load(join(self.lcls_path, 'idx-20130301060858801.cbf')) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet( ImageSetData(MemReader([mem_img]), MemMasker([mem_img]))) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) datablock = DataBlockFactory.from_imageset(imgset)[0] processor.process_datablock("20130301060858801", datablock) # index/integrate the image result = "idx-20130301060858801_integrated.pickle" #n_refls = range(140,152) # large ranges to handle platform-specific differences # 09/20/17 Changes to still indexer: refine candidate basis vectors in target symmetry if supplied #n_refls = range(128,140) # large ranges to handle platform-specific differences # 09/27/17 Bugfix for refine_candidates_with_known_symmetry n_refls = range( 140, 152) # large ranges to handle platform-specific differences table = pickle.load(open(result, 'rb')) assert len(table) in n_refls, len(table) assert 'id' in table assert (table['id'] == 0).count(False) == 0 print 'OK'
def __init__(self, source_image, object_folder, int_folder, final_filename, final, logfile, gain=0.32, center_intensity=0, params=None): '''Initialise the script.''' self.params = params self.int_base = int_folder # Read settings from the DIALS target (.phil) file # If none is provided, use default settings (and may God have mercy) if self.params.dials.target != None: with open(self.params.dials.target, 'r') as settings_file: settings_file_contents = settings_file.read() settings = parse(settings_file_contents) current_phil = phil_scope.fetch(source=settings) else: current_phil = phil_scope self.phil = current_phil.extract() # Set general file-handling settings file_basename = misc.make_filename(source_image) self.phil.output.datablock_filename = "{}/{}.json".format( object_folder, file_basename) self.phil.output.indexed_filename = "{}/{}_indexed.pickle".format( object_folder, file_basename) self.phil.output.strong_filename = "{}/{}_strong.pickle".format( object_folder, file_basename) self.phil.output.refined_experiments_filename = "{}/{}_refined_experiments.json".format( object_folder, file_basename) self.phil.output.integrated_experiments_filename = "{}/{}_integrated_experiments.json".format( object_folder, file_basename) self.phil.output.integrated_filename = "{}/{}_integrated.pickle".format( object_folder, file_basename) self.phil.output.profile_filename = "{}/{}_profile.phil".format( object_folder, file_basename) self.phil.output.integration_pickle = final_filename self.int_log = logfile # Set customized parameters beamX = self.params.image_conversion.beam_center.x beamY = self.params.image_conversion.beam_center.y if beamX != 0 or beamY != 0: self.phil.geometry.detector.slow_fast_beam_centre = '{} {}'.format( beamY, beamX) if self.params.image_conversion.distance != 0: self.phil.geometry.detector.distance = self.params.image_conversion.distance if self.params.image_conversion.mask is not None: self.phil.spotfinder.lookup.mask = self.params.image_conversion.mask self.phil.integration.lookup.mask = self.params.image_conversion.mask if self.params.dials.target_space_group is not None: sg = self.params.dials.target_space_group self.phil.indexing.known_symmetry.space_group = sg if self.params.dials.target_unit_cell is not None: uc = self.params.dials.target_unit_cell self.phil.indexing.known_symmetry.unit_cell = uc if self.params.dials.use_fft3d: self.phil.indexing.stills.method_list = [ 'fft1d', 'fft3d', 'real_space_grid_search' ] if self.params.dials.significance_filter.flag_on: if self.params.dials.significance_filter.sigma is not None: sigma = self.params.dials.significance_filter.sigma self.phil.significance_filter.enable = True self.phil.significance_filter.isigi_cutoff = sigma # # Write target file for this IOTA run # with misc.Capturing() as output: # mod_phil = current_phil.format(python_object=self.phil) # mod_phil.show() # txt_out = '' # for one_output in output: # txt_out += one_output + '\n' # local_target_file = os.path.join(self.int_base, 'target.phil') # with open(local_target_file, 'w') as tf: # tf.write(txt_out) self.img = [source_image] self.obj_base = object_folder self.fail = None self.frame = None self.final = final self.final['final'] = final_filename self.datablock = DataBlockFactory.from_filenames(self.img)[0] self.obj_filename = "int_{}".format(os.path.basename(self.img[0])) # Auto-set threshold and gain (not saved for target.phil) if self.params.dials.auto_threshold: # This is still experimental and I'm not sure if it does anything... # rad_avg = RadAverageCalculator(datablock=self.datablock) # means, res = rad_avg.make_radial_average(num_bins=20, lowres=90) # threshold = int(np.min(means) * 5) threshold = int(center_intensity) self.phil.spotfinder.threshold.dispersion.global_threshold = threshold if self.params.advanced.estimate_gain: self.phil.spotfinder.threshold.dispersion.gain = gain