def write_default_phil(self): if str.lower(self.backend) in ('cctbx.xfel', 'dials'): method = 'cctbx.xfel' elif str.lower(self.backend) in ('cctbx', 'ha14', 'labelit'): method = 'ha14' else: method = 'current' from iota.components.iota_input import write_defaults default_phil, _ = write_defaults(method=method, write_target_file=False, write_param_file=False) self.target_phil = default_phil.as_str()
def write_default_phil(self): if str.lower(self.backend) in ("cctbx.xfel", "dials"): method = "cctbx.xfel" elif str.lower(self.backend) in ("cctbx", "ha14", "labelit"): method = "ha14" else: method = "current" from iota.components.iota_input import write_defaults default_phil, _ = write_defaults(method=method, write_target_file=False, write_param_file=False) self.target_phil = default_phil.as_str()
def sanity_check(self): ''' Check for conditions necessary to starting the run @return: True if passed, False if failed ''' # Check for existence of appropriate target files. If none are specified, # ask to generate defaults; if user says no, fail sanity check. If file is # specified but doesn't exist, show error message and fail sanity check if self.target_phil == None: if self.params.advanced.integrate_with == 'cctbx': write_def = wx.MessageDialog( None, 'WARNING! No target file for CCTBX.XFEL. ' 'Generate defaults?', 'WARNING', wx.YES_NO | wx.NO_DEFAULT | wx.ICON_EXCLAMATION) if (write_def.ShowModal() == wx.ID_YES): self.target_phil, _ = inp.write_defaults( current_path=self.params.output, write_param_file=False, method='cctbx') return True else: return False elif self.params.advanced.integrate_with == 'dials': write_def = wx.MessageDialog( None, 'WARNING! No target file for DIALS. ' 'Generate defaults?', 'WARNING', wx.YES_NO | wx.NO_DEFAULT | wx.ICON_EXCLAMATION) if (write_def.ShowModal() == wx.ID_YES): self.target_phil, _ = inp.write_defaults( current_path=self.params.output, write_param_file=False, method='dials') return True else: return False else: return True
def initialize_interface(self): """ Override with UI-specific init procedure :param gparams: IOTA parameters from UI and script :param input_list: list of input filepaths :return: True = initialization OK; False = initialization Failed """ # Generate default backend settings if none are specified if self.target_phil is None: self.target_phil, _ = inp.write_defaults( current_path=self.params.output, write_param_file=False, method='current') # If input list for some reason isn't transmitted from main window, make it if self.input_list is None: self.input_list = self.make_input_list() # In case any sub-setting of the input is anticipated, create a list of # all found input for future reference (i.e. run recovery) self.all_input = self.input_list # Select range of images if turned on if self.params.advanced.image_range.flag_on: self.input_list = self.select_image_range(self.input_list) # Select a random subset of images if turned on if self.params.advanced.random_sample.flag_on and \ self.params.advanced.random_sample.number < len(self.input_list): self.input_list = self.select_random_subset(self.input_list) # Run the sanity check procedure & return result is_good, msg = self.sanity_check() return is_good, msg
def initialize_new_run(phil, input_dict=None, target_phil=None): ''' Create base integration folder; safe phil, input, and info to file ''' try: params = phil.extract() int_base, run_no = util.set_base_dir(dirname='integration', out_dir=params.output, get_run_no=True) if not os.path.isdir(int_base): os.makedirs(int_base) # Create input list file and populate param input line if input_dict: if len(input_dict['imagepaths']) >= 10: input_list_file = os.path.join(int_base, 'input.lst') with open(input_list_file, 'w') as lf: for f in input_dict['imagefiles']: lf.write('{}\n'.format(f)) params.input = [input_list_file] else: input_list_file = None params.input = input_dict['imagepaths'] else: input_list_file = None # Generate default backend PHIL, write to file, and update params target_fp = os.path.join(int_base,'target.phil') if target_phil: target_phil = inp.write_phil(phil_str=target_phil, dest_file=target_fp, write_target_file=True) else: if params.cctbx_xfel.target: target_phil = inp.write_phil(phil_file=params.cctbx_xfel.target, dest_file=target_fp, write_target_file=True) else: method = params.advanced.processing_backend target_phil, _ = inp.write_defaults(method=method, write_param_file=False, filepath=target_fp) params.cctbx_xfel.target = target_fp # Save PHIL for this run in base integration folder paramfile = os.path.join(int_base, 'iota_r{}.param'.format(run_no)) phil = phil.format(python_object=params) with open(paramfile, 'w') as philf: philf.write(phil.as_str()) # Initialize main log logfile = os.path.abspath(os.path.join(int_base, 'iota.log')) # Initialize proc.info object and save to file info = ProcInfo.from_args( iota_phil=phil.as_str(), target_phil=target_phil.as_str(), int_base=int_base, input_list_file=input_list_file, info_file=os.path.join(int_base, 'proc.info'), cluster_info_file=os.path.join(int_base, 'cluster.info'), paramfile=paramfile, logfile=logfile, run_number=run_no, description=params.description, status='initialized', have_results=False, errors=[], init_proc=False) info.export_json() return True, info, 'IOTA_XTERM_INIT: Initialization complete!' except Exception as e: import traceback traceback.print_exc() msg = 'IOTA_INIT_ERROR: Could not initialize run! {}'.format(e) return False, None, msg
def run(self): self.args, self.phil_args = parse_command_args( self.iver, self.help_message).parse_known_args() # Check for type of input if len(self.args.path) == 0 or self.args.path is None: # No input parse_command_args(self.iver, self.help_message).print_help() if self.args.default: # Write out default params and exit help_out, txt_out = inp.print_params() print '\n{:-^70}\n'.format('IOTA Parameters') print help_out inp.write_defaults(os.path.abspath(os.path.curdir), txt_out) misc.iota_exit() elif len(self.args.path) > 1: # If multiple paths / wildcards file_list = ginp.make_input_list(self.args.path) list_file = os.path.join(os.path.abspath(os.path.curdir), 'input.lst') with open(list_file, 'w') as lf: lf.write('\n'.join(file_list)) msg = "\nIOTA will run in AUTO mode using wildcard datapath:\n" \ "{} files found, compiled in {}\n".format(len(file_list), list_file) self.params, self.txt_out = inp.process_input( self.args, self.phil_args, list_file, 'auto', self.now) else: # If single path, check type carg = os.path.abspath(self.args.path[0]) if os.path.exists(carg): # If user provided a parameter file if os.path.isfile(carg) and os.path.basename(carg).endswith( '.param'): msg = '' self.params, self.txt_out = inp.process_input( self.args, self.phil_args, carg, 'file') # If user provided a list of input files elif os.path.isfile(carg) and os.path.basename(carg).endswith( '.lst'): msg = "\nIOTA will run in AUTO mode using {}:\n".format( carg) self.params, self.txt_out = inp.process_input( self.args, self.phil_args, carg, 'auto', self.now) # If user provided a single filepath elif os.path.isfile( carg) and not os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format( carg) self.params, self.txt_out = inp.process_input( self.args, self.phil_args, carg, 'auto', self.now) # If user provided a data folder elif os.path.isdir(carg): msg = "\nIOTA will run in AUTO mode using {}:\n".format( carg) self.params, self.txt_out = inp.process_input( self.args, self.phil_args, carg, 'auto', self.now) # If user provided gibberish else: print self.logo print "ERROR: Invalid input! Need parameter filename or data folder." misc.iota_exit() # Identify indexing / integration program if self.params.advanced.integrate_with == 'cctbx': prg = " with CCTBX.XFEL\n" elif self.params.advanced.integrate_with == 'dials': prg = " with DIALS\n" self.logo += prg print self.logo print '\n{}\n'.format(self.now) if msg != '': print msg if self.args.analyze != None: print 'ANALYSIS ONLY will be performed (analyzing run #{})'.format( self.args.analyze) self.analyze_prior_results('{:003d}'.format(int( self.args.analyze))) misc.iota_exit() if self.params.mp_method == 'mpi': rank, size = misc.get_mpi_rank_and_size() self.master_process = rank == 0 else: self.master_process = True # Call function to read input folder structure (or input file) and # generate list of image file paths if self.params.cctbx.selection.select_only.flag_on: self.gs_img_objects = self.make_int_object_list() self.input_list = [i.conv_img for i in self.gs_img_objects] else: self.input_list = self.make_input_list() # Check for -l option, output list of input files and exit if self.args.list: list_file = os.path.abspath("{}/input.lst".format(os.curdir)) # Check if other files of this name exist under the current folder list_folder = os.path.dirname(list_file) list_files = [ i for i in os.listdir(list_folder) if i.endswith(".lst") ] if len(list_files) > 0: list_file = os.path.join( list_folder, "input_{}.lst".format(len(list_files))) print '\nINPUT LIST ONLY option selected' print 'Input list in {} \n\n'.format(list_file) with open(list_file, "w") as lf: for i, input_file in enumerate(self.input_list, 1): lf.write('{}\n'.format(input_file)) print "{}: {}".format(i, input_file) lf.write('{}\n'.format(input_file)) print '\nExiting...\n\n' misc.iota_exit() # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles', out_dir=self.params.output) self.int_base = misc.set_base_dir('integration', out_dir=self.params.output) self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.log_base = os.path.join(self.int_base, 'logs') self.viz_base = os.path.join(self.int_base, 'visualization') self.tmp_base = os.path.join( '/tmp', '{}_{}'.format(os.getlogin(), time.time())) # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.log_base) os.makedirs(self.tmp_base) # Determine input base self.input_base = os.path.abspath( os.path.dirname(os.path.commonprefix(self.input_list))) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Log starting info misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) if self.params.advanced.integrate_with == 'cctbx': target_file = self.params.cctbx.target elif self.params.advanced.integrate_with == 'dials': target_file = self.params.dials.target misc.main_log( self.logfile, '{:-^80} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(target_file))) with open(target_file, 'r') as phil_file: phil_file_contents = phil_file.read() misc.main_log(self.logfile, phil_file_contents) # Write target file and record its location in params local_target_file = os.path.join(self.int_base, 'target.phil') with open(local_target_file, 'w') as tf: tf.write(phil_file_contents)
def __init__(self, img, index=None, termfile=None, paramfile=None, output_file=None, output_dir=None, backend='dials', action_code='spotfind', min_bragg=10, n_processors=1, verbose=False): self.img = img self.backend = backend self.paramfile = paramfile self.termfile = termfile self.n_processors = n_processors self.index = index self.verbose = verbose self.min_bragg = min_bragg if output_file is not None: if output_dir is not None: self.output = os.path.join(os.path.abspath(output_dir), output_file) else: self.output = os.path.abspath(output_file) else: self.output = None Thread.__init__(self) # Determine which processes will be included if action_code == 'spotfind': self.run_indexing = False self.run_integration = False elif action_code == 'index': self.run_indexing = True self.run_integration = False elif action_code == 'integrate': self.run_indexing = True self.run_integration = True # Initialize IOTA DIALS Processor if self.backend.lower() == 'dials': if self.paramfile is not None: with open(self.paramfile, 'r') as phil_file: phil_string = phil_file.read() user_phil = ip.parse(phil_string) self.cctbx_xfel_phil = phil_scope.fetch(source=user_phil) else: default_params, _ = write_defaults(method='dials', write_target_file=False, write_param_file=False) default_phil_string = '\n'.join(default_params) default_phil = ip.parse(default_phil_string) self.cctbx_xfel_phil = phil_scope.fetch(source=default_phil) self.params = self.cctbx_xfel_phil.extract() # Modify default DIALS parameters # These parameters will be set no matter what self.params.output.datablock_filename = None self.params.output.indexed_filename = None self.params.output.strong_filename = None self.params.output.refined_experiments_filename = None self.params.output.integrated_filename = None self.params.output.integrated_experiments_filename = None self.params.output.profile_filename = None self.params.output.integration_pickle = None # These parameters will be set only if there's no script if self.paramfile is None: self.params.indexing.stills.method_list = ['fft3d'] self.params.spotfinder.threshold.dispersion.global_threshold = 75 if self.backend == 'dials': self.processor = IOTADialsProcessor(params=self.params, write_pickle=False)
def __init__(self, iparams, write_pickle=True, write_logs=True, last_stage="integrate"): """Constructor. :param iparams: IOTA params :param write_pickle: Set to True to write out an integration pickle """ self.iparams = iparams self.write_pickle = write_pickle self.write_logs = write_logs self.last_stage = last_stage self.dlog_bookmark = 0 # Get Processor PHIL and initialize Processor if self.iparams.cctbx_xfel.target: with open(self.iparams.cctbx_xfel.target, "r") as tf: tphil_string = tf.read() tparams = phil_scope.fetch(source=parse(tphil_string)).extract() else: from iota.components.iota_input import write_defaults method = self.iparams.advanced.processing_backend target_phil, _ = write_defaults(method=method, write_param_file=False, write_target_file=False) tparams = phil_scope.fetch(source=target_phil).extract() Processor.__init__(self, params=tparams) # IOTA-specific settings from here # Turn off all peripheral output self.params.output.experiments_filename = None self.params.output.indexed_filename = None self.params.output.strong_filename = None self.params.output.refined_experiments_filename = None self.params.output.integrated_experiments_filename = None self.params.output.integrated_filename = None self.params.output.profile_filename = None # Set customized parameters beamX = self.iparams.image_import.beam_center.x beamY = self.iparams.image_import.beam_center.y if beamX != 0 or beamY != 0: self.params.geometry.detector.slow_fast_beam_centre = "{} {}".format( beamY, beamX) if self.iparams.image_import.distance != 0: self.params.geometry.detector.distance = self.iparams.image_import.distance if self.iparams.image_import.mask is not None: self.params.spotfinder.lookup.mask = self.iparams.image_import.mask self.params.integration.lookup.mask = self.iparams.image_import.mask if self.iparams.cctbx_xfel.target_space_group is not None: sg = self.iparams.cctbx_xfel.target_space_group self.params.indexing.known_symmetry.space_group = sg if self.iparams.cctbx_xfel.target_unit_cell is not None: uc = self.iparams.cctbx_xfel.target_unit_cell self.params.indexing.known_symmetry.unit_cell = uc if not self.params.indexing.stills.method_list: self.params.indexing.stills.method_list = [ "fft1d", "real_space_grid_search", ] if self.iparams.cctbx_xfel.use_fft3d: self.params.indexing.stills.method_list.insert(2, "fft3d") if self.iparams.cctbx_xfel.significance_filter.flag_on: sigma = self.iparams.cctbx_xfel.significance_filter.sigma sigma = sigma if sigma else 1 self.params.significance_filter.enable = True self.params.significance_filter.isigi_cutoff = sigma # Load reference geometry self.reference_detector = None if self.iparams.advanced.reference_geometry: from dxtbx.model.experiment_list import ExperimentListFactory try: ref_experiments = ExperimentListFactory.from_json_file( str(self.iparams.advanced.reference_geometry), check_format=False) except Exception as e: print("DEBUG: Could not make exp. list because: ", e) try: import dxtbx img = dxtbx.load( str(self.iparams.advanced.reference_geometry)) except Exception: print("DEBUG: Couldn't load geometry file {}" "".format(self.iparams.advanced.reference_geometry)) else: self.reference_detector = img.get_detector() else: assert len(ref_experiments.detectors()) == 1 self.reference_detector = ref_experiments.detectors()[0]
def initialize_new_run(phil, input_dict=None, target_phil=None): ''' Create base integration folder; safe phil, input, and info to file ''' try: params = phil.extract() int_base, run_no = util.set_base_dir(dirname='integration', out_dir=params.output, get_run_no=True) if not os.path.isdir(int_base): os.makedirs(int_base) # Create input list file and populate param input line if input_dict: if len(input_dict['imagepaths']) >= 25: input_list_file = os.path.join(int_base, 'input.lst') with open(input_list_file, 'w') as lf: for f in input_dict['imagefiles']: lf.write('{}\n'.format(f)) params.input = [input_list_file] else: # If there are too many imagefiles, re-constitute the "glob" format # by matching filepaths and replacing non-matching characters with # asterisks if len(input_dict['imagefiles']) >= 25: input_paths = [] for path in input_dict['imagepaths']: fileset = [ os.path.basename(i) for i in input_dict['imagefiles'] if path in i ] zips = [list(set(i)) for i in zip(*fileset)] chars = [i[0] if len(i) == 1 else '*' for i in zips] fname = ''.join(chars) while "*" * 2 in fname: fname = fname.replace("*" * 2, "*") input_paths.append(os.path.join(path, fname)) params.input = input_paths else: params.input = input_dict['imagefiles'] input_list_file = None else: input_list_file = None # Generate default backend PHIL, write to file, and update params target_fp = os.path.join(int_base, 'target.phil') if target_phil: target_phil = inp.write_phil(phil_str=target_phil, dest_file=target_fp, write_target_file=True) else: if params.cctbx_xfel.target: target_phil = inp.write_phil( phil_file=params.cctbx_xfel.target, dest_file=target_fp, write_target_file=True) else: method = params.advanced.processing_backend target_phil, _ = inp.write_defaults(method=method, write_param_file=False, filepath=target_fp) params.cctbx_xfel.target = target_fp # Save PHIL for this run in base integration folder paramfile = os.path.join(int_base, 'iota_r{}.param'.format(run_no)) phil = phil.format(python_object=params) with open(paramfile, 'w') as philf: philf.write(phil.as_str()) # Initialize main log logfile = os.path.abspath(os.path.join(int_base, 'iota.log')) # Initialize proc.info object and save to file info = ProcInfo.from_args( iota_phil=phil.as_str(), target_phil=target_phil.as_str(), int_base=int_base, input_list_file=input_list_file, info_file=os.path.join(int_base, 'proc.info'), cluster_info_file=os.path.join(int_base, 'cluster.info'), paramfile=paramfile, logfile=logfile, run_number=run_no, description=params.description, status='initialized', have_results=False, errors=[], init_proc=False) info.export_json() return True, info, 'IOTA_XTERM_INIT: Initialization complete!' except Exception as e: msg = 'IOTA_INIT_ERROR: Could not initialize run! {}'.format(e) return False, None, msg
def run(self): self.args, self.phil_args = parse_command_args(self.iver, self.help_message).parse_known_args() # Check for type of input if self.args.path == None: # No input parse_command_args(self.iver, self.help_message).print_help() if self.args.default: # Write out default params and exit help_out, txt_out = inp.print_params() print '\n{:-^70}\n'.format('IOTA Parameters') print help_out inp.write_defaults(os.path.abspath(os.path.curdir), txt_out) misc.iota_exit() else: # If input exists, check type carg = os.path.abspath(self.args.path) if os.path.exists(carg): # If user provided a parameter file if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'): msg = '' self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'file') # If user provided a list of input files elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a single filepath elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a data folder elif os.path.isdir(carg): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided gibberish else: print self.logo print "ERROR: Invalid input! Need parameter filename or data folder." misc.iota_exit() # Identify indexing / integration program if self.params.advanced.integrate_with == 'cctbx': prg = " with CCTBX.XFEL\n" elif self.params.advanced.integrate_with == 'dials': prg = " with DIALS\n" self.logo += prg print self.logo print '\n{}\n'.format(self.now) if msg != '': print msg if self.args.analyze != None: self.analyze_prior_results('{:003d}'.format(int(self.args.analyze))) misc.iota_exit() if self.params.mp_method == 'mpi': rank, size = misc.get_mpi_rank_and_size() self.master_process = rank == 0 else: self.master_process = True # Call function to read input folder structure (or input file) and # generate list of image file paths if self.params.cctbx.selection.select_only.flag_on: self.gs_img_objects = self.make_int_object_list() self.input_list = [i.conv_img for i in self.gs_img_objects] else: self.input_list = self.make_input_list() # Check for -l option, output list of input files and exit if self.args.list: if list_file == None: list_file = os.path.abspath("{}/input.lst".format(os.curdir)) print '\nINPUT LIST ONLY option selected' print 'Input list in {} \n\n'.format(list_file) with open(list_file, "w") as lf: for i, input_file in enumerate(self.input_list, 1): lf.write('{}\n'.format(input_file)) print "{}: {}".format(i, input_file) lf.write('{}\n'.format(input_file)) print '\nExiting...\n\n' misc.iota_exit() # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles', out_dir = self.params.output) self.int_base = misc.set_base_dir('integration', out_dir = self.params.output) self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.tmp_base = os.path.join(self.int_base, 'tmp') self.viz_base = os.path.join(self.int_base, 'visualization') # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.tmp_base) # Determine input base self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list))) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Log starting info misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) if self.params.advanced.integrate_with == 'cctbx': target_file = self.params.cctbx.target elif self.params.advanced.integrate_with == 'dials': target_file = self.params.dials.target misc.main_log(self.logfile, '{:-^80} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(target_file))) with open(target_file, 'r') as phil_file: phil_file_contents = phil_file.read() misc.main_log(self.logfile, phil_file_contents)