def make_int_object_list(self): """ Generates list of image objects from previous grid search """ from libtbx import easy_pickle as ep if self.params.cctbx.selection.select_only.grid_search_path == None: int_dir = misc.set_base_dir('integration', True) else: int_dir = self.params.cctbx.selection.select_only.grid_search_path img_objects = [] cmd.Command.start("Importing saved grid search results") for root, dirs, files in os.walk(int_dir): for filename in files: found_file = os.path.join(root, filename) if found_file.endswith(('int')): obj = ep.load(found_file) img_objects.append(obj) cmd.Command.end("Importing saved grid search results -- DONE") # Pick a randomized subset of images if self.params.advanced.random_sample.flag_on and \ self.params.advanced.random_sample.number < len(img_objects): gs_img_objects = self.select_random_subset(img_objects) else: gs_img_objects = img_objects return gs_img_objects
def update_settings(self): idxs = self.inp_box.ctr.GetItemCount() items = [self.inp_box.ctr.GetItemData(i) for i in range(idxs)] inputs = [] reference = None sequence = None for i in items: inp_type = i.type.type.GetString(i.type_selection) if inp_type in ('processed pickle list', 'processed pickle folder', 'processed pickle'): inputs.append(i.path) elif inp_type == 'reference MTZ': reference = i.path elif inp_type == 'sequence': sequence = i.path self.pparams.data = inputs if reference is not None: self.pparams.hklisoin = reference if self.opt_chk_useref.GetValue(): self.pparams.hklrefin = reference self.out_dir = self.out_box.ctr.GetValue() self.pparams.run_no = misc.set_base_dir( out_dir=self.out_dir) # Need to change self.pparams.title = self.project_title.ctr.GetValue() self.pparams.n_residues = self.opt_spc_nres.ctr.GetValue() self.pparams.n_processors = self.opt_spc_nproc.ctr.GetValue() update_phil = master_phil.format(python_object=self.pparams) self.regenerate_params(update_phil)
def make_int_object_list(self): """ Generates list of image objects from previous grid search """ from libtbx import easy_pickle as ep if self.params.cctbx.selection.select_only.grid_search_path == None: int_dir = misc.set_base_dir('integration', True) else: int_dir = self.params.cctbx.selection.select_only.grid_search_path img_objects = [] cmd.Command.start("Importing saved grid search results") for root, dirs, files in os.walk(int_dir): for filename in files: found_file = os.path.join(root, filename) if found_file.endswith(('int')): obj = ep.load(found_file) img_objects.append(obj) cmd.Command.end("Importing saved grid search results -- DONE") # Pick a randomized subset of images if self.params.advanced.random_sample.flag_on and \ self.params.advanced.random_sample.number < len(img_objects): gs_img_objects = self.select_random_subset(img_objects) else: gs_img_objects = img_objects return gs_img_objects
def init_settings(self): self.pparams = self.input_window.pparams self.pparams.data = [self.input_window.inp_box.ctr.GetValue()] self.out_dir = self.input_window.out_box.ctr.GetValue() self.pparams.run_no = misc.set_base_dir(out_dir=self.out_dir) # Need to change self.pparams.title = self.input_window.title_box.ctr.GetValue() if str(self.input_window.ref_box.ctr.GetValue()).lower() != '': self.pparams.hklisoin = self.input_window.ref_box.ctr.GetValue() if self.input_window.opt_chk_useref.GetValue(): self.pparams.hklrefin = self.input_window.ref_box.ctr.GetValue() self.pparams.n_residues = self.input_window.opt_spc_nres.GetValue() self.pparams.n_processors = self.input_window.opt_spc_nproc.GetValue()
def run(self): self.args, self.phil_args = parse_command_args( self.iver, self.help_message).parse_known_args() # Check for type of input if len(self.args.path) == 0 or self.args.path is None: # No input parse_command_args(self.iver, self.help_message).print_help() if self.args.default: # Write out default params and exit help_out, txt_out = inp.print_params() print '\n{:-^70}\n'.format('IOTA Parameters') print help_out inp.write_defaults(os.path.abspath(os.path.curdir), txt_out) misc.iota_exit() elif len(self.args.path) > 1: # If multiple paths / wildcards file_list = ginp.make_input_list(self.args.path) list_file = os.path.join(os.path.abspath(os.path.curdir), 'input.lst') with open(list_file, 'w') as lf: lf.write('\n'.join(file_list)) msg = "\nIOTA will run in AUTO mode using wildcard datapath:\n" \ "{} files found, compiled in {}\n".format(len(file_list), list_file) self.params, self.txt_out = inp.process_input( self.args, self.phil_args, list_file, 'auto', self.now) else: # If single path, check type carg = os.path.abspath(self.args.path[0]) if os.path.exists(carg): # If user provided a parameter file if os.path.isfile(carg) and os.path.basename(carg).endswith( '.param'): msg = '' self.params, self.txt_out = inp.process_input( self.args, self.phil_args, carg, 'file') # If user provided a list of input files elif os.path.isfile(carg) and os.path.basename(carg).endswith( '.lst'): msg = "\nIOTA will run in AUTO mode using {}:\n".format( carg) self.params, self.txt_out = inp.process_input( self.args, self.phil_args, carg, 'auto', self.now) # If user provided a single filepath elif os.path.isfile( carg) and not os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format( carg) self.params, self.txt_out = inp.process_input( self.args, self.phil_args, carg, 'auto', self.now) # If user provided a data folder elif os.path.isdir(carg): msg = "\nIOTA will run in AUTO mode using {}:\n".format( carg) self.params, self.txt_out = inp.process_input( self.args, self.phil_args, carg, 'auto', self.now) # If user provided gibberish else: print self.logo print "ERROR: Invalid input! Need parameter filename or data folder." misc.iota_exit() # Identify indexing / integration program if self.params.advanced.integrate_with == 'cctbx': prg = " with CCTBX.XFEL\n" elif self.params.advanced.integrate_with == 'dials': prg = " with DIALS\n" self.logo += prg print self.logo print '\n{}\n'.format(self.now) if msg != '': print msg if self.args.analyze != None: print 'ANALYSIS ONLY will be performed (analyzing run #{})'.format( self.args.analyze) self.analyze_prior_results('{:003d}'.format(int( self.args.analyze))) misc.iota_exit() if self.params.mp_method == 'mpi': rank, size = misc.get_mpi_rank_and_size() self.master_process = rank == 0 else: self.master_process = True # Call function to read input folder structure (or input file) and # generate list of image file paths if self.params.cctbx.selection.select_only.flag_on: self.gs_img_objects = self.make_int_object_list() self.input_list = [i.conv_img for i in self.gs_img_objects] else: self.input_list = self.make_input_list() # Check for -l option, output list of input files and exit if self.args.list: list_file = os.path.abspath("{}/input.lst".format(os.curdir)) # Check if other files of this name exist under the current folder list_folder = os.path.dirname(list_file) list_files = [ i for i in os.listdir(list_folder) if i.endswith(".lst") ] if len(list_files) > 0: list_file = os.path.join( list_folder, "input_{}.lst".format(len(list_files))) print '\nINPUT LIST ONLY option selected' print 'Input list in {} \n\n'.format(list_file) with open(list_file, "w") as lf: for i, input_file in enumerate(self.input_list, 1): lf.write('{}\n'.format(input_file)) print "{}: {}".format(i, input_file) lf.write('{}\n'.format(input_file)) print '\nExiting...\n\n' misc.iota_exit() # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles', out_dir=self.params.output) self.int_base = misc.set_base_dir('integration', out_dir=self.params.output) self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.log_base = os.path.join(self.int_base, 'logs') self.viz_base = os.path.join(self.int_base, 'visualization') self.tmp_base = os.path.join( '/tmp', '{}_{}'.format(os.getlogin(), time.time())) # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.log_base) os.makedirs(self.tmp_base) # Determine input base self.input_base = os.path.abspath( os.path.dirname(os.path.commonprefix(self.input_list))) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Log starting info misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) if self.params.advanced.integrate_with == 'cctbx': target_file = self.params.cctbx.target elif self.params.advanced.integrate_with == 'dials': target_file = self.params.dials.target misc.main_log( self.logfile, '{:-^80} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(target_file))) with open(target_file, 'r') as phil_file: phil_file_contents = phil_file.read() misc.main_log(self.logfile, phil_file_contents) # Write target file and record its location in params local_target_file = os.path.join(self.int_base, 'target.phil') with open(local_target_file, 'w') as tf: tf.write(phil_file_contents)
def run(self, gparams, target_phil=None, list_file=None): ''' Run initialization for IOTA GUI gparams = IOTA parameters from the GUI elements in PHIL format gtxt = text version of gparams list_file = if "Write Input List" button pressed, specifies name of list file ''' self.params = gparams self.target_phil = target_phil # If input list for some reason isn't transmitted from main window, make it if self.input_list is None: self.input_list = self.make_input_list() # Select range of images if turned on if self.params.advanced.image_range.flag_on: self.input_list = self.select_image_range(self.input_list) # Select a random subset of images if turned on if self.params.advanced.random_sample.flag_on and \ self.params.advanced.random_sample.number < len(self.input_list): self.input_list = self.select_random_subset(self.input_list) # Check for data not found if len(self.input_list) == 0: wx.MessageBox('ERROR: Data Not Found!', 'ERROR', wx.OK | wx.ICON_ERROR) return False # If list-only option selected, output list only if list_file != None: with open(list_file, "w") as lf: for i, input_file in enumerate(self.input_list, 1): lf.write('{}\n'.format(input_file)) return True # Run the sanity check procedure if not self.sanity_check(): return False # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.mp_method == 'multiprocessing': if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles', out_dir=self.params.output) self.int_base = misc.set_base_dir('integration', out_dir=self.params.output) self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.log_base = os.path.join(self.int_base, 'logs') self.viz_base = os.path.join(self.int_base, 'visualization') if str(self.params.advanced.temporary_output_folder).lower() in ( 'none', ''): self.tmp_base = os.path.join(self.int_base, 'tmp') else: self.tmp_base = os.path.join( self.params.advanced.temporary_output_folder) # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.log_base) try: if not os.path.isdir(self.tmp_base): os.makedirs(self.tmp_base) except OSError: pass # Determine input base common_pfx = os.path.abspath( os.path.dirname(os.path.commonprefix(self.input_list))) if len(self.params.input) == 1: self.input_base = os.path.commonprefix( [self.params.input[0], common_pfx]) else: self.input_base = common_pfx # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Write target file and record its location in params local_target_file = os.path.join(self.int_base, 'target.phil') if type(self.target_phil) == list: self.target_phil = '\n'.join(self.target_phil) with open(local_target_file, 'w') as tf: tf.write(self.target_phil) if self.params.advanced.integrate_with == 'cctbx': self.params.cctbx.target = local_target_file elif self.params.advanced.integrate_with == 'dials': self.params.dials.target = local_target_file # Collect final params and convert to PHIL object final_phil = inp.master_phil.format(python_object=self.params) # Generate text of params with misc.Capturing() as txt_output: final_phil.show() self.txt_out = '' for one_output in txt_output: self.txt_out += one_output + '\n' # Log starting info misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) # Log cctbx.xfel / DIALS settings misc.main_log( self.logfile, '{:-^80} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(local_target_file))) misc.main_log(self.logfile, self.target_phil) return True
def run(self): self.args, self.phil_args = parse_command_args(self.iver, self.help_message).parse_known_args() # Check for type of input if self.args.path == None: # No input parse_command_args(self.iver, self.help_message).print_help() if self.args.default: # Write out default params and exit help_out, txt_out = inp.print_params() print '\n{:-^70}\n'.format('IOTA Parameters') print help_out inp.write_defaults(os.path.abspath(os.path.curdir), txt_out) misc.iota_exit() else: # If input exists, check type carg = os.path.abspath(self.args.path) if os.path.exists(carg): # If user provided a parameter file if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'): msg = '' self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'file') # If user provided a list of input files elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a single filepath elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a data folder elif os.path.isdir(carg): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided gibberish else: print self.logo print "ERROR: Invalid input! Need parameter filename or data folder." misc.iota_exit() # Identify indexing / integration program if self.params.advanced.integrate_with == 'cctbx': prg = " with CCTBX.XFEL\n" elif self.params.advanced.integrate_with == 'dials': prg = " with DIALS\n" self.logo += prg print self.logo print '\n{}\n'.format(self.now) if msg != '': print msg if self.args.analyze != None: self.analyze_prior_results('{:003d}'.format(int(self.args.analyze))) misc.iota_exit() if self.params.mp_method == 'mpi': rank, size = misc.get_mpi_rank_and_size() self.master_process = rank == 0 else: self.master_process = True # Call function to read input folder structure (or input file) and # generate list of image file paths if self.params.cctbx.selection.select_only.flag_on: self.gs_img_objects = self.make_int_object_list() self.input_list = [i.conv_img for i in self.gs_img_objects] else: self.input_list = self.make_input_list() # Check for -l option, output list of input files and exit if self.args.list: if list_file == None: list_file = os.path.abspath("{}/input.lst".format(os.curdir)) print '\nINPUT LIST ONLY option selected' print 'Input list in {} \n\n'.format(list_file) with open(list_file, "w") as lf: for i, input_file in enumerate(self.input_list, 1): lf.write('{}\n'.format(input_file)) print "{}: {}".format(i, input_file) lf.write('{}\n'.format(input_file)) print '\nExiting...\n\n' misc.iota_exit() # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles', out_dir = self.params.output) self.int_base = misc.set_base_dir('integration', out_dir = self.params.output) self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.tmp_base = os.path.join(self.int_base, 'tmp') self.viz_base = os.path.join(self.int_base, 'visualization') # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.tmp_base) # Determine input base self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list))) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Log starting info misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) if self.params.advanced.integrate_with == 'cctbx': target_file = self.params.cctbx.target elif self.params.advanced.integrate_with == 'dials': target_file = self.params.dials.target misc.main_log(self.logfile, '{:-^80} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(target_file))) with open(target_file, 'r') as phil_file: phil_file_contents = phil_file.read() misc.main_log(self.logfile, phil_file_contents)