def print_results(self): """ Prints diagnostics from the final integration run. """ cons_s = Counter(self.s).most_common(1)[0][0] cons_h = Counter(self.h).most_common(1)[0][0] cons_a = Counter(self.a).most_common(1)[0][0] final_table = [] final_table.append("\n\n{:-^80}\n".format('ANALYSIS OF RESULTS')) #final_table.append("Total images: {}".format(len( # self.final_objects))) if self.params.advanced.integrate_with == 'cctbx': final_table.append("Avg. signal height: {:<8.3f} std. dev: {:<6.2f}"\ " max: {:<3} min: {:<3} consensus: {:<3}"\ "".format(np.mean(self.s), np.std(self.s), max(self.s), min(self.s), cons_s)) final_table.append("Avg. spot height: {:<8.3f} std. dev: {:<6.2f}"\ " max: {:<3} min: {:<3} consensus: {:<3}"\ "".format(np.mean(self.h), np.std(self.h), max(self.h), min(self.h), cons_h)) final_table.append("Avg. spot areas: {:<8.3f} std. dev: {:<6.2f}"\ " max: {:<3} min: {:<3} consensus: {:<3}"\ "".format(np.mean(self.a), np.std(self.a), max(self.a), min(self.a), cons_a)) final_table.append("Avg. resolution: {:<8.3f} std. dev: {:<6.2f}"\ " lowest: {:<6.3f} highest: {:<6.3f}"\ "".format(np.mean(self.hres), np.std(self.hres), max(self.hres), min(self.hres))) final_table.append("Avg. number of spots: {:<8.3f} std. dev: {:<6.2f}"\ "".format(np.mean(self.spots), np.std(self.spots))) final_table.append("Avg. mosaicity: {:<8.3f} std. dev: {:<6.2f}"\ "".format(np.mean(self.mos), np.std(self.mos))) # If more than one integrated image, plot various summary graphs if len(self.final_objects) > 1 and self.params.analysis.summary_graphs: plot = Plotter(self.params, self.final_objects, self.viz_dir) if ( self.params.advanced.integrate_with == 'cctbx' and self.params.cctbx.grid_search.type != None ): plot.plot_spotfinding_heatmap(write_files=True) plot.plot_res_histogram(write_files=True) med_beamX, med_beamY = plot.plot_beam_xy(write_files=True, return_values=True) final_table.append("Median Beam Center: X = {:<4.2f}, Y = {:<4.2f}"\ "".format(med_beamX, med_beamY)) for item in final_table: misc.main_log(self.logfile, item, (not self.gui_mode))
def run(self): ''' Run IOTA ''' # Import Images self.stage = 'import' self.run_import() # Remove rejected images from image object list acc_img_objects = [i.fail for i in self.img_objects if i.fail == None] cmd.Command.end("Accepted {} of {} images -- DONE " \ "".format(len(acc_img_objects), len(self.img_objects))) # Exit if none of the images have diffraction if str(self.init.params.image_triage.type).lower() != 'none': if len(acc_img_objects) == 0: misc.main_log(self.init.logfile, 'No images have diffraction!', True) misc.iota_exit() else: misc.main_log(self.init.logfile, "{} out of {} images have diffraction " "(at least {} Bragg peaks)" "".format(len(acc_img_objects), len(self.img_objects), self.init.params.image_triage.min_Bragg_peaks)) # Check for -c option and exit if true if self.init.params.image_conversion.convert_only: misc.iota_exit() # Process Images self.stage = 'process' self.run_process() # Analysis of integration results final_objects = [i for i in self.img_objects if i.fail == None] if len(final_objects) > 0: self.run_analysis() else: print 'No images successfully integrated!' # Exit IOTA misc.iota_exit()
def run(self): self.args, self.phil_args = parse_command_args(self.iver, self.help_message).parse_known_args() # Check for type of input if self.args.path == None: # No input parse_command_args(self.iver, self.help_message).print_help() if self.args.default: # Write out default params and exit help_out, txt_out = inp.print_params() print '\n{:-^70}\n'.format('IOTA Parameters') print help_out inp.write_defaults(os.path.abspath(os.path.curdir), txt_out) misc.iota_exit() else: # If input exists, check type carg = os.path.abspath(self.args.path) if os.path.exists(carg): # If user provided a parameter file if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'): msg = '' self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'file') # If user provided a list of input files elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a single filepath elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a data folder elif os.path.isdir(carg): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided gibberish else: print self.logo print "ERROR: Invalid input! Need parameter filename or data folder." misc.iota_exit() # Identify indexing / integration program if self.params.advanced.integrate_with == 'cctbx': prg = " with CCTBX.XFEL\n" elif self.params.advanced.integrate_with == 'dials': prg = " with DIALS\n" self.logo += prg print self.logo print '\n{}\n'.format(self.now) if msg != '': print msg if self.args.analyze != None: self.analyze_prior_results('{:003d}'.format(int(self.args.analyze))) misc.iota_exit() if self.params.mp_method == 'mpi': rank, size = misc.get_mpi_rank_and_size() self.master_process = rank == 0 else: self.master_process = True # Call function to read input folder structure (or input file) and # generate list of image file paths if self.params.cctbx.selection.select_only.flag_on: self.gs_img_objects = self.make_int_object_list() self.input_list = [i.conv_img for i in self.gs_img_objects] else: self.input_list = self.make_input_list() # Check for -l option, output list of input files and exit if self.args.list: if list_file == None: list_file = os.path.abspath("{}/input.lst".format(os.curdir)) print '\nINPUT LIST ONLY option selected' print 'Input list in {} \n\n'.format(list_file) with open(list_file, "w") as lf: for i, input_file in enumerate(self.input_list, 1): lf.write('{}\n'.format(input_file)) print "{}: {}".format(i, input_file) lf.write('{}\n'.format(input_file)) print '\nExiting...\n\n' misc.iota_exit() # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles', out_dir = self.params.output) self.int_base = misc.set_base_dir('integration', out_dir = self.params.output) self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.tmp_base = os.path.join(self.int_base, 'tmp') self.viz_base = os.path.join(self.int_base, 'visualization') # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.tmp_base) # Determine input base self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list))) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Log starting info misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) if self.params.advanced.integrate_with == 'cctbx': target_file = self.params.cctbx.target elif self.params.advanced.integrate_with == 'dials': target_file = self.params.dials.target misc.main_log(self.logfile, '{:-^80} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(target_file))) with open(target_file, 'r') as phil_file: phil_file_contents = phil_file.read() misc.main_log(self.logfile, phil_file_contents)
def process(self, single_image=False): """ Image processing; selects method, runs requisite modules """ if self.status != 'bypass grid search': self.status = 'processing' #for CCTBX indexing / integration if self.params.advanced.integrate_with == 'cctbx': terminate = False prev_status = self.status prev_fail = 'first cycle' prev_final = self.final prev_epv = 9999 while not terminate: if os.path.isfile(self.abort_file): self.fail = 'aborted' return self # Run grid search if haven't already if self.fail == None and 'grid search' not in self.status: self.integrate_cctbx('grid search', single_image=single_image) # Run selection if haven't already if self.fail == None and self.status != 'selection': self.select_cctbx() # If smart grid search is active run multiple rounds until convergence if self.params.cctbx.grid_search.type == 'smart': if self.fail == None and self.final['epv'] < prev_epv: prev_epv = self.final['epv'] prev_final = self.final prev_status = self.status prev_fail = self.fail self.hmed = self.final['sph'] self.amed = self.final['spa'] self.generate_grid() self.final['final'] = self.fin_file if len(self.grid) == 0: self.final = prev_final self.status = prev_status self.fail = prev_fail terminate = True continue if self.verbose: log_entry = '\nNew starting point: H = {}, A = {}\n'\ ''.format(self.hmed, self.amed) self.log_info.append(log_entry) else: if prev_fail != 'first cycle': self.final = prev_final self.status = prev_status self.fail = prev_fail if self.verbose: log_entry = '\nFinal set of parameters: H = {}, A = {}'\ ''.format(self.final['sph'], self.final['spa']) self.log_info.append(log_entry) terminate = True # If brute force grid search is selected run one round else: terminate = True # Run final integration if haven't already if self.fail == None and self.status != 'final': self.integrate_cctbx('integrate', single_image=single_image) # If verbose output selected (default), write to main log if self.verbose: log_entry = "\n".join(self.log_info) misc.main_log(self.main_log, log_entry) misc.main_log(self.main_log, '\n\n') # Make a temporary process log into a final process log if os.path.isfile(self.int_log): final_int_log = os.path.join(self.fin_path, os.path.basename(self.int_log).split('.')[0] + '.log') os.rename(self.int_log, final_int_log) # For DIALS integration (WORK IN PROGRESS) elif self.params.advanced.integrate_with == 'dials': if os.path.isfile(self.abort_file): self.fail = 'aborted' return self # Create DIALS integrator object from iota.components.iota_dials import Integrator integrator = Integrator(self.conv_img, self.obj_base, self.fin_base, self.fin_file, self.final, self.int_log, self.gain, self.params) # Run DIALS self.fail, self.final, int_log = integrator.run() if self.fail != None: self.status = 'final' self.log_info.append(int_log) log_entry = "\n".join(self.log_info) misc.main_log(self.main_log, log_entry) misc.main_log(self.main_log, '\n{:-^100}\n'.format('')) # Make a temporary process log into a final process log final_int_log = self.int_log.split('.')[0] + ".log" os.rename(self.int_log, final_int_log) self.status = 'final' ep.dump(self.obj_file, self)
def import_image(self): """ Image conversion: - Writes out data in pickle format (cctbx.xfel only) - Moves beam center into center of image, crops / pads image (cctbx.xfel only) - Adjusts beam center and distance (optional) - Thresholds and masks beamstop shadow (optional) """ if os.path.isfile(self.abort_file): self.fail = 'aborted' return self # Load image img_data, img_type = self.load_image() self.status = 'loaded' if img_data is None: self.log_info.append('\n{:-^100}\n'.format(self.raw_img)) self.log_info.append('FAILED TO IMPORT') self.status = 'failed import' self.fail = 'failed import' return self # if DIALS is selected, change image type to skip conversion step if self.params.advanced.integrate_with == 'dials': img_type = 'dials_input' # Log initial image information self.log_info.append('\n{:-^100}\n'.format(self.raw_img)) self.log_info.append('Imported image : {}'.format(self.raw_img)) self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\ 'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\ 'DIST = {}'.format(img_data['BEAM_CENTER_X'], img_data['BEAM_CENTER_Y'], img_data['PIXEL_SIZE'], img_data['SIZE1'], img_data['SIZE2'], img_data['DISTANCE'])) # Deactivate image squaring if beam center is in the center of the image # CCTBX.XFEL ONLY (Need a better displacement cutoff) if ( self.params.advanced.integrate_with == 'dials' or abs(img_data['BEAM_CENTER_X'] - img_data['BEAM_CENTER_Y']) < 0.1 ): self.params.image_conversion.square_mode = 'None' # Check if conversion/modification is required and carry them out if ( img_type == 'raw' or self.params.image_conversion.square_mode != "None" or self.params.image_conversion.beam_center.x != 0 or self.params.image_conversion.beam_center.y != 0 or self.params.image_conversion.beamstop != 0 or self.params.image_conversion.distance != 0 ): # Check for and/or create a converted pickles folder try: if not os.path.isdir(self.conv_base): os.makedirs(self.conv_base) except OSError: pass # Generate converted image pickle filename if self.params.image_conversion.rename_pickle_prefix != None: if str(self.params.image_conversion.rename_pickle_prefix).lower() == "auto": try: prefix = os.getlogin() except Exception: prefix = 'converted' else: prefix = self.params.image_conversion.rename_pickle_prefix number = int(os.path.basename(self.conv_base)) self.conv_img = os.path.abspath(os.path.join(self.conv_base, "{}_{}_{:05d}.pickle".format(prefix, number, self.img_index))) else: # This option preserves the input directory structure img_path = misc.make_image_path(self.raw_img, self.input_base, self.conv_base) self.conv_img = os.path.abspath(os.path.join(img_path, os.path.basename(self.raw_img).split('.')[0] + ".pickle")) try: if not os.path.isdir(img_path): os.makedirs(img_path) except OSError: pass # Convert raw image to image pickle beamstop = self.params.image_conversion.beamstop distance = self.params.image_conversion.distance beam_center = [self.params.image_conversion.beam_center.x, self.params.image_conversion.beam_center.y] square = self.params.image_conversion.square_mode if beam_center != [0,0]: pixel_size = img_data['PIXEL_SIZE'] img_data['BEAM_CENTER_X'] = int(round(beam_center[0] * pixel_size)) img_data['BEAM_CENTER_Y'] = int(round(beam_center[1] * pixel_size)) if distance != 0: img_data['DISTANCE'] = distance if square != "None": img_data = self.square_pickle(img_data) if beamstop != 0: img_data = self.mask_image(img_data) # Log converted image information self.log_info.append('Converted image : {}'.format(self.conv_img)) self.log_info.append('Parameters : BEAM_X = {:<4.2f}, BEAM_Y = {:<4.2f}, '\ 'PIXEL_SIZE = {:<8.6f}, IMG_SIZE = {:<4} X {:<4}, '\ 'DIST = {}'.format(img_data['BEAM_CENTER_X'], img_data['BEAM_CENTER_Y'], img_data['PIXEL_SIZE'], img_data['SIZE1'], img_data['SIZE2'], img_data['DISTANCE'])) self.input_base = self.conv_base self.status = 'converted' # Save converted image pickle ep.dump(self.conv_img, img_data) # Triage image (i.e. check for usable diffraction, using selected method) if str(self.params.image_triage.type).lower() != 'none': if self.params.advanced.integrate_with == 'cctbx': from iota.components.iota_cctbx import Triage triage = Triage(self.conv_img, self.gain, self.params) self.fail, log_entry, self.hmed, self.amed = triage.triage_image() elif self.params.advanced.integrate_with == 'dials': from iota.components.iota_dials import Triage triage = Triage(self.conv_img, self.gain, self.params) self.fail, log_entry = triage.triage_image() self.log_info.append(log_entry) self.status = 'triaged' else: self.fail = None # Generate integration result dictionary for cctbx.xfel or DIALS if self.params.advanced.integrate_with == 'cctbx': if self.params.cctbx.grid_search.type == 'smart': self.hrange = 1 self.arange = 1 else: self.hrange = self.params.cctbx.grid_search.height_range self.arange = self.params.cctbx.grid_search.area_range self.grid_points = [] self.generate_grid() elif self.params.advanced.integrate_with == 'dials': self.final = {'img':self.conv_img, 'a':0, 'b':0, 'c':0, 'alpha':0, 'beta':0, 'gamma':0, 'sg':'','strong':0, 'res':0, 'lres':0, 'mos':0, 'epv':0, 'info':'','final':None, 'program':'dials'} # Generate names for output folders and files: if not self.params.image_conversion.convert_only: self.obj_path = misc.make_image_path(self.conv_img, self.input_base, self.obj_base) self.obj_file = os.path.abspath(os.path.join(self.obj_path, os.path.basename(self.conv_img).split('.')[0] + ".int")) self.fin_path = misc.make_image_path(self.conv_img, self.input_base, self.fin_base) self.fin_file = os.path.abspath(os.path.join(self.fin_path, "int_{}.pickle".format(os.path.basename(self.conv_img).split('.')[0]))) self.final['final'] = self.fin_file self.final['img'] = self.conv_img self.int_log = os.path.join(self.fin_path, os.path.basename(self.conv_img).split('.')[0] + '.tmp') self.viz_path = misc.make_image_path(self.conv_img, self.input_base, self.viz_base) self.viz_file = os.path.join(self.viz_path, "int_{}.png".format(os.path.basename(self.conv_img).split('.')[0])) # Create actual folders (if necessary) try: if not os.path.isdir(self.obj_path): os.makedirs(self.obj_path) if not os.path.isdir(self.fin_path): os.makedirs(self.fin_path) if not os.path.isdir(self.viz_path): os.makedirs(self.viz_path) except OSError: pass # Save image object to file ep.dump(self.obj_file, self) self.status = 'imported' # If conversion only option is selected, write conversion info to log if self.params.image_conversion.convert_only: log_entry = "\n".join(self.log_info) misc.main_log(self.main_log, log_entry) return self
def print_results(self, final_table=None): """ Prints diagnostics from the final integration run. """ cons_s = Counter(self.s).most_common(1)[0][0] cons_h = Counter(self.h).most_common(1)[0][0] cons_a = Counter(self.a).most_common(1)[0][0] if final_table is None: final_table = [] final_table.append("\n\n{:-^80}\n".format('ANALYSIS OF RESULTS')) # In case no images were integrated if self.final_objects is None: final_table.append('NO IMAGES INTEGRATED!') else: if self.params.advanced.integrate_with == 'cctbx': final_table.append( "Avg. signal height: {:<8.3f} std. dev: " "{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}" "".format(np.mean(self.s), np.std(self.s), max(self.s), min(self.s), cons_s)) final_table.append( "Avg. spot height: {:<8.3f} std. dev: " "{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}" "".format(np.mean(self.h), np.std(self.h), max(self.h), min(self.h), cons_h)) final_table.append( "Avg. spot areas: {:<8.3f} std. dev: " "{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}" "".format(np.mean(self.a), np.std(self.a), max(self.a), min(self.a), cons_a)) final_table.append( "Avg. resolution: {:<8.3f} std. dev: " "{:<6.2f} lowest: {:<6.3f} highest: {:<6.3f}" "".format(np.mean(self.hres), np.std(self.hres), max(self.hres), min(self.hres))) final_table.append( "Avg. number of spots: {:<8.3f} std. dev: {:<6.2f}" "".format(np.mean(self.spots), np.std(self.spots))) final_table.append( "Avg. mosaicity: {:<8.3f} std. dev: {:<6.2f}" "".format(np.mean(self.mos), np.std(self.mos))) # If more than one integrated image, plot various summary graphs if len(self.final_objects) > 1: plot = Plotter(self.params, self.final_objects, self.viz_dir) if self.params.analysis.summary_graphs: if (self.params.advanced.integrate_with == 'cctbx' and self.params.cctbx.grid_search.type != None): plot.plot_spotfinding_heatmap(write_files=True) plot.plot_res_histogram(write_files=True) med_beamX, med_beamY, pixel_size = plot.plot_beam_xy( write_files=True, return_values=True) else: beamXY_info = plot.calculate_beam_xy() beamX, beamY = beamXY_info[:2] med_beamX = np.median(beamX) med_beamY = np.median(beamY) pixel_size = beamXY_info[-1] final_table.append( "Median Beam Center: X = {:<4.2f}, Y = {:<4.2f}" "".format(med_beamX, med_beamY)) self.analysis_result.__setattr__('beamX_mm', med_beamX) self.analysis_result.__setattr__('beamY_mm', med_beamY) self.analysis_result.__setattr__('pixel_size', pixel_size) self.analysis_result.__setattr__('final_table', final_table) for item in final_table: misc.main_log(self.logfile, item, (not self.gui_mode))
def run(self, gparams, target_phil=None, list_file=None): ''' Run initialization for IOTA GUI gparams = IOTA parameters from the GUI elements in PHIL format gtxt = text version of gparams list_file = if "Write Input List" button pressed, specifies name of list file ''' self.params = gparams self.target_phil = target_phil # If input list for some reason isn't transmitted from main window, make it if self.input_list is None: self.input_list = self.make_input_list() # Select range of images if turned on if self.params.advanced.image_range.flag_on: self.input_list = self.select_image_range(self.input_list) # Select a random subset of images if turned on if self.params.advanced.random_sample.flag_on and \ self.params.advanced.random_sample.number < len(self.input_list): self.input_list = self.select_random_subset(self.input_list) # Check for data not found if len(self.input_list) == 0: wx.MessageBox('ERROR: Data Not Found!', 'ERROR', wx.OK | wx.ICON_ERROR) return False # If list-only option selected, output list only if list_file != None: with open(list_file, "w") as lf: for i, input_file in enumerate(self.input_list, 1): lf.write('{}\n'.format(input_file)) return True # Run the sanity check procedure if not self.sanity_check(): return False # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.mp_method == 'multiprocessing': if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles', out_dir=self.params.output) self.int_base = misc.set_base_dir('integration', out_dir=self.params.output) self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.log_base = os.path.join(self.int_base, 'logs') self.viz_base = os.path.join(self.int_base, 'visualization') if str(self.params.advanced.temporary_output_folder).lower() in ( 'none', ''): self.tmp_base = os.path.join(self.int_base, 'tmp') else: self.tmp_base = os.path.join( self.params.advanced.temporary_output_folder) # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.log_base) try: if not os.path.isdir(self.tmp_base): os.makedirs(self.tmp_base) except OSError: pass # Determine input base self.input_base = os.path.abspath( os.path.dirname(os.path.commonprefix(self.input_list))) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Write target file and record its location in params local_target_file = os.path.join(self.int_base, 'target.phil') if type(self.target_phil) == list: self.target_phil = '\n'.join(self.target_phil) with open(local_target_file, 'w') as tf: tf.write(self.target_phil) if self.params.advanced.integrate_with == 'cctbx': self.params.cctbx.target = local_target_file elif self.params.advanced.integrate_with == 'dials': self.params.dials.target = local_target_file # Collect final params and convert to PHIL object final_phil = inp.master_phil.format(python_object=self.params) # Generate text of params with misc.Capturing() as txt_output: final_phil.show() self.txt_out = '' for one_output in txt_output: self.txt_out += one_output + '\n' # Log starting info misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) # Log cctbx.xfel / DIALS settings misc.main_log( self.logfile, '{:-^80} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(local_target_file))) misc.main_log(self.logfile, self.target_phil) return True
class Integrator(object): """ Class for image integration (w/ grid search params) """ def __init__(self, params, source_image=None, output_image=None, viz=None, log=None, tag='grid search', tmp_base=None, gain=1, single_image=False): self.params = params self.img = source_image self.out_img = output_image self.min_sigma = self.params.cctbx.selection.min_sigma self.target = os.path.abspath(self.params.cctbx.target) self.viz = viz self.tag = tag self.int_log = log self.charts = self.params.analysis.charts self.tmp_base = tmp_base self.single_image = single_image self.method = self.params.mp_method self.queue = self.params.mp_queue self.args = [ "target={}".format(self.target), "indexing.data={}".format(self.img), "spots_pickle=None", "subgroups_pickle=None", "refinements_pickle=None", "rmsd_tolerance=5.0", "mosflm_rmsd_tolerance=5.0", "integration.detector_gain={}".format(gain), "indexing.verbose_cv=True" ] # Add target unit cell if exists if self.params.cctbx.target_unit_cell is not None: t_uc = [ str(i) for i in self.params.cctbx.target_unit_cell.parameters() ] self.args.extend(['target_cell="{}"'.format(' '.join(t_uc))]) # Translate / add target lattice if exists t_lat = self.params.cctbx.target_lattice_type if t_lat is not None: if t_lat == 'triclinic': known_setting = 1 elif t_lat == 'monoclinic': known_setting = 2 elif t_lat in ('orthorhombic', 'rhombohedral'): known_setting = 5 elif t_lat == 'tetragonal': known_setting = 9 elif t_lat == 'hexagonal': known_setting = 12 elif t_lat == 'cubic': known_setting = 22 self.args.extend(['known_setting={}'.format(known_setting)]) # Centering type if exists t_ctype = self.params.cctbx.target_centering_type if t_ctype is not None: self.args.extend(['target_cell_centring_type={}'.format(t_ctype)]) # Resolution, if exists hires = self.params.cctbx.resolution_limits.high lowres = self.params.cctbx.resolution_limits.low if hires is None: hires = 1.5 if lowres is None: lowres = 99.9 self.args.extend([ 'force_method2_resolution_limit={}'.format(hires), 'distl_lowres_limit={}'.format(lowres), 'distl_highres_limit={}'.format(hires), 'distl.res.inner={}'.format(lowres), 'distl.res.outer={}'.format(hires) ]) # Add gain (necessary now) self.args.extend(['integration.detector_gain={}'.format(gain)]) with open(os.path.join(self.params.output, 'test.txt'), 'w') as tf: tf.write('\n'.join(self.args)) def integrate(self, grid_point): """ Runs the integration module in cctbx.xfel; used by either grid-search or final integration function. """ self.s = grid_point['sih'] self.h = grid_point['sph'] self.a = grid_point['spa'] args = self.args # Generate advanced arguments (and PDF subfolder) if self.charts and self.tag == 'grid search': filename = os.path.basename(self.img).split('.')[0] pdf_folder = os.path.join(self.viz, 'pdf_{}/s{}_h{}_a{}'\ ''.format(filename, self.s, self.h, self.a)) if not os.path.exists(pdf_folder): os.makedirs(pdf_folder) self.args.extend([ "integration.enable_residual_map=True", "integration.enable_residual_scatter=True", "integration.mosaic.enable_AD14F7B=True", "integration.graphics_backend=pdf", "integration.pdf_output_dir={}".format(pdf_folder) ]) if self.tag == 'integrate': args.append("indexing.completeness_pickle={}".format(self.out_img)) #Actually run integration using iota.bulletproof error_message = '' with misc.Capturing() as index_log: arguments = [ "distl.minimum_signal_height={}".format( str(self.s)), "distl.minimum_spot_height={}".format( str(self.h)), "distl.minimum_spot_area={}".format( str(self.a)), "indexing.open_wx_viewer=False" ] + list(args[1:]) tmppath = os.path.join(self.tmp_base, str(uuid.uuid4()) + ".pickle") assert not os.path.exists(tmppath) # invoke the indexer in a way that will protect iota from any crashes command = "iota.bulletproof {} {} {}" \ "".format(tmppath, self.target, " ".join(arguments)) try: easy_run.fully_buffered(command, join_stdout_stderr=True).show_stdout() if not os.path.exists(tmppath): print tmppath print command raise Exception("Indexing failed for an unknown reason") # iota.bulletproof saves the needed results from indexing in a tmp file result = easy_pickle.load(tmppath) os.remove(tmppath) if isinstance(result, str): raise Exception(result) else: int_final = result except Exception, e: int_final = None if hasattr(e, "classname"): print e.classname, "for %s:" % self.img, error_message = "{}: {}".format( e.classname, e[0].replace('\n', ' ')[:50]) else: print "Integration error for %s:" % self.img, error_message = "{}".format(str(e).replace('\n', ' ')[:50]) print e # Output results of integration (from the "info" object returned by # run_one_index_core) if int_final == None: if error_message != '': reason_for_failure = " - {}".format(error_message) else: reason_for_failure = '' int_status = 'not integrated' + reason_for_failure int_results = {'info': int_status} elif int_final['observations'][0] == None: int_status = 'no data recorded' int_results = {'info': int_status} else: try: obs = int_final['observations'][0] cell = obs.unit_cell().parameters() sg = int_final['pointgroup'] lres, hres = obs.d_max_min() # Calculate number of spots w/ high I / sigmaI Is = obs.data() sigmas = obs.sigmas() I_over_sigI = Is / sigmas #spots = len(Is) strong_spots = len( [i for i in I_over_sigI if i >= self.min_sigma]) # Mosaicity parameters mosaicity = round( (int_final.get('ML_half_mosaicity_deg', [0])[0]), 6) dom_size = int_final.get('ML_domain_size_ang', [0])[0] ewald_proximal_volume = int_final.get('ewald_proximal_volume', [0])[0] # Assemble output for log file and/or integration result file p_cell = "{:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}"\ "".format(cell[0], cell[1], cell[2], cell[3], cell[4], cell[5]) int_status = 'RES: {:<4.2f} NSREF: {:<4} SG: {:<5} CELL: {}'\ ''.format(hres, strong_spots, sg, p_cell) int_results = { 'sg': sg, 'a': cell[0], 'b': cell[1], 'c': cell[2], 'alpha': cell[3], 'beta': cell[4], 'gamma': cell[5], 'wavelength': int_final['wavelength'], 'distance': int_final['distance'], 'beamX': int_final['xbeam'], 'beamY': int_final['ybeam'], 'strong': strong_spots, 'res': hres, 'lres': lres, 'mos': mosaicity, 'epv': ewald_proximal_volume, 'info': int_status, 'ok': True } except ValueError: import traceback print print self.img raise Exception("".join( traceback.format_exception(*sys.exc_info()))) sys.exit() # write integration logfile if self.tag == 'integrate': misc.main_log(self.int_log, "{:-^100}\n{:-^100}\n{:-^100}\n"\ "".format("", " FINAL INTEGRATION: ", ""\ "S = {:>2}, H ={:>2}, A ={:>2} "\ "".format(self.s, self.h, self.a))) else: misc.main_log(self.int_log, "{:-^100}\n".format(" INTEGRATION: "\ "S = {:>2}, H ={:>2}, A ={:>2} "\ "".format(self.s, self.h, self.a))) for item in index_log: misc.main_log(self.int_log, item) misc.main_log(self.int_log, "\n[ {:^100} ]\n\n".format(int_status)) # In single-image mode, write a file with h, k, l, I, sigma if self.single_image == True and self.tag == 'integrate': hklI_filename = "{}.{}".format( os.path.basename(self.out_img).split('.')[0], 'hkli') hklI_file = os.path.join(os.path.dirname(self.out_img), hklI_filename) hklI = zip(obs.indices(), obs.data(), obs.sigmas()) for i in hklI: with open(hklI_file, 'a') as f: entry = '{},{},{},{},{}'.format(i[0][0], i[0][1], i[0][2], i[1], i[2]) f.write('{}\n'.format(entry)) return int_results
def print_summary(self, write_files=True): """ Prints summary and appends to general log file. Also outputs some of it on stdout. Also writes out output list files. """ summary = [] misc.main_log(self.logfile, "\n\n{:-^80}\n".format('SUMMARY'), (not self.gui_mode)) summary.append('raw images read in: {}'\ ''.format(len(self.all_objects))) summary.append('raw images with no diffraction: {}'\ ''.format(len(self.no_diff_objects))) summary.append('raw images with diffraction: {}'\ ''.format(len(self.diff_objects))) if self.params.advanced.integrate_with == 'cctbx': summary.append('failed indexing / integration: {}'\ ''.format(len(self.not_int_objects))) summary.append('failed prefilter: {}'\ ''.format(len(self.filter_fail_objects))) elif self.params.advanced.integrate_with == 'dials': summary.append('failed spotfinding: {}'\ ''.format(len(self.not_spf_objects))) summary.append('failed indexing: {}'\ ''.format(len(self.not_idx_objects))) summary.append('failed integration: {}'\ ''.format(len(self.not_int_objects))) summary.append('final integrated pickles: {}'\ ''.format(len(self.sorted_final_images))) for item in summary: misc.main_log(self.logfile, "{}".format(item), (not self.gui_mode)) misc.main_log(self.logfile, '\n\nIOTA version {0}'.format(self.ver)) misc.main_log(self.logfile, "{}\n".format(self.now)) # Write list files: if write_files: input_list_file = os.path.join(self.output_dir, 'input_images.lst') blank_images_file = os.path.join(self.output_dir, 'blank_images.lst') prefilter_fail_file = os.path.join(self.output_dir, 'failed_cctbx_prefilter.lst') spotfinding_fail_file = os.path.join(self.output_dir, 'failed_dials_spotfinding.lst') indexing_fail_file = os.path.join(self.output_dir, 'failed_dials_indexing.lst') not_integrated_file = os.path.join(self.output_dir, 'not_integrated.lst') integrated_file = os.path.join(self.output_dir, 'integrated.lst') int_images_file = os.path.join(self.output_dir, 'int_image_pickles.lst') if self.prime_data_path == None: self.prime_data_path = integrated_file if len(self.no_diff_objects) > 0: with open(blank_images_file, 'w') as bif: for obj in self.no_diff_objects: bif.write('{}\n'.format(obj.conv_img)) if len(self.diff_objects) > 0: with open(input_list_file, 'w') as ilf: for obj in self.diff_objects: ilf.write('{}\n'.format(obj.conv_img)) if len(self.not_int_objects) > 0: with open(not_integrated_file, 'w') as nif: for obj in self.not_int_objects: nif.write('{}\n'.format(obj.conv_img)) if self.params.advanced.integrate_with == 'cctbx' and len(self.filter_fail_objects) > 0: with open(prefilter_fail_file, 'w') as pff: for obj in self.filter_fail_objects: pff.write('{}\n'.format(obj.conv_img)) if self.params.advanced.integrate_with == 'dials': if len(self.not_spf_objects) > 0: with open(spotfinding_fail_file, 'w') as sff: for obj in self.not_spf_objects: sff.write('{}\n'.format(obj.conv_img)) if len(self.not_idx_objects) > 0: with open(indexing_fail_file, 'w') as iff: for obj in self.not_idx_objects: iff.write('{}\n'.format(obj.conv_img)) if len(self.final_objects) > 0: with open(integrated_file, 'w') as intf: for obj in self.sorted_final_images: intf.write('{}\n'.format(obj.final['final'])) with open(int_images_file, 'w') as ipf: for obj in self.sorted_final_images: ipf.write('{}\n'.format(obj.final['img']))
def unit_cell_analysis(self, write_files=True): """ Calls unit cell analysis module, which uses hierarchical clustering (Zeldin, et al, Acta D, 2015) to split integration results according to detected morphological groupings (if any). Most useful with preliminary integration without target unit cell specified. """ # Will not run clustering if only one integration result found or if turned off if len(self.final_objects) == 1: unit_cell = (self.final_objects[0].final['a'], self.final_objects[0].final['b'], self.final_objects[0].final['c'], self.final_objects[0].final['alpha'], self.final_objects[0].final['beta'], self.final_objects[0].final['gamma']) point_group = self.final_objects[0].final['sg'] misc.main_log(self.logfile, "\n\n{:-^80}\n".format(' UNIT CELL ANALYSIS '), True) uc_line = "{:<6} {:^4}: {:<6.2f}, {:<6.2f}, {:<6.2f}, {:<6.2f}, "\ "{:<6.2f}, {:<6.2f}".format('(1)', point_group, unit_cell[0], unit_cell[1], unit_cell[2], unit_cell[3], unit_cell[4], unit_cell[5]) misc.main_log(self.logfile, uc_line, True) self.cons_pg = point_group self.cons_uc = unit_cell else: uc_table = [] uc_summary = [] # if self.params.analysis.run_clustering: # # run hierarchical clustering analysis # from xfel.clustering.cluster import Cluster # # counter = 0 # ucs = Cluster.from_files(self.pickles, use_b=True) # clusters, _ = ucs.ab_cluster(self.params.analysis.cluster_threshold, # log=False, write_file_lists=False, # schnell=False, doplot=False) # uc_table.append("\n\n{:-^80}\n"\ # "".format(' UNIT CELL ANALYSIS ')) # # # extract clustering info and add to summary output list # for cluster in clusters: # sorted_pg_comp = sorted(cluster.pg_composition.items(), # key=lambda x: -1 * x[1]) # pg_nums = [pg[1] for pg in sorted_pg_comp] # cons_pg = sorted_pg_comp[np.argmax(pg_nums)] # # # write out lists of output pickles that comprise clusters with > 1 members # if len(cluster.members) > 1: # counter += 1 # # # Sort clustered images by mosaicity, lowest to highest # cluster_filenames = [j.path for j in cluster.members] # clustered_objects = [i for i in self.final_objects if \ # i.final['final'] in cluster_filenames] # sorted_cluster = sorted(clustered_objects, # key=lambda i: i.final['mos']) # # Write to file # if write_files: # output_file = os.path.join(self.output_dir, "uc_cluster_{}.lst".format(counter)) # for obj in sorted_cluster: # with open(output_file, 'a') as scf: # scf.write('{}\n'.format(obj.final['final'])) # # mark_output = os.path.basename(output_file) # else: # mark_output = '*' # output_file = None # else: # mark_output = '' # output_file = None # # # format and record output # uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\ # "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\ # "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) "\ # "{}".format('({})'.format(len(cluster.members)), cons_pg[0], # cluster.medians[0], cluster.stdevs[0], # cluster.medians[1], cluster.stdevs[1], # cluster.medians[2], cluster.stdevs[2], # cluster.medians[3], cluster.stdevs[3], # cluster.medians[4], cluster.stdevs[4], # cluster.medians[5], cluster.stdevs[5], # mark_output) # uc_table.append(uc_line) # uc_info = [len(cluster.members), cons_pg[0], cluster.medians, # output_file, uc_line] # uc_summary.append(uc_info) # else: # generate placeholder average unit cell - TEMPORARY, while we figure out how # to eliminate scipy from cluster module uc_table.append("\n\n{:-^80}\n" \ "".format(' UNIT CELL AVERAGING (no clustering) ')) uc_a = [i.final['a'] for i in self.final_objects] uc_b = [i.final['b'] for i in self.final_objects] uc_c = [i.final['c'] for i in self.final_objects] uc_alpha = [i.final['alpha'] for i in self.final_objects] uc_beta = [i.final['beta'] for i in self.final_objects] uc_gamma = [i.final['gamma'] for i in self.final_objects] uc_sg = [i.final['sg'] for i in self.final_objects] cons_pg = Counter(uc_sg).most_common(1)[0][0] uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \ "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \ "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) " \ "{}".format('({})'.format(len(self.final_objects)), cons_pg, np.median(uc_a), np.std(uc_a), np.median(uc_b), np.std(uc_b), np.median(uc_c), np.std(uc_c), np.median(uc_alpha), np.std(uc_alpha), np.median(uc_beta), np.std(uc_beta), np.median(uc_gamma), np.std(uc_gamma), '') unit_cell = (np.median(uc_a), np.median(uc_b), np.median(uc_c), np.median(uc_alpha), np.median(uc_beta), np.median(uc_gamma)) uc_table.append(uc_line) uc_info = [len(self.final_objects), cons_pg, unit_cell, None, uc_line] uc_summary.append(uc_info) uc_table.append('\nMost common unit cell:\n') # select the most prevalent unit cell (most members in cluster) uc_freqs = [i[0] for i in uc_summary] uc_pick = uc_summary[np.argmax(uc_freqs)] uc_table.append(uc_pick[4]) self.cons_pg = uc_pick[1] self.cons_uc = uc_pick[2] if uc_pick[3] != None: self.prime_data_path = uc_pick[3] for item in uc_table: misc.main_log(self.logfile, item, (not self.gui_mode)) if self.gui_mode: return self.cons_pg, self.cons_uc
def run(self): self.args, self.phil_args = parse_command_args(self.iver, self.help_message).parse_known_args() # Check for type of input if len(self.args.path) == 0 or self.args.path is None: # No input parse_command_args(self.iver, self.help_message).print_help() if self.args.default: # Write out default params and exit help_out, txt_out = inp.print_params() print '\n{:-^70}\n'.format('IOTA Parameters') print help_out inp.write_defaults(os.path.abspath(os.path.curdir), txt_out) misc.iota_exit() elif len(self.args.path) > 1: # If multiple paths / wildcards file_list = ginp.make_input_list(self.args.path) list_file = os.path.join(os.path.abspath(os.path.curdir), 'input.lst') with open(list_file, 'w') as lf: lf.write('\n'.join(file_list)) msg = "\nIOTA will run in AUTO mode using wildcard datapath:\n" \ "{} files found, compiled in {}\n".format(len(file_list), list_file) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, list_file, 'auto', self.now) else: # If single path, check type carg = os.path.abspath(self.args.path[0]) if os.path.exists(carg): # If user provided a parameter file if os.path.isfile(carg) and os.path.basename(carg).endswith('.param'): msg = '' self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'file') # If user provided a list of input files elif os.path.isfile(carg) and os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a single filepath elif os.path.isfile(carg) and not os.path.basename(carg).endswith('.lst'): msg = "\nIOTA will run in SINGLE-FILE mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided a data folder elif os.path.isdir(carg): msg = "\nIOTA will run in AUTO mode using {}:\n".format(carg) self.params, self.txt_out = inp.process_input(self.args, self.phil_args, carg, 'auto', self.now) # If user provided gibberish else: print self.logo print "ERROR: Invalid input! Need parameter filename or data folder." misc.iota_exit() # Identify indexing / integration program if self.params.advanced.integrate_with == 'cctbx': prg = " with CCTBX.XFEL\n" elif self.params.advanced.integrate_with == 'dials': prg = " with DIALS\n" self.logo += prg print self.logo print '\n{}\n'.format(self.now) if msg != '': print msg if self.args.analyze != None: print 'ANALYSIS ONLY will be performed (analyzing run #{})'.format( self.args.analyze) self.analyze_prior_results('{:003d}'.format(int(self.args.analyze))) misc.iota_exit() if self.params.mp_method == 'mpi': rank, size = misc.get_mpi_rank_and_size() self.master_process = rank == 0 else: self.master_process = True # Call function to read input folder structure (or input file) and # generate list of image file paths if self.params.cctbx.selection.select_only.flag_on: self.gs_img_objects = self.make_int_object_list() self.input_list = [i.conv_img for i in self.gs_img_objects] else: self.input_list = self.make_input_list() # Check for -l option, output list of input files and exit if self.args.list: list_file = os.path.abspath("{}/input.lst".format(os.curdir)) # Check if other files of this name exist under the current folder list_folder = os.path.dirname(list_file) list_files = [i for i in os.listdir(list_folder) if i.endswith(".lst")] if len(list_files) > 0: list_file = os.path.join(list_folder, "input_{}.lst".format(len(list_files))) print '\nINPUT LIST ONLY option selected' print 'Input list in {} \n\n'.format(list_file) with open(list_file, "w") as lf: for i, input_file in enumerate(self.input_list, 1): lf.write('{}\n'.format(input_file)) print "{}: {}".format(i, input_file) lf.write('{}\n'.format(input_file)) print '\nExiting...\n\n' misc.iota_exit() # If fewer images than requested processors are supplied, set the number of # processors to the number of images if self.params.n_processors > len(self.input_list): self.params.n_processors = len(self.input_list) # Generate base folder paths self.conv_base = misc.set_base_dir('converted_pickles', out_dir = self.params.output) self.int_base = misc.set_base_dir('integration', out_dir = self.params.output) self.obj_base = os.path.join(self.int_base, 'image_objects') self.fin_base = os.path.join(self.int_base, 'final') self.log_base = os.path.join(self.int_base, 'logs') self.viz_base = os.path.join(self.int_base, 'visualization') self.tmp_base = os.path.join('/tmp', '{}_{}'.format(os.getlogin(), time.time())) # Generate base folders os.makedirs(self.int_base) os.makedirs(self.obj_base) os.makedirs(self.fin_base) os.makedirs(self.log_base) os.makedirs(self.tmp_base) # Determine input base self.input_base = os.path.abspath(os.path.dirname(os.path.commonprefix(self.input_list))) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Log starting info misc.main_log(self.logfile, '{:=^80} \n'.format(' IOTA MAIN LOG ')) misc.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) misc.main_log(self.logfile, self.txt_out) if self.params.advanced.integrate_with == 'cctbx': target_file = self.params.cctbx.target elif self.params.advanced.integrate_with == 'dials': target_file = self.params.dials.target misc.main_log(self.logfile, '{:-^80} \n\n' ''.format(' TARGET FILE ({}) CONTENTS ' ''.format(target_file))) with open(target_file, 'r') as phil_file: phil_file_contents = phil_file.read() misc.main_log(self.logfile, phil_file_contents)
def unit_cell_analysis(self): """ Calls unit cell analysis module, which uses hierarchical clustering (Zeldin, et al, Acta D, 2015) to split integration results according to detected morphological groupings (if any). Most useful with preliminary integration without target unit cell specified. """ # Will not run clustering if only one integration result found or if turned off if self.final_objects is None: self.cons_uc = None self.cons_pg = None misc.main_log(self.logfile, "\n\n{:-^80}\n".format(' UNIT CELL ANALYSIS '), True) misc.main_log(self.logfile, '\n UNIT CELL CANNOT BE DETERMINED!', True) elif len(self.final_objects) == 1: unit_cell = (self.final_objects[0].final['a'], self.final_objects[0].final['b'], self.final_objects[0].final['c'], self.final_objects[0].final['alpha'], self.final_objects[0].final['beta'], self.final_objects[0].final['gamma']) point_group = self.final_objects[0].final['sg'] misc.main_log(self.logfile, "\n\n{:-^80}\n".format(' UNIT CELL ANALYSIS '), True) uc_line = "{:<6} {:^4}: {:<6.2f}, {:<6.2f}, {:<6.2f}, {:<6.2f}, "\ "{:<6.2f}, {:<6.2f}".format('(1)', point_group, unit_cell[0], unit_cell[1], unit_cell[2], unit_cell[3], unit_cell[4], unit_cell[5]) misc.main_log(self.logfile, uc_line, True) self.cons_pg = point_group self.cons_uc = unit_cell else: uc_table = [] uc_summary = [] if self.params.analysis.run_clustering: # run hierarchical clustering analysis from xfel.clustering.cluster import Cluster counter = 0 threshold = self.params.analysis.cluster_threshold cluster_limit = self.params.analysis.cluster_limit if self.params.analysis.cluster_n_images > 0: n_images = self.params.analysis.cluster_n_images else: n_images = len(self.final_objects) obj_list = [] if n_images < len(self.final_objects): import random for i in range(n_images): random_number = random.randrange( 0, len(self.final_objects)) if self.final_objects[random_number] in obj_list: while self.final_objects[ random_number] in obj_list: random_number = random.randrange( 0, len(self.final_objects)) obj_list.append(self.final_objects[random_number]) else: obj_list.append(self.final_objects[random_number]) if obj_list == []: obj_list = self.final_objects # Cluster from iterable (this doesn't keep filenames - bad!) # with Capturing() as suppressed_output: # uc_iterable = [] # for obj in obj_list: # unit_cell = (float(obj.final['a']), # float(obj.final['b']), # float(obj.final['c']), # float(obj.final['alpha']), # float(obj.final['beta']), # float(obj.final['gamma']), # obj.final['sg']) # uc_iterable.append(unit_cell) # ucs = Cluster.from_iterable(iterable=uc_iterable) # Cluster from files (slow, but will keep for now) ucs = Cluster.from_files(pickle_list=self.pickles) # Do clustering clusters, _ = ucs.ab_cluster(threshold=threshold, log=False, write_file_lists=False, schnell=False, doplot=False) uc_table.append("\n\n{:-^80}\n"\ "".format(' UNIT CELL ANALYSIS ')) # extract clustering info and add to summary output list if cluster_limit is None: if len(self.pickles) / 10 >= 10: cluster_limit = 10 else: cluster_limit = len(self.pickles) / 10 for cluster in clusters: sorted_pg_comp = sorted(cluster.pg_composition.items(), key=lambda x: -1 * x[1]) pg_nums = [pg[1] for pg in sorted_pg_comp] cons_pg = sorted_pg_comp[np.argmax(pg_nums)] if len(cluster.members) > cluster_limit: counter += 1 # Sort clustered images by mosaicity, lowest to highest cluster_filenames = [j.path for j in cluster.members] clustered_objects = [i for i in self.final_objects if \ i.final['final'] in cluster_filenames] sorted_cluster = sorted(clustered_objects, key=lambda i: i.final['mos']) # Write to file if self.params.analysis.cluster_write_files: output_file = os.path.join( self.output_dir, "uc_cluster_{}.lst".format(counter)) for obj in sorted_cluster: with open(output_file, 'a') as scf: scf.write('{}\n'.format( obj.final['final'])) mark_output = os.path.basename(output_file) else: mark_output = '*' output_file = None # Populate clustering info for GUI display uc_no_stdev = "{:<6.2f} {:<6.2f} {:<6.2f} " \ "{:<6.2f} {:<6.2f} {:<6.2f} " \ "".format(cluster.medians[0], cluster.medians[1], cluster.medians[2], cluster.medians[3], cluster.medians[4], cluster.medians[5]) cluster_info = { 'number': len(cluster.members), 'pg': cons_pg[0], 'uc': uc_no_stdev, 'filename': mark_output } self.clusters.append(cluster_info) else: mark_output = '' output_file = None # format and record output uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\ "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\ "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) "\ "{}".format('({})'.format(len(cluster.members)), cons_pg[0], cluster.medians[0], cluster.stdevs[0], cluster.medians[1], cluster.stdevs[1], cluster.medians[2], cluster.stdevs[2], cluster.medians[3], cluster.stdevs[3], cluster.medians[4], cluster.stdevs[4], cluster.medians[5], cluster.stdevs[5], mark_output) uc_table.append(uc_line) lattices = ', '.join( ['{} ({})'.format(i[0], i[1]) for i in sorted_pg_comp]) uc_info = [ len(cluster.members), cons_pg[0], cluster.medians, output_file, uc_line, lattices ] uc_summary.append(uc_info) else: # generate average unit cell uc_table.append("\n\n{:-^80}\n" \ "".format(' UNIT CELL AVERAGING (no clustering) ')) uc_a = [i.final['a'] for i in self.final_objects] uc_b = [i.final['b'] for i in self.final_objects] uc_c = [i.final['c'] for i in self.final_objects] uc_alpha = [i.final['alpha'] for i in self.final_objects] uc_beta = [i.final['beta'] for i in self.final_objects] uc_gamma = [i.final['gamma'] for i in self.final_objects] uc_sg = [i.final['sg'] for i in self.final_objects] cons_pg = Counter(uc_sg).most_common(1)[0][0] all_pgs = Counter(uc_sg).most_common() uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \ "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \ "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) " \ "{}".format('({})'.format(len(self.final_objects)), cons_pg, np.median(uc_a), np.std(uc_a), np.median(uc_b), np.std(uc_b), np.median(uc_c), np.std(uc_c), np.median(uc_alpha), np.std(uc_alpha), np.median(uc_beta), np.std(uc_beta), np.median(uc_gamma), np.std(uc_gamma), '') unit_cell = (np.median(uc_a), np.median(uc_b), np.median(uc_c), np.median(uc_alpha), np.median(uc_beta), np.median(uc_gamma)) uc_table.append(uc_line) lattices = ', '.join( ['{} ({})'.format(i[0], i[1]) for i in all_pgs]) uc_info = [ len(self.final_objects), cons_pg, unit_cell, None, uc_line, lattices ] uc_summary.append(uc_info) uc_table.append('\nMost common unit cell:\n') # select the most prevalent unit cell (most members in cluster) uc_freqs = [i[0] for i in uc_summary] uc_pick = uc_summary[np.argmax(uc_freqs)] uc_table.append(uc_pick[4]) uc_table.append('\nBravais Lattices in Biggest Cluster: {}' ''.format(uc_pick[5])) self.cons_pg = uc_pick[1] self.cons_uc = uc_pick[2] if uc_pick[3] != None: self.prime_data_path = uc_pick[3] for item in uc_table: misc.main_log(self.logfile, item, (not self.gui_mode)) self.analysis_result.__setattr__('clusters', self.clusters) self.analysis_result.__setattr__('cons_pg', self.cons_pg) self.analysis_result.__setattr__('cons_uc', self.cons_uc) if self.gui_mode: return self.cons_pg, self.cons_uc, self.clusters