def print_results(self, final_table=None): """ Prints diagnostics from the final integration run. """ assert self.info if not final_table: final_table = ["\n\n{:-^80}\n".format('ANALYSIS OF RESULTS')] if not self.info.categories['integrated']: final_table.append('NO IMAGES INTEGRATED!') else: label_lens = [len(v['label']) for k, v in self.info.stats.items()] max_label = int(5 * round(float(np.max(label_lens)) / 5)) + 5 for k, v in self.info.stats.items(): if k in ('lres', 'res', 'beamX', 'beamY'): continue line = '{: <{l}}: max = {:<6.2f} min = {:<6.2f} ' \ 'avg = {:<6.2f} ({:<6.2f})' \ ''.format(v['label'], v['max'], v['min'], v['mean'], v['std'], l=max_label) final_table.append(line) # TODO: Figure out what to do with summary charts # # If more than one integrated image, plot various summary graphs # if len(self.info.categories['integrated']) > 1: # plot = Plotter(self.params, self.info) # if self.params.analysis.summary_graphs: # if ( self.params.advanced.processing_backend == 'ha14' and # self.params.cctbx_ha14.grid_search.type is not None # ): # plot.plot_spotfinding_heatmap(write_files=True) # plot.plot_res_histogram(write_files=True) # med_beamX, med_beamY, pixel_size = plot.plot_beam_xy(write_files=True, # return_values=True) # else: # with warnings.catch_warnings(): # # To catch any 'mean of empty slice' runtime warnings # warnings.simplefilter("ignore", category=RuntimeWarning) # beamXY_info = plot.calculate_beam_xy() # beamX, beamY = beamXY_info[:2] # med_beamX = np.median(beamX) # med_beamY = np.median(beamY) # pixel_size = beamXY_info[-1] final_table.append("{: <{l}}: X = {:<4.2f}, Y = {:<4.2f}" "".format('Median Beam Center', self.info.stats['beamX']['mean'], self.info.stats['beamY']['mean'], l=max_label)) # Special entry for resolution last v = self.info.stats['res'] final_table.append('{: <{l}}: low = {:<6.2f} high = {:<6.2f} ' \ 'avg = {:<6.2f} ({:<6.2f})' \ ''.format(v['label'], v['max'], v['min'], v['mean'], v['std'], l=max_label)) for item in final_table: util.main_log(self.info.logfile, item, False) self.info.update(final_table=final_table)
def process(self): """Run importer and/or processor.""" start_time = time.time() # Process images with prog_message( "Processing {} images".format(len(self.info.unprocessed)), prog="PROCESSING", out_type=self.out_type, ): if self.out_type == "progress": self.prog_count = 0 self.gs_prog = cmd.ProgressBar(title="PROCESSING") img_objects = self.run_process(iterable=self.info.unprocessed) proc_time = datetime.timedelta(seconds=int(time.time() - start_time)) hours, minutes, seconds = str(proc_time).split(":") main_log( self.info.logfile, "Total processing time: {} hours, {} minutes, {} seconds" "".format(hours, minutes, seconds), print_tag=False, ) # Run analysis if not in GUI mode if not "gui" in self.out_type: with prog_message( "Analyzing results", prog="ANALYSIS", out_type=self.out_type ): self.info.finished_objects = [o.obj_file for o in img_objects] self.run_analysis() if "silent" not in self.out_type: if self.info.have_results: print("\n".join(self.info.final_table)) print("\n".join(self.info.uc_table)) print("\n".join(self.info.summary)) else: print("\n **** NO IMAGES INTEGRATED! ****") # Determine total runtime if not "gui" in self.out_type: runtime = datetime.timedelta(seconds=int(time.time() - start_time)) hours, minutes, seconds = str(runtime).split(":") main_log( self.info.logfile, "Total run time: {} hours, {} minutes, {} seconds" "".format(hours, minutes, seconds), print_tag=True, )
def import_and_process(self, input_entry): img_object = self.importer.run(input_entry) if img_object.status == 'imported': img_object = self.integrator.run(img_object) # Update main log if hasattr(self.info, 'logfile'): main_log_entry = "\n".join(img_object.log_info) util.main_log(self.info.logfile, main_log_entry) util.main_log(self.info.logfile, '\n{:-^100}\n'.format('')) # Set status to final img_object.status = 'final' return img_object
def print_summary(self, write_files=True): """Prints summary and appends to general log file. Also outputs some of it on stdout. Also writes out output list files. """ assert self.info if not self.info.categories["integrated"]: util.main_log( self.info.logfile, "NO IMAGES SUCCESSFULLY PROCESSSED!", (not self.gui_mode), ) return summary = [] summary.append("\n\n{:-^80}\n".format("SUMMARY")) categories = [ "total", "failed_triage", "have_diffraction", "failed_spotfinding", "failed_indexing", "failed_grid_search", "failed_integration", "failed_filter", "integrated", ] for cat in categories: lst, fail, fn, _ = self.info.categories[cat] path = os.path.join(self.info.int_base, fn) if len(lst) > 0 or cat in ("integrated", "diffraction"): summary.append("{: <20}: {}".format("{} ".format(fail), len(lst))) with open(path, "w") as cf: for item in lst: if isinstance(item, tuple) or isinstance(item, list): item = ", ".join([str(i) for i in item]) cf.write("{}\n".format(item)) if cat == "integrated" and write_files: if not hasattr(self, "prime_data_path"): self.prime_data_path = path summary.append("\n\nIOTA version {0}".format(iota_version)) summary.append("{}\n".format(now)) for item in summary: util.main_log(self.info.logfile, "{}".format(item), False) self.info.update(summary=summary)
def import_and_process(self, input_entry): img_object = self.importer.run(input_entry) if img_object.status == "imported": with util.Capturing() as junk_output: img_object = self.integrator.run(img_object) # Update main log if hasattr(self.info, "logfile"): main_log_entry = "\n".join(img_object.log_info) util.main_log(self.info.logfile, main_log_entry) util.main_log(self.info.logfile, "\n{:-^100}\n".format("")) # Set status to final img_object.status = "final" return img_object
def process(self): """ Run importer and/or processor """ start_time = time.time() # Process images with prog_message('Processing {} images'.format( len(self.info.unprocessed)), prog='PROCESSING', out_type=self.out_type): if self.out_type == 'progress': self.prog_count = 0 self.gs_prog = cmd.ProgressBar(title='PROCESSING') img_objects = self.run_process(iterable=self.info.unprocessed) proc_time = datetime.timedelta(seconds=int(time.time() - start_time)) hours, minutes, seconds = str(proc_time).split(':') util.main_log( self.info.logfile, "Total processing time: {} hours, {} minutes, {} seconds" "".format(hours, minutes, seconds), print_tag=False) # Run analysis if not in GUI mode if not 'gui' in self.out_type: with prog_message('Analyzing results', prog='ANALYSIS', out_type=self.out_type): self.info.finished_objects = [o.obj_file for o in img_objects] self.run_analysis() if 'silent' not in self.out_type: if self.info.have_results: print('\n'.join(self.info.final_table)) print('\n'.join(self.info.uc_table)) print('\n'.join(self.info.summary)) else: print('\n **** NO IMAGES INTEGRATED! ****') # Determine total runtime if not 'gui' in self.out_type: runtime = datetime.timedelta(seconds=int(time.time() - start_time)) hours, minutes, seconds = str(runtime).split(':') util.main_log(self.info.logfile, "Total run time: {} hours, {} minutes, {} seconds" "".format(hours, minutes, seconds), print_tag=True)
def print_summary(self, write_files=True): """ Prints summary and appends to general log file. Also outputs some of it on stdout. Also writes out output list files. """ assert self.info if not self.info.categories['integrated']: util.main_log(self.info.logfile, "NO IMAGES SUCCESSFULLY PROCESSSED!", (not self.gui_mode)) return summary = [] summary.append("\n\n{:-^80}\n".format('SUMMARY')) categories = [ 'total', 'failed_triage', 'have_diffraction', 'failed_spotfinding', 'failed_indexing', 'failed_grid_search', 'failed_integration', 'failed_filter', 'integrated' ] for cat in categories: lst, fail, fn, _ = self.info.categories[cat] path = os.path.join(self.info.int_base, fn) if len(lst) > 0 or cat in ('integrated', 'diffraction'): summary.append('{: <20}: {}'.format('{} '.format(fail), len(lst))) with open(path, 'w') as cf: for item in lst: if isinstance(item, tuple) or isinstance(item, list): item = ', '.join([str(i) for i in item]) cf.write('{}\n'.format(item)) if cat == 'integrated' and write_files: if not hasattr(self, 'prime_data_path'): self.prime_data_path = path summary.append('\n\nIOTA version {0}'.format(iota_version)) summary.append("{}\n".format(now)) for item in summary: util.main_log(self.info.logfile, "{}".format(item), False) self.info.update(summary=summary)
def initialize_main_log(self): """ Initialize main log (iota.log) and record starting parameters :return: True if successful, False if fails """ try: # Generate text of params self.iota_phil_string = util.convert_phil_to_text(self.iota_phil) # Initialize main log self.logfile = os.path.abspath(os.path.join(self.int_base, 'iota.log')) # Log starting info util.main_log(self.logfile, '{:*^80} \n'.format(' IOTA MAIN LOG ')) util.main_log(self.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) util.main_log(self.logfile, self.iota_phil_string) # Log cctbx.xfel / DIALS settings util.main_log(self.logfile, '{:-^80} \n'.format('BACKEND SETTINGS')) util.main_log(self.logfile, self.target_phil) return True, 'IOTA_INIT: MAIN LOG INITIALIZED' except Exception as e: return False, 'IOTA_INIT_ERROR: MAIN LOG NOT INITIALIZED, {}'.format(e)
def unit_cell_analysis(self): """ Calls unit cell analysis module, which uses hierarchical clustering (Zeldin, et al, Acta D, 2015) to split integration results according to detected morphological groupings (if any). Most useful with preliminary integration without target unit cell specified. """ # Will not run clustering if only one integration result found or if turned off if not self.info.categories['integrated']: util.main_log(self.info.logfile, "\n\n{:-^80}\n".format(' UNIT CELL ANALYSIS '), True) util.main_log(self.info.logfile, '\n UNIT CELL CANNOT BE DETERMINED!', True) elif len(self.info.categories['integrated']) == 1: unit_cell = (self.info.cluster_iterable[0][:5]) point_group = self.info.cluster_iterable[0][6] util.main_log(self.info.logfile, "\n\n{:-^80}\n".format(' UNIT CELL ANALYSIS '), True) uc_line = "{:<6} {:^4}: {:<6.2f}, {:<6.2f}, {:<6.2f}, {:<6.2f}, " \ "{:<6.2f}, {:<6.2f}".format('(1)', point_group, unit_cell[0], unit_cell[1], unit_cell[2], unit_cell[3], unit_cell[4], unit_cell[5]) util.main_log(self.info.logfile, uc_line, True) self.info.best_pg = str(point_group) self.info.best_uc = unit_cell else: uc_table = [] uc_summary = [] if self.params.analysis.clustering.flag_on: # run hierarchical clustering analysis from xfel.clustering.cluster import Cluster counter = 0 self.info.clusters = [] threshold = self.params.analysis.clustering.threshold cluster_limit = self.params.analysis.clustering.limit final_pickles = self.info.categories['integrated'][0] pickles = [] if self.params.analysis.clustering.n_images > 0: import random for i in range( len(self.params.analysis.clustering.n_images)): random_number = random.randrange(0, len(final_pickles)) if final_pickles[random_number] in pickles: while final_pickles[random_number] in pickles: random_number = random.randrange( 0, len(final_pickles)) pickles.append(final_pickles[random_number]) else: pickles = final_pickles # Cluster from files (slow, but will keep for now) ucs = Cluster.from_files(pickle_list=pickles) # Do clustering clusters, _ = ucs.ab_cluster(threshold=threshold, log=False, write_file_lists=False, schnell=False, doplot=False) uc_table.append("\n\n{:-^80}\n" \ "".format(' UNIT CELL ANALYSIS ')) # extract clustering info and add to summary output list if cluster_limit is None: if len(pickles) / 10 >= 10: cluster_limit = 10 else: cluster_limit = len(pickles) / 10 for cluster in clusters: sorted_pg_comp = sorted(cluster.pg_composition.items(), key=lambda x: -1 * x[1]) pg_nums = [pg[1] for pg in sorted_pg_comp] cons_pg = sorted_pg_comp[np.argmax(pg_nums)] if len(cluster.members) > cluster_limit: counter += 1 # Write to file cluster_filenames = [j.path for j in cluster.members] if self.params.analysis.clustering.write_files: output_file = os.path.join( self.info.int_base, "uc_cluster_{}.lst".format(counter)) for fn in cluster_filenames: with open(output_file, 'a') as scf: scf.write('{}\n'.format(fn)) mark_output = os.path.basename(output_file) else: mark_output = '*' output_file = None else: mark_output = '' output_file = None # Populate clustering info for GUI display uc_init = uctbx.unit_cell(cluster.medians) symmetry = crystal.symmetry(unit_cell=uc_init, space_group_symbol='P1') groups = metric_subgroups(input_symmetry=symmetry, max_delta=3) top_group = groups.result_groups[0] best_sg = str(groups.lattice_group_info()).split('(')[0] best_uc = top_group['best_subsym'].unit_cell().parameters() # best_sg = str(top_group['best_subsym'].space_group_info()) uc_no_stdev = "{:<6.2f} {:<6.2f} {:<6.2f} " \ "{:<6.2f} {:<6.2f} {:<6.2f} " \ "".format(best_uc[0], best_uc[1], best_uc[2], best_uc[3], best_uc[4], best_uc[5]) cluster_info = { 'number': len(cluster.members), 'pg': best_sg, 'uc': uc_no_stdev, 'filename': mark_output } self.info.clusters.append(cluster_info) # format and record output # TODO: How to propagate stdevs after conversion from Niggli? # uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\ # "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), "\ # "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) "\ # "{}".format('({})'.format(len(cluster.members)), cons_pg[0], # cluster.medians[0], cluster.stdevs[0], # cluster.medians[1], cluster.stdevs[1], # cluster.medians[2], cluster.stdevs[2], # cluster.medians[3], cluster.stdevs[3], # cluster.medians[4], cluster.stdevs[4], # cluster.medians[5], cluster.stdevs[5], # mark_output) # uc_table.append(uc_line) uc_table.append("{:<6}: {} {}".format( len(cluster.members), uc_no_stdev, mark_output)) lattices = ', '.join( ['{} ({})'.format(i[0], i[1]) for i in sorted_pg_comp]) # uc_info = [len(cluster.members), cons_pg[0], cluster.medians, # output_file, uc_line, lattices] uc_info = [ len(cluster.members), best_sg, best_uc, output_file, uc_no_stdev, lattices ] uc_summary.append(uc_info) else: # generate average unit cell uc_table.append("\n\n{:-^80}\n" \ "".format(' UNIT CELL AVERAGING (no clustering) ')) uc_a, uc_b, uc_c, uc_alpha, \ uc_beta, uc_gamma, uc_sg = list(zip(*self.info.cluster_iterable)) cons_pg = Counter(uc_sg).most_common(1)[0][0] all_pgs = Counter(uc_sg).most_common() unit_cell = (np.median(uc_a), np.median(uc_b), np.median(uc_c), np.median(uc_alpha), np.median(uc_beta), np.median(uc_gamma)) # Populate clustering info for GUI display uc_init = uctbx.unit_cell(unit_cell) symmetry = crystal.symmetry(unit_cell=uc_init, space_group_symbol='P1') groups = metric_subgroups(input_symmetry=symmetry, max_delta=3) top_group = groups.result_groups[0] best_sg = str(groups.lattice_group_info()).split('(')[0] best_uc = top_group['best_subsym'].unit_cell().parameters() # best_sg = str(top_group['best_subsym'].space_group_info()) uc_no_stdev = "{:<6.2f} {:<6.2f} {:<6.2f} " \ "{:<6.2f} {:<6.2f} {:<6.2f} " \ "".format(best_uc[0], best_uc[1], best_uc[2], best_uc[3], best_uc[4], best_uc[5]) cluster_info = { 'number': len(self.info.cluster_iterable), 'pg': best_sg, 'uc': uc_no_stdev, 'filename': None } self.info.clusters.append(cluster_info) # uc_line = "{:<6} {:^4}: {:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \ # "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}), " \ # "{:<6.2f} ({:>5.2f}), {:<6.2f} ({:>5.2f}) " \ # "{}".format('({})'.format(len(self.final_objects)), cons_pg, # np.median(uc_a), np.std(uc_a), # np.median(uc_b), np.std(uc_b), # np.median(uc_c), np.std(uc_c), # np.median(uc_alpha), np.std(uc_alpha), # np.median(uc_beta), np.std(uc_beta), # np.median(uc_gamma), np.std(uc_gamma), '') # # uc_table.append(uc_line) uc_table.append(uc_no_stdev) lattices = ', '.join( ['{} ({})'.format(i[0], i[1]) for i in all_pgs]) # uc_info = [len(self.final_objects), cons_pg, unit_cell, None, # uc_line, lattices] uc_info = [ len(self.info.cluster_iterable), best_sg, best_uc, None, uc_no_stdev, lattices ] uc_summary.append(uc_info) uc_table.append('\nMost common unit cell:\n') # select the most prevalent unit cell (most members in cluster) uc_freqs = [i[0] for i in uc_summary] uc_pick = uc_summary[np.argmax(uc_freqs)] uc_table.append(uc_pick[4]) uc_table.append('\nBravais Lattices in Biggest Cluster: {}' ''.format(uc_pick[5])) self.info.best_pg = str(uc_pick[1]) self.info.best_uc = uc_pick[2] if uc_pick[3] is not None: self.prime_data_path = uc_pick[3] for item in uc_table: util.main_log(self.info.logfile, item, False) self.info.update(uc_table=uc_table) if self.gui_mode: return self.info.clusters
def initialize_processing(paramfile, run_no): ''' Initialize processing for a set of images :param paramfile: text file with IOTA parameters :param run_no: number of the processing run :return: info: INFO object params: IOTA params ''' try: phil, _ = inp.get_input_phil(paramfile=paramfile) except Exception as e: msg = 'IOTA_PROC_ERROR: Cannot import IOTA parameters! {}'.format(e) return False, msg else: params = phil.extract() # Reconstruct integration base path and get info object int_base = os.path.join(params.output, 'integration/{:03d}'.format(run_no)) try: info_file = os.path.join(int_base, 'proc.info') info = ProcInfo.from_json(filepath=info_file) except Exception as e: msg = 'IOTA_PROC_ERROR: Cannot import INFO object! {}'.format(e) return False, msg # Generate input list and input base if not hasattr(info, 'input_list'): info.generate_input_list(params=params) common_pfx = os.path.abspath( os.path.dirname(os.path.commonprefix(info.input_list))) input_base = common_pfx if os.path.isdir(os.path.abspath(params.input[0])): new_common_pfx = os.path.commonprefix([os.path.abspath(params.input[0]), common_pfx]) if new_common_pfx not in ('', '.'): input_base = new_common_pfx # Generate subfolder paths paths = dict(obj_base=os.path.join(int_base, 'image_objects'), fin_base=os.path.join(int_base, 'final'), log_base=os.path.join(int_base, 'logs'), viz_base=os.path.join(int_base, 'visualization'), tmp_base=os.path.join(int_base, 'tmp'), input_base=input_base) # FIXME: ordering of items in dictionaries changes depending on python version for bkey, bvalue in paths.items(): if bkey == "input_base": # I think this is what the original code wanted to do continue if not os.path.isdir(bvalue): os.makedirs(bvalue) info.update(paths) # Generate filepaths for various info files info_files = dict( obj_list_file=os.path.join(info.tmp_base, 'finished_objects.lst'), idx_file=os.path.join(info.int_base, 'observations.pickle') ) info.update(info_files) # Initialize stat containers info = generate_stat_containers(info=info, params=params) # Initialize main log util.main_log(info.logfile, '{:*^80} \n'.format(' IOTA MAIN LOG ')) util.main_log(info.logfile, '{:-^80} \n'.format(' SETTINGS FOR THIS RUN ')) util.main_log(info.logfile, info.iota_phil) util.main_log(info.logfile, '{:-^80} \n'.format('BACKEND SETTINGS')) util.main_log(info.logfile, info.target_phil) info.export_json() return info, params
def print_summary(self, write_files=True): """ Prints summary and appends to general log file. Also outputs some of it on stdout. Also writes out output list files. """ summary = [] util.main_log(self.logfile, "\n\n{:-^80}\n".format('SUMMARY'), (not self.gui_mode)) summary.append('raw images read in: {}'\ ''.format(len(self.all_objects))) summary.append('raw images with no diffraction: {}'\ ''.format(len(self.no_diff_objects))) summary.append('raw images with diffraction: {}'\ ''.format(len(self.diff_objects))) if self.params.advanced.processing_backend == 'ha14': summary.append('failed indexing / integration: {}'\ ''.format(len(self.not_int_objects))) summary.append('failed prefilter: {}'\ ''.format(len(self.filter_fail_objects))) elif self.params.advanced.processing_backend == 'cctbx.xfel': summary.append('failed spotfinding: {}'\ ''.format(len(self.not_spf_objects))) summary.append('failed indexing: {}'\ ''.format(len(self.not_idx_objects))) summary.append('failed filter: {}' ''.format(len(self.filter_fail_objects))) summary.append('failed integration: {}'\ ''.format(len(self.not_int_objects))) summary.append('final integrated pickles: {}'\ ''.format(len(self.sorted_final_images))) for item in summary: util.main_log(self.logfile, "{}".format(item), (not self.gui_mode)) util.main_log(self.logfile, '\n\nIOTA version {0}'.format(self.ver)) util.main_log(self.logfile, "{}\n".format(self.now)) # Write list files: if write_files: input_list_file = os.path.join(self.output_dir, 'input_images.lst') blank_images_file = os.path.join(self.output_dir, 'blank_images.lst') prefilter_fail_file = os.path.join(self.output_dir, 'failed_cctbx_prefilter.lst') spotfinding_fail_file = os.path.join(self.output_dir, 'failed_dials_spotfinding.lst') indexing_fail_file = os.path.join(self.output_dir, 'failed_dials_indexing.lst') not_integrated_file = os.path.join(self.output_dir, 'not_integrated.lst') integrated_file = os.path.join(self.output_dir, 'integrated.lst') int_images_file = os.path.join(self.output_dir, 'int_image_pickles.lst') analyzer_file = os.path.join(self.output_dir, 'analysis.pickle') if self.prime_data_path is None: self.prime_data_path = integrated_file if len(self.no_diff_objects) > 0: with open(blank_images_file, 'w') as bif: for obj in self.no_diff_objects: bif.write('{}\n'.format(obj.img_path)) if len(self.diff_objects) > 0: with open(input_list_file, 'w') as ilf: for obj in self.diff_objects: ilf.write('{}\n'.format(obj.img_path)) if len(self.not_int_objects) > 0: with open(not_integrated_file, 'w') as nif: for obj in self.not_int_objects: nif.write('{}\n'.format(obj.img_path)) if len(self.filter_fail_objects) > 0: with open(prefilter_fail_file, 'w') as pff: for obj in self.filter_fail_objects: pff.write('{}\n'.format(obj.img_path)) if self.params.advanced.processing_backend == 'cctbx.xfel': if len(self.not_spf_objects) > 0: with open(spotfinding_fail_file, 'w') as sff: for obj in self.not_spf_objects: sff.write('{}\n'.format(obj.img_path)) if len(self.not_idx_objects) > 0: with open(indexing_fail_file, 'w') as iff: for obj in self.not_idx_objects: iff.write('{}\n'.format(obj.img_path)) if len(self.final_objects) > 0: with open(integrated_file, 'w') as intf: for obj in self.sorted_final_images: intf.write('{}\n'.format(obj.final['final'])) with open(int_images_file, 'w') as ipf: for obj in self.sorted_final_images: ipf.write('{}\n'.format(obj.final['img'])) # Dump the Analyzer object into a pickle file for later fast recovery ep.dump(analyzer_file, self.analysis_result)
def print_results(self, final_table=None): """ Prints diagnostics from the final integration run. """ cons_s = Counter(self.s).most_common(1)[0][0] cons_h = Counter(self.h).most_common(1)[0][0] cons_a = Counter(self.a).most_common(1)[0][0] if final_table is None: final_table = ["\n\n{:-^80}\n".format('ANALYSIS OF RESULTS')] # In case no images were integrated if self.final_objects is None: final_table.append('NO IMAGES INTEGRATED!') else: if self.params.advanced.processing_backend == 'ha14': final_table.append("Avg. signal height: {:<8.3f} std. dev: " "{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}" "".format(np.mean(self.s), np.std(self.s), max(self.s), min(self.s), cons_s)) final_table.append("Avg. spot height: {:<8.3f} std. dev: " "{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}" "".format(np.mean(self.h), np.std(self.h), max(self.h), min(self.h), cons_h)) final_table.append("Avg. spot areas: {:<8.3f} std. dev: " "{:<6.2f} max: {:<3} min: {:<3} consensus: {:<3}" "".format(np.mean(self.a), np.std(self.a), max(self.a), min(self.a), cons_a)) final_table.append("Avg. resolution: {:<8.3f} std. dev: " "{:<6.2f} lowest: {:<6.3f} highest: {:<6.3f}" "".format(np.mean(self.hres), np.std(self.hres), max(self.hres), min(self.hres))) final_table.append("Avg. number of spots: {:<8.3f} std. dev: {:<6.2f}" "".format(np.mean(self.spots), np.std(self.spots))) final_table.append("Avg. mosaicity: {:<8.3f} std. dev: {:<6.2f}" "".format(np.mean(self.mos), np.std(self.mos))) # If more than one integrated image, plot various summary graphs if len(self.final_objects) > 1: plot = Plotter(self.params, self.final_objects, self.viz_dir) if self.params.analysis.summary_graphs: if ( self.params.advanced.processing_backend == 'ha14' and self.params.cctbx_ha14.grid_search.type is not None ): plot.plot_spotfinding_heatmap(write_files=True) plot.plot_res_histogram(write_files=True) med_beamX, med_beamY, pixel_size = plot.plot_beam_xy(write_files=True, return_values=True) else: with warnings.catch_warnings(): # To catch any 'mean of empty slice' runtime warnings warnings.simplefilter("ignore", category=RuntimeWarning) beamXY_info = plot.calculate_beam_xy() beamX, beamY = beamXY_info[:2] med_beamX = np.median(beamX) med_beamY = np.median(beamY) pixel_size = beamXY_info[-1] final_table.append("Median Beam Center: X = {:<4.2f}, Y = {:<4.2f}" "".format(med_beamX, med_beamY)) self.analysis_result.__setattr__('beamX_mm', med_beamX) self.analysis_result.__setattr__('beamY_mm', med_beamY) self.analysis_result.__setattr__('pixel_size', pixel_size) self.analysis_result.__setattr__('final_table', final_table) for item in final_table: util.main_log(self.logfile, item, (not self.gui_mode))
class Processor(object): """ Class for image integration (w/ grid search params) """ def __init__(self, params, source_image=None, output_image=None, viz=None, log=None, tag='grid search', tmp_base=None, gain=1, single_image=False): self.params = params self.img = source_image self.out_img = output_image self.min_sigma = self.params.cctbx_ha14.selection.min_sigma self.target = os.path.abspath(self.params.cctbx_ha14.target) self.viz = viz self.tag = tag self.int_log = log self.charts = self.params.analysis.charts self.tmp_base = tmp_base self.single_image = single_image self.method = self.params.mp.method self.queue = self.params.mp.queue self.args = [ "target={}".format(self.target), "indexing.data={}".format(self.img), "spots_pickle=None", "subgroups_pickle=None", "refinements_pickle=None", "rmsd_tolerance=5.0", "mosflm_rmsd_tolerance=5.0", "integration.detector_gain={}".format(gain), "indexing.verbose_cv=True" ] # Add target unit cell if exists if self.params.cctbx_ha14.target_unit_cell is not None: t_uc = [ str(i) for i in self.params.cctbx_ha14.target_unit_cell.parameters() ] self.args.extend(['target_cell="{}"'.format(' '.join(t_uc))]) # Translate / add target lattice if exists t_lat = util.makenone(self.params.cctbx_ha14.target_lattice_type) if t_lat: if t_lat == 'triclinic': known_setting = 1 elif t_lat == 'monoclinic': known_setting = 2 elif t_lat in ('orthorhombic', 'rhombohedral'): known_setting = 5 elif t_lat == 'tetragonal': known_setting = 9 elif t_lat == 'hexagonal': known_setting = 12 elif t_lat == 'cubic': known_setting = 22 else: known_setting = None if known_setting: self.args.extend(['known_setting={}'.format(known_setting)]) # Centering type if exists t_ctype = self.params.cctbx_ha14.target_centering_type if t_ctype is not None: self.args.extend(['target_cell_centring_type={}'.format(t_ctype)]) # Resolution, if exists hires = self.params.cctbx_ha14.resolution_limits.high lowres = self.params.cctbx_ha14.resolution_limits.low if hires is None: hires = 1.5 if lowres is None: lowres = 99.9 self.args.extend([ 'force_method2_resolution_limit={}'.format(hires), 'distl_lowres_limit={}'.format(lowres), 'distl_highres_limit={}'.format(hires), 'distl.res.inner={}'.format(lowres), 'distl.res.outer={}'.format(hires) ]) # Add gain (necessary now) self.args.extend(['integration.detector_gain={}'.format(gain)]) with open(os.path.join(self.params.output, 'test.txt'), 'w') as tf: tf.write('\n'.join(self.args)) def integrate(self, grid_point): """ Runs the integration module in cctbx.xfel; used by either grid-search or final integration function. """ self.s = grid_point['sih'] self.h = grid_point['sph'] self.a = grid_point['spa'] args = self.args # Generate advanced arguments (and PDF subfolder) if self.charts and self.tag == 'grid search': filename = os.path.basename(self.img).split('.')[0] pdf_folder = os.path.join(self.viz, 'pdf_{}/s{}_h{}_a{}'\ ''.format(filename, self.s, self.h, self.a)) if not os.path.exists(pdf_folder): os.makedirs(pdf_folder) self.args.extend([ "integration.enable_residual_map=True", "integration.enable_residual_scatter=True", "integration.mosaic.enable_AD14F7B=True", "integration.graphics_backend=pdf", "integration.pdf_output_dir={}".format(pdf_folder) ]) if self.tag == 'integrate': args.append("indexing.completeness_pickle={}".format(self.out_img)) #Actually run integration using iota.bulletproof error_message = '' with util.Capturing() as index_log: arguments = [ "distl.minimum_signal_height={}".format( str(self.s)), "distl.minimum_spot_height={}".format( str(self.h)), "distl.minimum_spot_area={}".format( str(self.a)), "indexing.open_wx_viewer=False" ] + list(args[1:]) tmppath = os.path.join(self.tmp_base, str(uuid.uuid4()) + ".pickle") assert not os.path.exists(tmppath) # invoke the indexer in a way that will protect iota from any crashes command = "iota.bulletproof {} {} {}" \ "".format(tmppath, self.target, " ".join(arguments)) try: easy_run.fully_buffered(command, join_stdout_stderr=True).show_stdout() if not os.path.exists(tmppath): print(tmppath) print(command) raise Exception("Indexing failed for an unknown reason") # iota.bulletproof saves the needed results from indexing in a tmp file result = easy_pickle.load(tmppath) os.remove(tmppath) if isinstance(result, str): raise Exception(result) else: int_final = result except Exception, e: int_final = None if hasattr(e, "classname"): print( e.classname, "for %s:" % self.img, ) error_message = "{}: {}".format( e.classname, e[0].replace('\n', ' ')[:50]) else: print("Integration error for %s:" % self.img, ) error_message = "{}".format(str(e).replace('\n', ' ')[:50]) print(e) # Output results of integration (from the "info" object returned by # run_one_index_core) if int_final is None: if error_message != '': reason_for_failure = " - {}".format(error_message) else: reason_for_failure = '' int_status = 'not integrated' + reason_for_failure int_results = {'info': int_status} elif int_final['observations'][0] is None: int_status = 'no data recorded' int_results = {'info': int_status} else: try: obs = int_final['observations'][0] cell = obs.unit_cell().parameters() sg = int_final['pointgroup'] lres, hres = obs.d_max_min() # Calculate number of spots w/ high I / sigmaI Is = obs.data() sigmas = obs.sigmas() I_over_sigI = Is / sigmas #spots = len(Is) strong_spots = len( [i for i in I_over_sigI if i >= self.min_sigma]) # Mosaicity parameters mosaicity = round( (int_final.get('ML_half_mosaicity_deg', [0])[0]), 6) dom_size = int_final.get('ML_domain_size_ang', [0])[0] ewald_proximal_volume = int_final.get('ewald_proximal_volume', [0])[0] # Assemble output for log file and/or integration result file p_cell = "{:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}, {:>6.2f}"\ "".format(cell[0], cell[1], cell[2], cell[3], cell[4], cell[5]) int_status = 'RES: {:<4.2f} NSREF: {:<4} SG: {:<5} CELL: {}'\ ''.format(hres, strong_spots, sg, p_cell) int_results = { 'sg': sg, 'a': cell[0], 'b': cell[1], 'c': cell[2], 'alpha': cell[3], 'beta': cell[4], 'gamma': cell[5], 'wavelength': int_final['wavelength'], 'distance': int_final['distance'], 'beamX': int_final['xbeam'], 'beamY': int_final['ybeam'], 'strong': strong_spots, 'res': hres, 'lres': lres, 'mos': mosaicity, 'epv': ewald_proximal_volume, 'info': int_status, 'ok': True } except ValueError: print(self.img) # write integration logfile if self.tag == 'integrate': util.main_log(self.int_log, "{:-^100}\n{:-^100}\n{:-^100}\n"\ "".format("", " FINAL INTEGRATION: ", ""\ "S = {:>2}, H ={:>2}, A ={:>2} "\ "".format(self.s, self.h, self.a))) else: util.main_log(self.int_log, "{:-^100}\n".format(" INTEGRATION: "\ "S = {:>2}, H ={:>2}, A ={:>2} "\ "".format(self.s, self.h, self.a))) for item in index_log: util.main_log(self.int_log, item) util.main_log(self.int_log, "\n[ {:^100} ]\n\n".format(int_status)) # In single-image mode, write a file with h, k, l, I, sigma if self.single_image == True and self.tag == 'integrate': hklI_filename = "{}.{}".format( os.path.basename(self.out_img).split('.')[0], 'hkli') hklI_file = os.path.join(os.path.dirname(self.out_img), hklI_filename) hklI = zip(obs.indices(), obs.data(), obs.sigmas()) for i in hklI: with open(hklI_file, 'a') as f: entry = '{},{},{},{},{}'.format(i[0][0], i[0][1], i[0][2], i[1], i[2]) f.write('{}\n'.format(entry)) return int_results
def process(self, img_object, single_image=False): """ Image processing; selects method, runs requisite modules """ self.img_object = img_object if self.img_object.status != 'bypass grid search': self.img_object.status = 'processing' #for CCTBX indexing / integration terminate = False prev_status = self.img_object.status prev_fail = 'first cycle' prev_final = self.img_object.final prev_epv = 9999 while not terminate: # Run grid search if haven't already if (self.img_object.fail is None and 'grid search' not in self.img_object.status): self.integrate_cctbx('grid search', single_image=single_image) # Run selection if haven't already if (self.img_object.fail is None and self.img_object.status != 'selection'): self.select_cctbx() # If smart grid search is active run multiple rounds until convergence if self.params.cctbx_ha14.grid_search.type == 'smart': if (self.img_object.fail is None and self.img_object.final['epv'] < prev_epv): prev_epv = self.img_object.final['epv'] prev_final = self.img_object.final prev_status = self.img_object.status prev_fail = self.img_object.fail self.hmed = self.img_object.final['sph'] self.amed = self.img_object.final['spa'] self.img_object.generate_grid() self.img_object.final['final'] = self.img_object.int_file if len(self.img_object.grid) == 0: self.img_object.final = prev_final self.img_object.status = prev_status self.img_object.fail = prev_fail terminate = True continue if self.verbose: log_entry = '\nNew starting point: H = {}, A = {}\n'\ ''.format(self.hmed, self.amed) self.img_object.log_info.append(log_entry) else: if prev_fail != 'first cycle': self.img_object.final = prev_final self.img_object.status = prev_status self.img_object.fail = prev_fail if self.verbose: log_entry = '\nFinal set of parameters: H = {}, A = {}'\ ''.format(self.img_object.final['sph'], self.img_object.final['spa']) self.img_object.log_info.append(log_entry) terminate = True # If brute force grid search is selected run one round else: terminate = True # Run final integration if haven't already if self.img_object.fail is None and self.img_object.status != 'final': self.integrate_cctbx('integrate', single_image=single_image) # If verbose output selected (default), write to main log if self.verbose: log_entry = "\n".join(self.img_object.log_info) util.main_log(self.init.logfile, log_entry) util.main_log(self.init.logfile, '\n\n') # Make a temporary process log into a final process log if os.path.isfile(self.img_object.int_log): l_fname = util.make_filename(self.img_object.int_log, new_ext='log') final_int_log = os.path.join(self.img_object.log_path, l_fname) os.rename(self.img_object.int_log, final_int_log) # Save results into a pickle file self.img_object.status = 'final' from libtbx import easy_pickle as ep ep.dump(self.img_object.obj_file, self.img_object) return self.img_object
def process(self): """ Run Process module from command line """ if self.stage == 'import': # Import Images self.create_image_iterable() if self.init.params.cctbx_ha14.selection.select_only.flag_on: msg = "Reading {} image objects".format( len(self.init.gs_img_objects)) title = 'READING IMAGE OBJECTS' else: msg = "Importing {} images".format(len(self.init.input_list)) title = 'IMPORTING IMAGES' cmd.Command.start(msg) self.prog_count = 0 self.gs_prog = cmd.ProgressBar(title=title) self.run_process() # Remove rejected images from image object list acc_img_objects = [ i.fail for i in self.img_objects if i.fail is None ] cmd.Command.end("Accepted {} of {} images -- DONE " \ "".format(len(acc_img_objects), len(self.img_objects))) # Exit if none of the images have diffraction if str(self.init.params.cctbx_ha14.image_triage.type).lower( ) != 'none': if len(acc_img_objects) == 0: util.main_log(self.init.logfile, 'No images have diffraction!', True) util.iota_exit() else: util.main_log( self.init.logfile, "{} out of {} images have diffraction " "(at least {} Bragg peaks)" "".format( len(acc_img_objects), len(self.img_objects), self.init.params.cctbx_ha14.image_triage. min_Bragg_peaks)) self.stage = 'process' # Check for -c option and exit if true if self.init.params.cctbx_ha14.image_conversion.convert_only: util.iota_exit() # Process Images self.create_image_iterable() cmd.Command.start("Processing {} images".format(len(self.iterable))) self.prog_count = 0 self.gs_prog = cmd.ProgressBar(title='PROCESSING') self.run_process() cmd.Command.end("Processing {} images -- DONE".format( len(self.iterable))) # Analysis of integration results final_objects = [i for i in self.img_objects if i.fail is None] if len(final_objects) > 0: self.run_analysis() # Write info object / file (TODO: Revisit this later!) from iota.components.iota_threads import ObjectReaderThread self.object_reader = ObjectReaderThread( self, info=self.init.info, source=self.img_objects, info_file=self.init.info_file) self.object_reader.start() else: print('No images successfully integrated!') # Exit IOTA util.iota_exit()
def initialize_processing(paramfile, run_no): """Initialize processing for a set of images. :param paramfile: text file with IOTA parameters :param run_no: number of the processing run :return: info: INFO object params: IOTA params """ try: phil, _ = inp.get_input_phil(paramfile=paramfile) except Exception as e: msg = "IOTA_PROC_ERROR: Cannot import IOTA parameters! {}".format(e) return False, msg else: params = phil.extract() # Reconstruct integration base path and get info object int_base = os.path.join(params.output, "integration/{:03d}".format(run_no)) try: info_file = os.path.join(int_base, "proc.info") info = ProcInfo.from_json(filepath=info_file) except Exception as e: msg = "IOTA_PROC_ERROR: Cannot import INFO object! {}".format(e) return False, msg # Generate input list and input base if not hasattr(info, "input_list"): info.generate_input_list(params=params) filepath_list = [] for item in info.input_list: if isinstance(item, list) or isinstance(item, tuple): fp = [i for i in item if os.path.exists(str(i))] if fp and len(fp) == 1: filepath_list.append(fp[0]) else: if os.path.exists(item): filepath_list.append(item) common_pfx = os.path.abspath( os.path.dirname(os.path.commonprefix(filepath_list))) input_base = common_pfx if os.path.isdir(os.path.abspath(params.input[0])): new_common_pfx = os.path.commonprefix( [os.path.abspath(params.input[0]), common_pfx]) if new_common_pfx not in ("", "."): input_base = new_common_pfx # Generate subfolder paths paths = dict( obj_base=os.path.join(int_base, "image_objects"), fin_base=os.path.join(int_base, "final"), log_base=os.path.join(int_base, "logs"), dials_log_base=os.path.join(int_base, "logs/dials_logs"), viz_base=os.path.join(int_base, "visualization"), tmp_base=os.path.join(int_base, "tmp"), input_base=input_base, ) for bkey, bvalue in paths.items(): if bkey == "input_base": continue if not os.path.isdir(bvalue): os.makedirs(bvalue) info.update(paths) # Generate filepaths for various info files info_files = dict( obj_list_file=os.path.join(info.tmp_base, "finished_objects.lst"), idx_file=os.path.join(info.int_base, "observations.pickle"), ) info.update(info_files) # Initialize stat containers info = generate_stat_containers(info=info, params=params) # Initialize main log util.main_log(info.logfile, "{:*^80} \n".format(" IOTA MAIN LOG ")) util.main_log(info.logfile, "{:-^80} \n".format(" SETTINGS FOR THIS RUN ")) util.main_log(info.logfile, info.iota_phil) util.main_log(info.logfile, "{:-^80} \n".format("BACKEND SETTINGS")) util.main_log(info.logfile, info.target_phil) info.export_json() return info, params