def __str__(self): """Return a string representation of the results. Returns: str: """ output = [] output.append("Scoring individual symmetry elements") d = self.as_dict() output.append( table_utils.format(self.sym_ops_table(d), has_header=True, delim=" ")) output.append("Scoring all possible sub-groups") output.append( table_utils.format(self.subgroups_table(d), has_header=True, delim=" ")) output.append( "Best solution: %s" % self.best_solution.subgroup["best_subsym"].space_group_info()) output.append("Unit cell: %s" % self.best_solution.subgroup["best_subsym"].unit_cell()) output.append("Reindex operator: %s" % (self.best_solution.subgroup["cb_op_inp_best"] * self.cb_op_inp_min)) output.append("Laue group probability: %.3f" % self.best_solution.likelihood) output.append("Laue group confidence: %.3f" % self.best_solution.confidence) return "\n".join(output)
def show_times_vs_complexity(times, header): table_header = [ "# HKL", "cpu-time", "simple-tm", "fft-time", "R(%)", "gpu_d-tm", "gpu_f-tm", "R(%)", "d-min(angstrom)" ] table_data = [table_header] for i, t, d, g, gf, gfR, s, f, fR in times: table_row = [ "%.0f" % i, "%.2f" % t, "%.2f" % s, "%.2f" % f, "%.2f" % fR, "%.2f" % g, "%.2f" % gf, "%.5f" % gfR, "%5.2f" % d ] table_data.append(table_row) print table_utils.format(table_data, has_header=True, justify="left", prefix="| ", postfix=" |")
def print_table(self): from libtbx import table_utils from libtbx.str_utils import format_value table_header = ["Tile","Dist","Nobs","aRmsd","Rmsd","delx","dely","disp","rotdeg","Rsigma","Tsigma"] table_data = [] table_data.append(table_header) sort_radii = flex.sort_permutation(flex.double(self.radii)) tile_rmsds = flex.double() radial_sigmas = flex.double(len(self.tiles) // 4) tangen_sigmas = flex.double(len(self.tiles) // 4) for idx in range(len(self.tiles) // 4): x = sort_radii[idx] if self.tilecounts[x] < 3: wtaveg = 0.0 radial = (0,0) tangential = (0,0) rmean,tmean,rsigma,tsigma=(0,0,1,1) else: wtaveg = self.weighted_average_angle_deg_from_tile(x) radial,tangential,rmean,tmean,rsigma,tsigma = get_radial_tangential_vectors(self,x) radial_sigmas[x]=rsigma tangen_sigmas[x]=tsigma table_data.append( [ format_value("%3d", x), format_value("%7.2f", self.radii[x]), format_value("%6d", self.tilecounts[x]), format_value("%5.2f", self.asymmetric_tile_rmsd[x]), format_value("%5.2f", self.tile_rmsd[x]), format_value("%5.2f", self.mean_cv[x][0]), format_value("%5.2f", self.mean_cv[x][1]), format_value("%5.2f", matrix.col(self.mean_cv[x]).length()), format_value("%6.2f", wtaveg), format_value("%6.2f", rsigma), format_value("%6.2f", tsigma), ]) table_data.append([""]*len(table_header)) rstats = flex.mean_and_variance(radial_sigmas,self.tilecounts.as_double()) tstats = flex.mean_and_variance(tangen_sigmas,self.tilecounts.as_double()) table_data.append( [ format_value("%3s", "ALL"), format_value("%s", ""), format_value("%6d", self.overall_N), format_value("%5.2f", math.sqrt(flex.mean(self.delrsq))), format_value("%5.2f", self.overall_rmsd), format_value("%5.2f", self.overall_cv[0]), format_value("%5.2f", self.overall_cv[1]), format_value("%5.2f", flex.mean(flex.double([matrix.col(cv).length() for cv in self.mean_cv]))), format_value("%s", ""), format_value("%6.2f", rstats.mean()), format_value("%6.2f", tstats.mean()), ]) print print table_utils.format(table_data,has_header=1,justify='center',delim=" ")
def info(self): from libtbx import table_utils U = matrix.sqr(self.experiment.crystal.get_U()) B = matrix.sqr(self.experiment.crystal.get_B()) a_star_ = U * B * a_star b_star_ = U * B * b_star c_star_ = U * B * c_star Binvt = B.inverse().transpose() a_ = U * Binvt * a b_ = U * Binvt * b c_ = U * Binvt * c names = self.experiment.goniometer.get_names() axes = self.experiment.goniometer.get_axes() rows = [['Experimental axis', 'a*', 'b*', 'c*']] rows.append([names[0]] + [ '%.3f' % smallest_angle(axis.angle(matrix.col(axes[0]), deg=True)) for axis in (a_star_, b_star_, c_star_) ]) rows.append(['Beam'] + [ '%.3f' % smallest_angle(axis.angle(self.s0, deg=True)) for axis in (a_star_, b_star_, c_star_) ]) rows.append([names[2]] + [ '%.3f' % smallest_angle(axis.angle(matrix.col(axes[2]), deg=True)) for axis in (a_star_, b_star_, c_star_) ]) print( 'Angles between reciprocal cell axes and principal experimental axes:' ) print(table_utils.format(rows=rows, has_header=True)) print() rows = [['Experimental axis', 'a', 'b', 'c']] rows.append([names[0]] + [ '%.3f' % smallest_angle(axis.angle(matrix.col(axes[0]), deg=True)) for axis in (a_, b_, c_) ]) rows.append(['Beam'] + [ '%.3f' % smallest_angle(axis.angle(self.s0, deg=True)) for axis in (a_, b_, c_) ]) rows.append([names[2]] + [ '%.3f' % smallest_angle(axis.angle(matrix.col(axes[2]), deg=True)) for axis in (a_, b_, c_) ]) print('Angles between unit cell axes and principal experimental axes:') print(table_utils.format(rows=rows, has_header=True)) print()
def info(self): from libtbx import table_utils U = self.experiment.crystal.get_U() B = self.experiment.crystal.get_B() a_star_ = U * B * a_star b_star_ = U * B * b_star c_star_ = U * B * c_star Binvt = B.inverse().transpose() a_ = U * Binvt * a b_ = U * Binvt * b c_ = U * Binvt * c def smallest_angle(angle): return min(angle, 180-angle) names = self.experiment.goniometer.get_names() axes = self.experiment.goniometer.get_axes() rows = [['Experimental axis', 'a*', 'b*', 'c*']] rows.append([names[0]] + [ '%.3f' %smallest_angle(axis.angle(matrix.col(axes[0]), deg=True)) for axis in (a_star_, b_star_, c_star_)]) rows.append(['Beam'] + [ '%.3f' %smallest_angle(axis.angle(self.s0, deg=True)) for axis in (a_star_, b_star_, c_star_)]) rows.append([names[2]] + [ '%.3f' %smallest_angle(axis.angle(matrix.col(axes[2]), deg=True)) for axis in (a_star_, b_star_, c_star_)]) print 'Angles between reciprocal cell axes and principal experimental axes:' print table_utils.format(rows=rows, has_header=True) print rows = [['Experimental axis', 'a', 'b', 'c']] rows.append([names[0]] + [ '%.3f' %smallest_angle(axis.angle(matrix.col(axes[0]), deg=True)) for axis in (a_, b_, c_)]) rows.append(['Beam'] + [ '%.3f' %smallest_angle(axis.angle(self.s0, deg=True)) for axis in (a_, b_, c_)]) rows.append([names[2]] + [ '%.3f' %smallest_angle(axis.angle(matrix.col(axes[2]), deg=True)) for axis in (a_, b_, c_)]) print 'Angles between unit cell axes and principal experimental axes:' print table_utils.format(rows=rows, has_header=True) print
def print_table(spotfinder_result, keys, out=None): if out is None: import sys out = sys.stdout from libtbx import table_utils rows = table(spotfinder_result, keys) print >> out, table_utils.format(rows, has_header=True)
def labelit_printout(self, out=None): from libtbx import table_utils if out is None: import sys out = sys.stdout table_data = [[ "Solution", "Metric fit", "rmsd", "min/max cc", "#spots", "lattice", "unit_cell", "volume", "cb_op" ]] for item in self: uc = item.refined_crystal.get_unit_cell() P = uc.parameters() min_max_cc_str = "-/-" if item.min_cc is not None and item.max_cc is not None: min_max_cc_str = "%.3f/%.3f" % (item.min_cc, item.max_cc) if item.recommended: status = '*' else: status = '' table_data.append([ "%1s%7d" % (status, item.setting_number), "%(max_angular_difference)6.4f" % item, "%5.3f" % item.rmsd, min_max_cc_str, "%d" % item.Nmatches, "%(bravais)s" % item, "%6.2f %6.2f %6.2f %6.2f %6.2f %6.2f" % P, "%.0f" % uc.volume(), "%s" % item['cb_op_inp_best'].as_abc() ]) print(table_utils.format(table_data, has_header=1, justify='right', delim=' '), file=out) print("* = recommended solution", file=out)
def show_experiments(self, experiments, reflections, d_min=None): if d_min is not None: reciprocal_lattice_points = reflections["rlp"] d_spacings = 1 / reciprocal_lattice_points.norms() reflections = reflections.select(d_spacings > d_min) for i_expt, expt in enumerate(experiments): logger.info("model %i (%i reflections):" % (i_expt + 1, (reflections["id"] == i_expt).count(True))) logger.info(expt.crystal) indexed_flags = reflections.get_flags(reflections.flags.indexed) imageset_id = reflections["imageset_id"] rows = [["Imageset", "# indexed", "# unindexed", "% indexed"]] for i in range(flex.max(imageset_id) + 1): imageset_indexed_flags = indexed_flags.select(imageset_id == i) indexed_count = imageset_indexed_flags.count(True) unindexed_count = imageset_indexed_flags.count(False) rows.append([ str(i), str(indexed_count), str(unindexed_count), "{:.1%}".format(indexed_count / (indexed_count + unindexed_count)), ]) from libtbx import table_utils logger.info( table_utils.format(rows, has_header=True, prefix="| ", postfix=" |"))
def print_table(stats, perm=None, n_rows=None, out=None): if out is None: out = sys.stdout rows = table(stats, perm=perm, n_rows=n_rows) print(table_utils.format(rows, has_header=True, prefix="| ", postfix=" |"), file=out)
def get_formatted_table(self, experiment_count_per_bin, total_experiment_count): '''Produce a table with experiment count over resolution bins''' table_data = [["Bin", "Resolution Range", "Lattices", "Accepted (%)"]] for i_bin in self.resolution_binner.range_used(): col_legend = '%-13s' % self.resolution_binner.bin_legend( i_bin=i_bin, show_bin_number=False, show_bin_range=False, show_d_range=True, show_counts=False) exp_count_abs = '%8d' % experiment_count_per_bin[i_bin] exp_count_percent = '%5.2f' % (100. * experiment_count_per_bin[i_bin] / total_experiment_count) table_data.append( ['%3d' % i_bin, col_legend, exp_count_abs, exp_count_percent]) table_data.append([""] * len(table_data[0])) table_data.append(["All", "", '%8d' % total_experiment_count]) return "\n Image Statistics\n" + table_utils.format( table_data, has_header=1, justify='center', delim=' ')
def make_log_beam_stop(self, log_message, flags): self.norma_work = self.norma_work.map_to_asu() self.miller_obs = self.miller_obs.map_to_asu() flags = flags.map_to_asu() data = self.miller_obs.select(~flags.data()) evals = self.norma_work.select(~flags.data()) header = ("Index", "d-spacing", "F_obs", "E-value", "centric") table = "No outliers were found" rows = [] if data.data().size() > 0: if data.data().size() < 500: for hkl, d, fobs, e, c in zip( data.indices(), data.d_spacings().data(), data.data(), evals.data(), data.centric_flags().data() ): this_row = [str(hkl), "%4.2f" % (d), "%6.1f" % (fobs), "%4.2f" % (math.sqrt(e)), str(c)] rows.append(this_row) table = table_utils.format( [header] + rows, comments=None, has_header=True, separate_rows=False, prefix="| ", postfix=" |" ) else: table = """Over 500 outliers have been found.""" final = log_message + "\n" + table + "\n \n" return final
def make_log_wilson(self, log_message, flags, p_values): """ produces a 'nice' table of outliers and their reason for being an outlier using basic or extreme wilson statistics """ header = ("Index", "E_obs", "Centric", "p-value") flags = flags.common_set(self.norma_work) p_vals = p_values.common_set(self.norma_work) rogues = self.norma_work.select(~flags.data()) p_vals = p_vals.select(~flags.data()) rows = [] table = "No outliers were found." for hkl, e, c, p in zip(rogues.indices(), rogues.data(), rogues.centric_flags().data(), p_vals.data()): if e > 0: this_row = [str(hkl), "%5.3f" % (math.sqrt(e)), str(c), "%5.3e" % (p)] else: this_row = [str(hkl), "%5.3f" % (0), str(c), " inf"] rows.append(this_row) if len(rows) > 0: table = table_utils.format( [header] + rows, comments=None, has_header=True, separate_rows=False, prefix="| ", postfix=" |" ) final = log_message + "\n" + table return final
def make_log_wilson(self, log_message, flags, p_values): """ produces a 'nice' table of outliers and their reason for being an outlier using basic or extreme wilson statistics """ header = ("Index", "E_obs", "Centric", "p-value") flags = flags.common_set(self.norma_work) p_vals = p_values.common_set(self.norma_work) rogues = self.norma_work.select(~flags.data()) p_vals = p_vals.select(~flags.data()) rows = [] table = "No outliers were found." for hkl, e, c, p in zip(rogues.indices(), rogues.data(), rogues.centric_flags().data(), p_vals.data()): if e > 0: this_row = [ str(hkl), "%5.3f" % (math.sqrt(e)), str(c), "%5.3e" % (p) ] else: this_row = [str(hkl), "%5.3f" % (0), str(c), " inf"] rows.append(this_row) if len(rows) > 0: table = table_utils.format([header] + rows, comments=None, has_header=True, separate_rows=False, prefix='| ', postfix=' |') final = log_message + "\n" + table return final
def show_times_vs_complexity(times, header): table_header = [ "# HKL", "cpu-time", "simple-tm", "fft-time","R(%)", "gpu_d-tm", "gpu_f-tm","R(%)", "d-min(angstrom)" ] table_data = [table_header] for i,t,d,g,gf,gfR,s,f,fR in times: table_row = ["%.0f"%i,"%.2f"%t,"%.2f"%s,"%.2f"%f,"%.2f"%fR, "%.2f"%g,"%.2f"%gf,"%.5f"%gfR,"%5.2f"%d] table_data.append(table_row) print table_utils.format(table_data,has_header=True,justify="left", prefix="| ",postfix=" |")
def labelit_printout(self,out=None): from libtbx import table_utils if out is None: import sys out = sys.stdout table_data = [["Solution","Metric fit","rmsd", "min/max cc", "#spots", "lattice","unit_cell","volume", "cb_op"]] for item in self: uc = item.refined_crystal.get_unit_cell() P = uc.parameters() min_max_cc_str = "-/-" if item.min_cc is not None and item.max_cc is not None: min_max_cc_str = "%.3f/%.3f" %(item.min_cc, item.max_cc) table_data.append(['%6d'%item.setting_number, "%(max_angular_difference)6.4f"%item, "%5.3f"%item.rmsd, min_max_cc_str, "%d"%item.Nmatches, "%(bravais)s"%item, "%6.2f %6.2f %6.2f %6.2f %6.2f %6.2f"%P, "%.0f"%uc.volume(), "%s"%item['cb_op_inp_best'].as_abc()]) print >> out, table_utils.format( table_data, has_header=1, justify='right', delim=' ')
def show(self): from libtbx import table_utils self.info() rows = [] names = self.experiment.goniometer.get_names() for angles, solutions in self.unique_solutions.iteritems(): for (v1, v2) in solutions: rows.append( (self._vector_as_str(v1), self._vector_as_str(v2), '% 7.2f' %angles[0], '% 7.2f' %angles[1], )) rows = [('v1', 'v2', names[1], names[0])] + \ sorted(rows) print 'Independent solutions:' print table_utils.format(rows=rows, has_header=True)
def print_table(stats, perm=None, n_rows=None, out=None): if out is None: import sys out = sys.stdout from libtbx import table_utils rows = table(stats, perm=perm, n_rows=n_rows) print >> out, table_utils.format( rows, has_header=True, prefix="| ", postfix=" |")
def show(self): from libtbx import table_utils self.info() rows = [] names = self.experiment.goniometer.get_names() for angles, vector_pairs in self.unique_solutions.iteritems(): settings_str = '[%s]' %( ', '.join( '(%s, %s)' %(self._vector_as_str(v1), self._vector_as_str(v2)) for v1, v2 in vector_pairs)) rows.append(( settings_str, '% 7.2f' %angles[0], '% 7.2f' %angles[1], )) rows = [('Settings', names[1], names[0])] + \ sorted(rows) print 'Independent solutions:' print table_utils.format(rows=rows, has_header=True)
def show(self): from libtbx import table_utils self.info() rows = [] names = self.experiment.goniometer.get_names() space_group = self.experiment.crystal.get_space_group() reciprocal = self.frame == 'reciprocal' for angles, vector_pairs in self.unique_solutions.iteritems(): v1, v2 = list(vector_pairs)[0] rows.append(( describe(v1, space_group, reciprocal=reciprocal), describe(v2, space_group, reciprocal=reciprocal), '% 7.3f' %angles[0], '% 7.3f' %angles[1], )) rows = [('Primary axis', 'Secondary axis', names[1], names[0])] + \ sorted(rows) print 'Independent solutions:' print table_utils.format(rows=rows, has_header=True)
def write_output(self): rows = [["cluster_id", "# datasets", "height", "datasets"]] for cid in sorted(self._cluster_dict.keys()): cluster = self._cluster_dict[cid] datasets = cluster['datasets'] rows.append([str(cid), str(len(datasets)), '%.2f' %cluster['height'], ' '.join(['%s'] * len(datasets)) % tuple(datasets)]) with open('intensity_clustering.txt', 'wb') as f: from libtbx import table_utils print >> f, table_utils.format( rows, has_header=True, prefix="|", postfix="|")
def get_table(self): from libtbx import table_utils rows = [["dataset", "batches", "delta_cc_i", "sigma"]] labels = self._labels() normalised_score = self._normalised_delta_cc_i() perm = flex.sort_permutation(self.delta_cc) for i in perm: bmin = flex.min(self.batches[i].data()) bmax = flex.max(self.batches[i].data()) rows.append( [str(labels[i]), '%i to %i' %(bmin, bmax), '% .3f' %self.delta_cc[i], '% .2f' %normalised_score[i]]) return table_utils.format(rows, has_header=True, prefix="|", postfix="|")
def select_and_transform(self, matches_cut_off=0.75): ## hopsa max_cc = -1.0 location = 0 table_data = [] for ii in range(len(self.nice_cb_ops)): table_data.append([ self.nice_cb_ops[ii].as_hkl(), "%4.3f" % (self.cc_values[ii]), "%4.1f" % (self.matches[ii]), ' ' ]) if self.matches[ii] >= matches_cut_off: if max_cc < self.cc_values[ii]: max_cc = self.cc_values[ii] location = ii legend = ('Operator', 'Correlation', 'matches (%)', 'choice') table_data[location][3] = ' <--- ' self.table = table_utils.format([legend] + table_data, comments=None, has_header=True, separate_rows=False, prefix='| ', postfix=' |') print >> self.out print >> self.out, "Reference analyses" print >> self.out, " The following reindexing operators have been found:" print >> self.out print >> self.out, self.table print >> self.out if str(self.nice_cb_ops[location].as_hkl()) == "h,k,l": print >> self.out, "The data doesn't need to be reindexed! Indexing is consistent between these datasets." else: print >> self.out, "If the data is reindexed with operator (%s), the correlation of" % ( self.nice_cb_ops[location].as_hkl()) print >> self.out, "the intensities to the reference data is maximized. " if self.file_name is not None: print self.file_name print >> self.out, "This can be done for instance with:" print >> self.out, " phenix.reflection_file_converter %s --change_of_basis=\"%s\" <output_options> " % ( self.file_name, self.nice_cb_ops[location].as_hkl()) print >> self.out, "-------------------------------------------------------------------------------" ## change things in primitive setting transform_b = self.set_b_ori.change_basis( self.set_b_to_niggli).change_basis( self.ops_in_niggli_setting[location]).change_basis( self.set_b_to_niggli.inverse()).map_to_asu( ).set_observation_type(self.set_b_ori) return (transform_b)
def select_and_transform(self, matches_cut_off=0.75 ): ## hopsa max_cc=-1.0 location = 0 table_data=[] for ii in range(len(self.nice_cb_ops)): table_data.append( [self.nice_cb_ops[ii].as_hkl(), "%4.3f"%(self.cc_values[ii]), "%4.1f"%(self.matches[ii]), ' '] ) if self.matches[ii]>=matches_cut_off: if max_cc<self.cc_values[ii]: max_cc = self.cc_values[ii] location = ii legend = ('Operator', 'Correlation', 'matches (%)', 'choice') table_data[location][3]=' <--- ' self.table = table_utils.format([legend]+table_data, comments=None, has_header=True, separate_rows=False, prefix='| ', postfix=' |') print >> self.out print >> self.out, "Reference analyses" print >> self.out, " The following reindexing operators have been found:" print >> self.out print >> self.out, self.table print >> self.out if str(self.nice_cb_ops[location].as_hkl()) == "h,k,l": print >> self.out, "The data doesn't need to be reindexed! Indexing is consistent between these datasets." else: print >> self.out, "If the data is reindexed with operator (%s), the correlation of"%( self.nice_cb_ops[location].as_hkl() ) print >> self.out, "the intensities to the reference data is maximized. " if self.file_name is not None: print self.file_name print >> self.out, "This can be done for instance with:" print >> self.out, " phenix.reflection_file_converter %s --change_of_basis=\"%s\" <output_options> "%(self.file_name, self.nice_cb_ops[location].as_hkl() ) print >> self.out, "-------------------------------------------------------------------------------" ## change things in primitive setting transform_b = self.set_b_ori.change_basis( self.set_b_to_niggli ).change_basis( self.ops_in_niggli_setting[location] ).change_basis( self.set_b_to_niggli.inverse() ).map_to_asu().set_observation_type( self.set_b_ori) return ( transform_b )
def __str__(self): rows = [] rows.append([ "unit_cell", "volume", "volume score", "#indexed", "% indexed", "% indexed score", "rmsd_xy", "rmsd_xy score", "overall score", ]) score_by_fraction_indexed = self.score_by_fraction_indexed() score_by_volume = self.score_by_volume() score_by_rmsd_xy = self.score_by_rmsd_xy() combined_scores = self.combined_scores() perm = flex.sort_permutation(combined_scores) rmsd_x, rmsd_y, rmsd_z = flex.vec3_double( s.rmsds for s in self.all_solutions).parts() rmsd_xy = flex.sqrt(flex.pow2(rmsd_x) + flex.pow2(rmsd_y)) for i in perm: s = self.all_solutions[i] rows.append([ format( s.crystal.get_unit_cell(), "{:.2f} {:.2f} {:.2f} {:.1f} {:.1f} {:.1f}", ), "%.0f" % s.crystal.get_unit_cell().volume(), "%.2f" % score_by_volume[i], str(s.n_indexed), "%.0f" % (s.fraction_indexed * 100), "%.2f" % score_by_fraction_indexed[i], "%.2f" % rmsd_xy[i], "%.2f" % score_by_rmsd_xy[i], "%.2f" % combined_scores[i], ]) from libtbx import table_utils return table_utils.format(rows=rows, has_header=True)
def _create_flag_count_table(table): """Generate a summary table of flag values in a reflection table. :param table: A reflection table :returns: A string of the formatted flags table """ # Calculate the counts of entries that match each flag numpy_flags = table["flags"].as_numpy_array() flag_count = { flag: numpy.sum(numpy_flags & value != 0) for value, flag in table.flags.values.items() } # Work out the numeric-value order of the flags flag_order = sorted(table.flags.values.values(), key=lambda x: x.real) # Build the actual table flag_rows = [["Flag", "Count", "%"]] max_count_len = max(5, len(str(max(flag_count.values())))) last_flag = None for flag in flag_order: indent = "" # As a hint for reading, indent any 'summary' flags. # A summary flag is any flag which overlaps with the previous one. if last_flag and (last_flag.real & flag.real): indent = " " last_flag = flag # Add the row to the table we're building flag_rows.append( [ indent + flag.name, "{:{:d}d}".format(flag_count[flag], max_count_len), "{:5.01f}".format(100 * flag_count[flag] / len(table)), ] ) # Build the array of output strings text = [] text.append("Reflection flags:") text.append( table_utils.format(flag_rows, has_header=True, prefix="| ", postfix=" |") ) return "\n".join(text)
def write_output(self): rows = [["cluster_id", "# datasets", "height", "datasets"]] for cid in sorted(self._cluster_dict.keys()): cluster = self._cluster_dict[cid] datasets = cluster['datasets'] rows.append([ str(cid), str(len(datasets)), '%.2f' % cluster['height'], ' '.join(['%s'] * len(datasets)) % tuple(datasets) ]) with open('intensity_clustering.txt', 'wb') as f: from libtbx import table_utils print >> f, table_utils.format(rows, has_header=True, prefix="|", postfix="|")
def labelit_printout(self): from libtbx import table_utils table_data = [[ "Solution", "Metric fit", "rmsd", "min/max cc", "#spots", "lattice", "unit_cell", "volume", "cb_op", ]] for item in self: uc = item.refined_crystal.get_unit_cell() P = uc.parameters() min_max_cc_str = "-/-" if item.min_cc is not None and item.max_cc is not None: min_max_cc_str = "%.3f/%.3f" % (item.min_cc, item.max_cc) if item.recommended: status = "*" else: status = "" table_data.append([ "%1s%7d" % (status, item.setting_number), "%(max_angular_difference)6.4f" % item, "%5.3f" % item.rmsd, min_max_cc_str, "%d" % item.Nmatches, "%(bravais)s" % item, "%6.2f %6.2f %6.2f %6.2f %6.2f %6.2f" % P, "%.0f" % uc.volume(), "%s" % item["cb_op_inp_best"].as_abc(), ]) output = table_utils.format(table_data, has_header=1, justify="right", delim=" ") output = output + "\n* = recommended solution\n" return output
def __str__(self): rows = [] rows.append([ "unit_cell", "volume", "n_indexed", "fraction_indexed", "likelihood" ]) for i, s in enumerate(self.all_solutions): s = self.all_solutions[i] rows.append([ format( s.crystal.get_unit_cell(), "{:.2f} {:.2f} {:.2f} {:.1f} {:.1f} {:.1f}", ), "%.0f" % s.crystal.get_unit_cell().volume(), str(s.n_indexed), "%.0f" % (s.fraction_indexed * 100), "%.2f" % s.model_likelihood, ]) from libtbx import table_utils return table_utils.format(rows=rows, has_header=True)
def make_log_beam_stop(self, log_message, flags): self.norma_work = self.norma_work.map_to_asu() self.miller_obs = self.miller_obs.map_to_asu() flags = flags.map_to_asu() data = self.miller_obs.select(~flags.data()) evals = self.norma_work.select(~flags.data()) header = ("Index", "d-spacing", "F_obs", "E-value", "centric") table = "No outliers were found" rows = [] if data.data().size() > 0: if data.data().size() < 500: for hkl, d, fobs, e, c in zip(data.indices(), data.d_spacings().data(), data.data(), evals.data(), data.centric_flags().data()): this_row = [ str(hkl), "%4.2f" % (d), "%6.1f" % (fobs), "%4.2f" % (math.sqrt(e)), str(c) ] rows.append(this_row) table = table_utils.format([header] + rows, comments=None, has_header=True, separate_rows=False, prefix='| ', postfix=' |') else: table = """Over 500 outliers have been found.""" final = log_message + "\n" + table + "\n \n" return final
def make_log_model(self, log_message, flags, ll_gain, p_values, e_obs, e_calc, sigmaa, plot_out=None): header = ("Index", "d-spacing", "E_obs", "E_model", "Score", "p-value", "sigmaa", "centric") table = "No outliers were found" rows = [] rogues = e_obs.select(~flags.data()) p_array = p_values.select(~flags.data()) ll_array = ll_gain.select(~flags.data()) ec_array = e_calc.select(~flags.data()) sa_array = sigmaa.select(~flags.data()) centric_flags = self.miller_obs.centric_flags().select(~flags.data()) if rogues.indices().size() > 0: if rogues.indices().size() < 500: sigmas = rogues.sigmas() if rogues.sigmas() == None: sigmas = rogues.d_spacings().data() * 0 + 10.0 for hkl, d, eo, ec, llg, p, sa, c, s, e in zip( rogues.indices(), rogues.d_spacings().data(), rogues.data(), ec_array.data(), ll_array.data(), p_array.data(), sa_array.data(), centric_flags.data(), sigmas, rogues.epsilons().data().as_double(), ): this_row = [ str(hkl), "%4.2f" % (d), "%6.3f" % (eo), "%6.3f" % (ec), "%5.2f" % (llg), "%5.3e" % (p), "%4.3f" % (sa), str(c), ] rows.append(this_row) if plot_out is not None: outlier_plots.plotit( fobs=eo, sigma=s, fcalc=ec, alpha=sa, beta=1.0 - sa * sa, epsilon=e, centric=c, out=plot_out, plot_title=str(hkl), ) table = table_utils.format( [header] + rows, comments=None, has_header=True, separate_rows=False, prefix="| ", postfix=" |" ) else: table = "More then 500 outliers were found. This is very suspicious. Check data or limits." final = log_message + "\n" + table return final
def run(args): import libtbx.load_env usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, read_reflections=True, read_datablocks=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message) from libtbx.utils import Sorry params, options = parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) if not any([reflections, experiments, datablocks]): parser.print_help() return if len(reflections) != 1: raise Sorry('exactly 1 reflection table must be specified') if len(datablocks) != 1: if experiments: if len(experiments.imagesets()) != 1: raise Sorry('exactly 1 datablock must be specified') imageset = experiments.imagesets()[0] else: raise Sorry('exactly 1 datablock must be specified') else: imageset = datablocks[0].extract_imagesets()[0] reflections = reflections[0] if params.id is not None: reflections = reflections.select(reflections['id'] == params.id) stats = per_image_analysis.stats_imageset( imageset, reflections, resolution_analysis=params.resolution_analysis, plot=params.individual_plots) per_image_analysis.print_table(stats) from libtbx import table_utils overall_stats = per_image_analysis.stats_single_image( imageset, reflections, resolution_analysis=params.resolution_analysis) rows = [ ("Overall statistics", ""), ("#spots", "%i" %overall_stats.n_spots_total), ("#spots_no_ice", "%i" %overall_stats.n_spots_no_ice), #("total_intensity", "%.0f" %overall_stats.total_intensity), ("d_min", "%.2f" %overall_stats.estimated_d_min), ("d_min (distl method 1)", "%.2f (%.2f)" %( overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ("d_min (distl method 2)", "%.2f (%.2f)" %( overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ] print table_utils.format(rows, has_header=True, prefix="| ", postfix=" |") if params.json is not None: import json with open(params.json, 'wb') as fp: json.dump(stats.__dict__, fp) if params.plot is not None: per_image_analysis.plot_stats(stats, filename=params.plot)
if col_count1==0: raise Exception("no reflections in any bins") for i_bin in miller_set_avg.binner().range_used(): col_count = '%8d' % results.count_frames( appropriate_min_corr, miller_set_avg.binner().selection(i_bin)) col_legend = '%-13s' % miller_set_avg.binner().bin_legend( i_bin=i_bin, show_bin_number=False, show_bin_range=False, show_d_range=True, show_counts=False) xpercent = results.count_frames(appropriate_min_corr, miller_set_avg.binner().selection(i_bin))/float(col_count1) percent = '%5.2f'% (100.*xpercent) table_data.append(['%3d' % i_bin, col_legend, col_count,percent]) n_frames = (scaler.frames['cc'] > appropriate_min_corr).count(True) table_data.append([""] * len(table_data[0])) table_data.append(["All", "", '%8d' % n_frames]) print >> out print >> out, table_utils.format( table_data, has_header=1, justify='center', delim=' ') reindexing_ops = {"h,k,l":0} # get a list of all reindexing ops for this dataset if work_params.merging.reverse_lookup is not None: for key in scaler.reverse_lookup: if reindexing_ops.get(scaler.reverse_lookup[key], None) is None: reindexing_ops[scaler.reverse_lookup[key]]=0 reindexing_ops[scaler.reverse_lookup[key]]+=1 from xfel.cxi.cxi_cc import run_cc for key in reindexing_ops.keys(): run_cc(work_params,reindexing_op=key,output=out) return result if (__name__ == "__main__"):
def print_table_2(self): from libtbx import table_utils from libtbx.str_utils import format_value table_header = ["Tile","Dist","Nobs","aRmsd","Rmsd","delx","dely","disp","rotdeg", "Rsigma","Tsigma","Transx","Transy","DelRot","Rotdeg"] table_data = [] table_data.append(table_header) sort_radii = flex.sort_permutation(flex.double(self.radii)) tile_rmsds = flex.double() radial_sigmas = flex.double(len(self.tiles) // 4) tangen_sigmas = flex.double(len(self.tiles) // 4) wtaveg = [0.]*(len(self.tiles) // 4) for x in range(len(self.tiles) // 4): if self.tilecounts[x] >= 3: wtaveg[x] = self.weighted_average_angle_deg_from_tile(x, self.post_mean_cv[x], self.correction_vector_x, self.correction_vector_y) def add_line_to_table(idx): x = sort_radii[idx] if self.tilecounts[x] < 3: radial = (0,0) tangential = (0,0) rmean,tmean,rsigma,tsigma=(0,0,1,1) else: radial,tangential,rmean,tmean,rsigma,tsigma = get_radial_tangential_vectors(self,x, self.post_mean_cv[x], self.correction_vector_x, self.correction_vector_y, self.model_calcx-self.refined_cntr_x, self.model_calcy-self.refined_cntr_y) table_data.append( [ format_value("%3d", x), format_value("%7.2f", self.radii[x]), format_value("%6d", self.tilecounts[x]), format_value("%5.2f", self.asymmetric_tile_rmsd[x]), format_value("%5.2f", self.tile_rmsd[x]), format_value("%5.2f", self.post_mean_cv[x][0]), format_value("%5.2f", self.post_mean_cv[x][1]), format_value("%5.2f", matrix.col(self.post_mean_cv[x]).length()), format_value("%6.2f", wtaveg[x]), format_value("%6.2f", rsigma), format_value("%6.2f", tsigma), format_value("%5.2f", self.tile_translations.x[2*x]), format_value("%5.2f", self.tile_translations.x[2*x+1]), "", format_value("%5.2f", self.tile_rotations.x[x]) ]) # order the printout by sensor, starting from innermost new_order = [] mutable = list(sort_radii) idx = 0 unit_translation_increments = flex.double(len(mutable)*2) while 1: if idx >= len(mutable): break if self.radii[mutable[idx]]==0.0: idx+=1; continue tile_select = mutable[idx] if tile_select%2 == 0: # even sensor_tiles = (tile_select, tile_select+1) sensor_ptrs = (idx, mutable.index(tile_select+1)) else: # odd sensor_tiles = (tile_select-1, tile_select) sensor_ptrs = ( mutable.index(tile_select-1), idx) if self.tilecounts[mutable[sensor_ptrs[0]]] + self.tilecounts[mutable[sensor_ptrs[1]]] < \ self.params.min_count: idx+=1 continue sum_weight = 0.0 sum_wt_x = 0.0 sum_wt_y = 0.0 for iptr, ptr in enumerate(sensor_ptrs): if ptr in new_order: break if self.tilecounts[mutable[ptr]] > 0: #print mutable[ptr] add_line_to_table (ptr) sum_weight += self.tilecounts[mutable[ptr]] sum_wt_x += self.tilecounts[mutable[ptr]] * self.tile_translations.x[2*mutable[ptr]] sum_wt_y += self.tilecounts[mutable[ptr]] * self.tile_translations.x[2*mutable[ptr]+1] new_order.append(ptr) if iptr==1: #print sensor_line = [""]*len(table_header) sensor_line[2]="%6d"%sum_weight sensor_line[11]="%5.2f"%round(sum_wt_x/sum_weight,0) sensor_line[12]="%5.2f"%round(sum_wt_y/sum_weight,0) unit_translation_increments[2*mutable[ptr]-2] = round(sum_wt_x/sum_weight,0) unit_translation_increments[2*mutable[ptr]-1] = round(sum_wt_y/sum_weight,0) unit_translation_increments[2*mutable[ptr]] = round(sum_wt_x/sum_weight,0) unit_translation_increments[2*mutable[ptr]+1] = round(sum_wt_y/sum_weight,0) table_data.append(sensor_line) table_data.append([""]*len(table_header)) idx+=1 if idx>=len(mutable): break print "Grouped by sensor, listing lowest Q-angle first:" print table_utils.format(table_data,has_header=1,justify='center',delim=" ") return unit_translation_increments
def print_table(self): from libtbx import table_utils from libtbx.str_utils import format_value table_header = ["Tile","Dist","Nobs","aRmsd","Rmsd","delx","dely","disp","rotdeg", "Rsigma","Tsigma","Transx","Transy","DelRot","Rotdeg"] table_data = [] table_data.append(table_header) sort_radii = flex.sort_permutation(flex.double(self.radii)) tile_rmsds = flex.double() radial_sigmas = flex.double(len(self.tiles) // 4) tangen_sigmas = flex.double(len(self.tiles) // 4) wtaveg = [0.]*(len(self.tiles) // 4) for x in range(len(self.tiles) // 4): if self.tilecounts[x] >= 3: wtaveg[x] = self.weighted_average_angle_deg_from_tile(x, self.post_mean_cv[x], self.correction_vector_x, self.correction_vector_y) for idx in range(len(self.tiles) // 4): x = sort_radii[idx] if self.tilecounts[x] < 3: radial = (0,0) tangential = (0,0) rmean,tmean,rsigma,tsigma=(0,0,1,1) else: radial,tangential,rmean,tmean,rsigma,tsigma = get_radial_tangential_vectors(self,x, self.post_mean_cv[x], self.correction_vector_x, self.correction_vector_y, self.model_calcx-self.refined_cntr_x, self.model_calcy-self.refined_cntr_y) # paired rotations of two ASICS on the same sensor if x%2==0: # previous method: delrot = "%5.2f"%(wtaveg[x]-wtaveg[x+1]) delrot = "%5.2f"%(self.tile_rotations.x[x] - self.tile_rotations.x[1+x]) else: delrot = "" radial_sigmas[x]=rsigma tangen_sigmas[x]=tsigma table_data.append( [ format_value("%3d", x), format_value("%7.2f", self.radii[x]), format_value("%6d", self.tilecounts[x]), format_value("%5.2f", self.asymmetric_tile_rmsd[x]), format_value("%5.2f", self.tile_rmsd[x]), format_value("%5.2f", self.post_mean_cv[x][0]), format_value("%5.2f", self.post_mean_cv[x][1]), format_value("%5.2f", matrix.col(self.post_mean_cv[x]).length()), format_value("%6.2f", wtaveg[x]), format_value("%6.2f", rsigma), format_value("%6.2f", tsigma), format_value("%5.2f", self.tile_translations.x[2*x]), format_value("%5.2f", self.tile_translations.x[2*x+1]), copy.copy(delrot), format_value("%5.2f", self.tile_rotations.x[x]) ]) table_data.append([""]*len(table_header)) rstats = flex.mean_and_variance(radial_sigmas,self.tilecounts.as_double()) tstats = flex.mean_and_variance(tangen_sigmas,self.tilecounts.as_double()) table_data.append( [ format_value("%3s", "ALL"), format_value("%s", ""), format_value("%6d", self.overall_N), format_value("%5.2f", math.sqrt(flex.mean(self.delrsq))), format_value("%5.2f", self.overall_rmsd), format_value("%5.2f", self.overall_cv[0]), format_value("%5.2f", self.overall_cv[1]), format_value("%5.2f", flex.mean(flex.double([cv.length() for cv in self.post_mean_cv]))), format_value("%s", ""), format_value("%6.2f", rstats.mean()), format_value("%6.2f", tstats.mean()), format_value("%s", ""), format_value("%s", ""), #root mean squared difference in same-sensor (adjacent)-ASIC rotations, weighted by minimum # of observations on either ASIC of the sensor format_value("%5.2f", math.sqrt( flex.sum( flex.double([ (min([self.tilecounts[2*isen],self.tilecounts[2*isen+1]])) * (self.tile_rotations.x[2*isen] - self.tile_rotations.x[1+2*isen])**2 for isen in xrange(len(self.tiles) // 8)] ) )/ flex.sum( flex.double( [(min([self.tilecounts[2*isen],self.tilecounts[2*isen+1]])) for isen in xrange(len(self.tiles) // 8)] ) ) )), format_value("%s", ""), ]) print print table_utils.format(table_data,has_header=1,justify='center',delim=" ")
def show_reflections( reflections, show_intensities=False, show_profile_fit=False, show_centroids=False, show_all_reflection_data=False, show_flags=False, max_reflections=None, show_identifiers=False, ): text = [] import collections from orderedset import OrderedSet formats = collections.OrderedDict( ( ("miller_index", "%i, %i, %i"), ("d", "%.2f"), ("qe", "%.3f"), ("dqe", "%.3f"), ("id", "%i"), ("imageset_id", "%i"), ("panel", "%i"), ("flags", "%i"), ("background.mean", "%.1f"), ("background.dispersion", "%.1f"), ("background.mse", "%.1f"), ("background.sum.value", "%.1f"), ("background.sum.variance", "%.1f"), ("intensity.prf.value", "%.1f"), ("intensity.prf.variance", "%.1f"), ("intensity.sum.value", "%.1f"), ("intensity.sum.variance", "%.1f"), ("intensity.cor.value", "%.1f"), ("intensity.cor.variance", "%.1f"), ("intensity.scale.value", "%.1f"), ("intensity.scale.variance", "%.1f"), ("Ih_values", "%.1f"), ("lp", "%.3f"), ("num_pixels.background", "%i"), ("num_pixels.background_used", "%i"), ("num_pixels.foreground", "%i"), ("num_pixels.valid", "%i"), ("partial_id", "%i"), ("partiality", "%.4f"), ("profile.correlation", "%.3f"), ("profile.rmsd", "%.3f"), ("xyzcal.mm", "%.2f, %.2f, %.2f"), ("xyzcal.px", "%.2f, %.2f, %.2f"), ("delpsical.rad", "%.3f"), ("delpsical2", "%.3f"), ("delpsical.weights", "%.3f"), ("xyzobs.mm.value", "%.2f, %.2f, %.2f"), ("xyzobs.mm.variance", "%.4e, %.4e, %.4e"), ("xyzobs.px.value", "%.2f, %.2f, %.2f"), ("xyzobs.px.variance", "%.4f, %.4f, %.4f"), ("s1", "%.4f, %.4f, %.4f"), ("s2", "%.4f, %.4f, %.4f"), ("shoebox", "%.1f"), ("rlp", "%.4f, %.4f, %.4f"), ("zeta", "%.3f"), ("x_resid", "%.3f"), ("x_resid2", "%.3f"), ("y_resid", "%.3f"), ("y_resid2", "%.3f"), ("kapton_absorption_correction", "%.3f"), ("kapton_absorption_correction_sigmas", "%.3f"), ("inverse_scale_factor", "%.3f"), ("inverse_scale_factor_variance", "%.3f"), ) ) for rlist in reflections: from dials.array_family import flex from dials.algorithms.shoebox import MaskCode foreground_valid = MaskCode.Valid | MaskCode.Foreground text.append("") text.append("Reflection list contains %i reflections" % (len(rlist))) if len(rlist) == 0: continue rows = [["Column", "min", "max", "mean"]] for k, col in rlist.cols(): if k in formats and "%" not in formats.get(k, "%s"): # Allow blanking out of entries that wouldn't make sense rows.append( [ k, formats.get(k, "%s"), formats.get(k, "%s"), formats.get(k, "%s"), ] ) elif type(col) in (flex.double, flex.int, flex.size_t): if type(col) in (flex.int, flex.size_t): col = col.as_double() rows.append( [ k, formats.get(k, "%s") % flex.min(col), formats.get(k, "%s") % flex.max(col), formats.get(k, "%s") % flex.mean(col), ] ) elif type(col) in (flex.vec3_double, flex.miller_index): if isinstance(col, flex.miller_index): col = col.as_vec3_double() rows.append( [ k, formats.get(k, "%s") % col.min(), formats.get(k, "%s") % col.max(), formats.get(k, "%s") % col.mean(), ] ) elif isinstance(col, flex.shoebox): rows.append([k, "", "", ""]) si = col.summed_intensity().observed_value() rows.append( [ " summed I", formats.get(k, "%s") % flex.min(si), formats.get(k, "%s") % flex.max(si), formats.get(k, "%s") % flex.mean(si), ] ) x1, x2, y1, y2, z1, z2 = col.bounding_boxes().parts() bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double() rows.append( [ " N pix", formats.get(k, "%s") % flex.min(bbox_sizes), formats.get(k, "%s") % flex.max(bbox_sizes), formats.get(k, "%s") % flex.mean(bbox_sizes), ] ) fore_valid = col.count_mask_values(foreground_valid).as_double() rows.append( [ " N valid foreground pix", formats.get(k, "%s") % flex.min(fore_valid), formats.get(k, "%s") % flex.max(fore_valid), formats.get(k, "%s") % flex.mean(fore_valid), ] ) text.append( table_utils.format(rows, has_header=True, prefix="| ", postfix=" |") ) if show_flags: text.append(_create_flag_count_table(rlist)) if show_identifiers: if rlist.experiment_identifiers(): text.append( """Experiment identifiers id-map values:\n%s""" % ( "\n".join( "id:" + str(k) + " -> experiment identifier:" + str(rlist.experiment_identifiers()[k]) for k in rlist.experiment_identifiers().keys() ) ) ) intensity_keys = ( "miller_index", "d", "intensity.prf.value", "intensity.prf.variance", "intensity.sum.value", "intensity.sum.variance", "background.mean", "profile.correlation", "profile.rmsd", ) profile_fit_keys = ("miller_index", "d") centroid_keys = ( "miller_index", "d", "xyzcal.mm", "xyzcal.px", "xyzobs.mm.value", "xyzobs.mm.variance", "xyzobs.px.value", "xyzobs.px.variance", ) keys_to_print = OrderedSet() if show_intensities: for k in intensity_keys: keys_to_print.add(k) if show_profile_fit: for k in profile_fit_keys: keys_to_print.add(k) if show_centroids: for k in centroid_keys: keys_to_print.add(k) if show_all_reflection_data: for k in formats: keys_to_print.add(k) def format_column(key, data, format_strings=None): if isinstance(data, flex.vec3_double): c_strings = [ c.as_string(format_strings[i].strip()) for i, c in enumerate(data.parts()) ] elif isinstance(data, flex.miller_index): c_strings = [ c.as_string(format_strings[i].strip()) for i, c in enumerate(data.as_vec3_double().parts()) ] elif isinstance(data, flex.size_t): c_strings = [data.as_int().as_string(format_strings[0].strip())] elif isinstance(data, flex.shoebox): x1, x2, y1, y2, z1, z2 = data.bounding_boxes().parts() bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double() c_strings = [bbox_sizes.as_string(format_strings[0].strip())] key += " (N pix)" else: c_strings = [data.as_string(format_strings[0].strip())] column = flex.std_string() max_element_lengths = [c.max_element_length() for c in c_strings] for i in range(len(c_strings[0])): column.append( ("%%%is" % len(key)) % ", ".join( ("%%%is" % max_element_lengths[j]) % c_strings[j][i] for j in range(len(c_strings)) ) ) return column if keys_to_print: keys = [k for k in keys_to_print if k in rlist] if max_reflections is not None: max_reflections = min(len(rlist), max_reflections) else: max_reflections = len(rlist) columns = [] for k in keys: columns.append( format_column(k, rlist[k], format_strings=formats[k].split(",")) ) text.append("") text.append("Printing %i of %i reflections:" % (max_reflections, len(rlist))) line = [] for j in range(len(columns)): key = keys[j] if key == "shoebox": key += " (N pix)" width = max(len(key), columns[j].max_element_length()) line.append("%%%is" % width % key) text.append(" ".join(line)) for i in range(max_reflections): line = [] for j in range(len(columns)): line.append(columns[j][i]) text.append(" ".join(line)) return "\n".join(text)
def print_table(self): from libtbx import table_utils from libtbx.str_utils import format_value table_header = [ "Tile", "Dist", "Nobs", "aRmsd", "Rmsd", "delx", "dely", "disp", "rotdeg", "Rsigma", "Tsigma", "Transx", "Transy", "DelRot", ] table_data = [] table_data.append(table_header) sort_radii = flex.sort_permutation(flex.double(self.radii)) tile_rmsds = flex.double() radial_sigmas = flex.double(64) tangen_sigmas = flex.double(64) wtaveg = [0.0] * 64 for x in xrange(64): if self.tilecounts[x] >= 3: wtaveg[x] = self.weighted_average_angle_deg_from_tile( x, self.post_mean_cv[x], self.correction_vector_x, self.correction_vector_y ) for idx in xrange(64): x = sort_radii[idx] if self.tilecounts[x] < 3: radial = (0, 0) tangential = (0, 0) rmean, tmean, rsigma, tsigma = (0, 0, 1, 1) else: radial, tangential, rmean, tmean, rsigma, tsigma = get_radial_tangential_vectors(self, x) # paired rotations of two ASICS on the same sensor if x % 2 == 0: delrot = "%5.2f" % (wtaveg[x] - wtaveg[x + 1]) else: delrot = "" radial_sigmas[x] = rsigma tangen_sigmas[x] = tsigma table_data.append( [ format_value("%3d", x), format_value("%7.2f", self.radii[x]), format_value("%6d", self.tilecounts[x]), format_value("%5.2f", self.asymmetric_tile_rmsd[x]), format_value("%5.2f", self.tile_rmsd[x]), format_value("%5.2f", self.post_mean_cv[x][0]), format_value("%5.2f", self.post_mean_cv[x][1]), format_value("%5.2f", matrix.col(self.post_mean_cv[x]).length()), format_value("%6.2f", wtaveg[x]), format_value("%6.2f", rsigma), format_value("%6.2f", tsigma), format_value("%5.2f", self.x[2 * x]), format_value("%5.2f", self.x[2 * x + 1]), copy.copy(delrot), ] ) table_data.append([""] * len(table_header)) rstats = flex.mean_and_variance(radial_sigmas, self.tilecounts.as_double()) tstats = flex.mean_and_variance(tangen_sigmas, self.tilecounts.as_double()) table_data.append( [ format_value("%3s", "ALL"), format_value("%s", ""), format_value("%6d", self.overall_N), format_value("%5.2f", math.sqrt(flex.mean(self.delrsq))), format_value("%5.2f", self.overall_rmsd), format_value("%5.2f", self.overall_cv[0]), format_value("%5.2f", self.overall_cv[1]), format_value("%5.2f", flex.mean(flex.double([cv.length() for cv in self.post_mean_cv]))), format_value("%s", ""), format_value("%6.2f", rstats.mean()), format_value("%6.2f", tstats.mean()), format_value("%s", ""), format_value("%s", ""), format_value("%s", ""), ] ) print print table_utils.format(table_data, has_header=1, justify="center", delim=" ")
show_bin_number=False, show_bin_range=False, show_d_range=True, show_counts=False) xpercent = results.count_frames( appropriate_min_corr, miller_set_avg.binner().selection(i_bin)) / float(col_count1) percent = '%5.2f' % (100. * xpercent) table_data.append(['%3d' % i_bin, col_legend, col_count, percent]) n_frames = (scaler.frames['cc'] > appropriate_min_corr).count(True) table_data.append([""] * len(table_data[0])) table_data.append(["All", "", '%8d' % n_frames]) print >> out print >> out, table_utils.format(table_data, has_header=1, justify='center', delim=' ') reindexing_ops = { "h,k,l": 0 } # get a list of all reindexing ops for this dataset if work_params.merging.reverse_lookup is not None: for key in scaler.reverse_lookup: if reindexing_ops.get(scaler.reverse_lookup[key], None) is None: reindexing_ops[scaler.reverse_lookup[key]] = 0 reindexing_ops[scaler.reverse_lookup[key]] += 1 from xfel.cxi.cxi_cc import run_cc for key in reindexing_ops.keys(): run_cc(work_params, reindexing_op=key, output=out)
def run_cc(params, reindexing_op, output): uniform, selected_uniform, have_iso_ref = load_cc_data( params, reindexing_op, output) NBIN = params.output.n_bins if have_iso_ref: slope, offset, corr_iso, N_iso = correlation(selected_uniform[1], selected_uniform[0], params.include_negatives) print >> output, "C.C. iso is %.1f%% on %d indices" % (100 * corr_iso, N_iso) slope, offset, corr_int, N_int = correlation(selected_uniform[2], selected_uniform[3], params.include_negatives) print >> output, "C.C. int is %.1f%% on %d indices" % (100. * corr_int, N_int) if have_iso_ref: binned_cc_ref, binned_cc_ref_N = binned_correlation( selected_uniform[1], selected_uniform[0], params.include_negatives) #binned_cc_ref.show(f=output) ref_scale = scale_factor(selected_uniform[1], selected_uniform[0], weights=flex.pow(selected_uniform[1].sigmas(), -2), use_binning=True) #ref_scale.show(f=output) ref_riso = r1_factor(selected_uniform[1], selected_uniform[0], scale_factor=ref_scale, use_binning=True) #ref_riso.show(f=output) ref_scale_all = scale_factor(selected_uniform[1], selected_uniform[0], weights=flex.pow( selected_uniform[1].sigmas(), -2)) ref_riso_all = r1_factor(selected_uniform[1], selected_uniform[0], scale_factor=ref_scale_all) binned_cc_int, binned_cc_int_N = binned_correlation( selected_uniform[2], selected_uniform[3], params.include_negatives) #binned_cc_int.show(f=output) oe_scale = scale_factor( selected_uniform[2], selected_uniform[3], weights=flex.pow(selected_uniform[2].sigmas(), -2) + flex.pow(selected_uniform[3].sigmas(), -2), use_binning=True) #oe_scale.show(f=output) oe_rint = r1_factor(selected_uniform[2], selected_uniform[3], scale_factor=oe_scale, use_binning=True) #oe_rint.show(f=output) oe_rsplit = r_split(selected_uniform[2], selected_uniform[3], use_binning=True) oe_scale_all = scale_factor( selected_uniform[2], selected_uniform[3], weights=flex.pow(selected_uniform[2].sigmas(), -2) + flex.pow(selected_uniform[3].sigmas(), -2), ) oe_rint_all = r1_factor(selected_uniform[2], selected_uniform[3], scale_factor=oe_scale_all) oe_rsplit_all = r_split(selected_uniform[2], selected_uniform[3]) if have_iso_ref: print >> output, "R factors Riso = %.1f%%, Rint = %.1f%%" % ( 100. * ref_riso_all, 100. * oe_rint_all) else: print >> output, "R factor Rint = %.1f%%" % (100. * oe_rint_all) split_sigma_data = split_sigma_test(selected_uniform[2], selected_uniform[3], scale=oe_scale, use_binning=True, show_plot=False) split_sigma_data_all = split_sigma_test(selected_uniform[2], selected_uniform[3], scale=oe_scale_all, use_binning=False, show_plot=False) print >> output if reindexing_op == "h,k,l": print >> output, "Table of Scaling Results:" else: print >> output, "Table of Scaling Results Reindexing as %s:" % reindexing_op from libtbx import table_utils table_header = [ "", "", "", "CC", " N", "CC", " N", "R", "R", "R", "Scale", "Scale", "SpSig" ] table_header2 = [ "Bin", "Resolution Range", "Completeness", "int", "int", "iso", "iso", "int", "split", "iso", "int", "iso", "Test" ] table_data = [] table_data.append(table_header) table_data.append(table_header2) items = binned_cc_int.binner.range_used() # XXX Make it clear what the completeness here actually is! cumulative_counts_given = 0 cumulative_counts_complete = 0 for bin in items: table_row = [] table_row.append("%3d" % bin) table_row.append("%-13s" % binned_cc_int.binner.bin_legend(i_bin=bin, show_bin_number=False, show_bin_range=False, show_d_range=True, show_counts=False)) table_row.append("%13s" % binned_cc_int.binner.bin_legend(i_bin=bin, show_bin_number=False, show_bin_range=False, show_d_range=False, show_counts=True)) cumulative_counts_given += binned_cc_int.binner._counts_given[bin] cumulative_counts_complete += binned_cc_int.binner._counts_complete[ bin] table_row.append("%.1f%%" % (100. * binned_cc_int.data[bin])) table_row.append("%7d" % (binned_cc_int_N.data[bin])) if have_iso_ref and binned_cc_ref.data[bin] is not None: table_row.append("%.1f%%" % (100 * binned_cc_ref.data[bin])) else: table_row.append("--") if have_iso_ref and binned_cc_ref_N.data[bin] is not None: table_row.append("%6d" % (binned_cc_ref_N.data[bin])) else: table_row.append("--") if oe_rint.data[bin] is not None: table_row.append("%.1f%%" % (100. * oe_rint.data[bin])) else: table_row.append("--") if oe_rsplit.data[bin] is not None: table_row.append("%.1f%%" % (100 * oe_rsplit.data[bin])) else: table_row.append("--") if have_iso_ref and ref_riso.data[bin] is not None: table_row.append("%.1f%%" % (100 * ref_riso.data[bin])) else: table_row.append("--") if oe_scale.data[bin] is not None: table_row.append("%.3f" % oe_scale.data[bin]) else: table_row.append("--") if have_iso_ref and ref_scale.data[bin] is not None: table_row.append("%.3f" % ref_scale.data[bin]) else: table_row.append("--") if split_sigma_data.data[bin] is not None: table_row.append("%.4f" % split_sigma_data.data[bin]) else: table_row.append("--") table_data.append(table_row) table_data.append([""] * len(table_header)) table_row = [ format_value("%3s", "All"), format_value("%-13s", " "), format_value( "%13s", "[%d/%d]" % (cumulative_counts_given, cumulative_counts_complete)), format_value("%.1f%%", 100 * corr_int), format_value("%7d", N_int) ] if have_iso_ref: table_row.extend( (format_value("%.1f%%", 100 * corr_iso), format_value("%6d", N_iso))) else: table_row.extend(("--", "--")) table_row.extend((format_value("%.1f%%", 100 * oe_rint_all), format_value("%.1f%%", 100 * oe_rsplit_all))) if have_iso_ref: table_row.append(format_value("%.1f%%", 100 * ref_riso_all)) else: table_row.append("--") table_row.append(format_value("%.3f", oe_scale_all)) if have_iso_ref: table_row.append(format_value("%.3f", ref_scale_all)) else: table_row.append("--") if split_sigma_data_all is not None: table_row.append("%.1f" % split_sigma_data_all) else: table_row.append("--") table_data.append(table_row) print >> output print >> output, table_utils.format(table_data, has_header=2, justify='center', delim=" ") print >> output, """CCint is the CC-1/2 defined by Diederichs; correlation between odd/even images. Similarly, Scale int and R int are the scaling factor and scaling R factor between odd/even images. "iso" columns compare the whole XFEL dataset to the isomorphous reference.""" print >> output, """Niso: result vs. reference common set""", if params.include_negatives: print >> output, """including negative merged intensities (set by phil parameter).""" elif params.scaling.log_cutoff is None: print >> output else: print >> output, """with intensites < %7.2g filtered out (controlled by scaling.log_cutoff phil parameter set to %5.1f)""" % (math.exp( params.scaling.log_cutoff), params.scaling.log_cutoff) if have_iso_ref: assert N_iso == flex.sum( flex.double([x for x in binned_cc_ref_N.data if x is not None])) assert N_int == flex.sum( flex.double([x for x in binned_cc_int_N.data if x is not None])) if params.scaling.show_plots: from matplotlib import pyplot as plt plt.plot(flex.log(selected_uniform[-2].data()), flex.log(selected_uniform[-1].data()), 'r.') plt.show() if have_iso_ref: plt.plot(flex.log(selected_uniform[0].data()), flex.log(selected_uniform[1].data()), 'r.') plt.show() print >> output
def run(args): sweep_directories = [] templates = [] n_strong_spots = flex.int() n_strong_spots_dmin_4 = flex.int() d_strong_spots_99th_percentile = flex.double() d_strong_spots_95th_percentile = flex.double() d_strong_spots_50th_percentile = flex.double() n_unindexed_spots = flex.int() n_indexed_lattices = flex.int() n_integrated_lattices = flex.int() sweep_dir_cryst = flex.std_string() orig_dir = os.path.abspath(os.curdir) rmsds = flex.vec3_double() cell_params = flex.sym_mat3_double() n_indexed = flex.double() d_min_indexed = flex.double() rmsds = flex.vec3_double() nproc = easy_mp.get_processes(libtbx.Auto) #nproc = 1 results = easy_mp.parallel_map( func=run_once, iterable=args, processes=nproc, method="multiprocessing", preserve_order=True, asynchronous=True, preserve_exception_message=True, ) for result in results: if result is None: continue sweep_directories.append(result.sweep_dir) templates.append(result.template) n_strong_spots.append(result.n_strong_spots) n_strong_spots_dmin_4.append(result.n_strong_spots_dmin_4) n_unindexed_spots.append(result.n_unindexed_spots) n_indexed_lattices.append(result.n_indexed_lattices) n_integrated_lattices.append(result.n_integrated_lattices) d_strong_spots_50th_percentile.append(result.d_strong_spots_50th_percentile) d_strong_spots_95th_percentile.append(result.d_strong_spots_95th_percentile) d_strong_spots_99th_percentile.append(result.d_strong_spots_99th_percentile) cell_params.extend(result.cell_params) n_indexed.extend(result.n_indexed) d_min_indexed.extend(result.d_min_indexed) rmsds.extend(result.rmsds) sweep_dir_cryst.extend(result.sweep_dir_cryst) table_data = [('sweep_dir', 'template', '#strong_spots', '#unindexed_spots', '#lattices', 'd_spacing_50th_percentile', 'd_spacing_95th_percentile', 'd_spacing_99th_percentile',)] for i in range(len(sweep_directories)): table_data.append((sweep_directories[i], templates[i], str(n_strong_spots[i]), str(n_unindexed_spots[i]), str(n_indexed_lattices[i]), str(d_strong_spots_50th_percentile[i]), str(d_strong_spots_95th_percentile[i]), str(d_strong_spots_99th_percentile[i]), )) with open('results.txt', 'wb') as f: print >> f, table_utils.format( table_data, has_header=True, justify='right') table_data = [('sweep_dir', 'cell_a', 'cell_b', 'cell_c', 'alpha', 'beta', 'gamma', '#indexed_reflections', 'd_min_indexed', 'rmsd_x', 'rmsd_y', 'rmsd_phi')] for i in range(len(cell_params)): table_data.append((sweep_dir_cryst[i], str(cell_params[i][0]), str(cell_params[i][1]), str(cell_params[i][2]), str(cell_params[i][3]), str(cell_params[i][4]), str(cell_params[i][5]), str(n_indexed[i]), str(d_min_indexed[i]), str(rmsds[i][0]), str(rmsds[i][1]), str(rmsds[i][2]), )) with open('results_indexed.txt', 'wb') as f: print >> f, table_utils.format( table_data, has_header=True, justify='right') cell_a = flex.double([params[0] for params in cell_params]) cell_b = flex.double([params[1] for params in cell_params]) cell_c = flex.double([params[2] for params in cell_params]) cell_alpha = flex.double([params[3] for params in cell_params]) cell_beta = flex.double([params[4] for params in cell_params]) cell_gamma = flex.double([params[5] for params in cell_params]) from matplotlib import pyplot from matplotlib.backends.backend_pdf import PdfPages pyplot.rc('font', family='serif') pyplot.rc('font', serif='Times New Roman') red, blue = '#B2182B', '#2166AC' hist = flex.histogram(n_strong_spots_dmin_4.as_double(), n_slots=20) hist.show() fig = pyplot.figure() ax = fig.add_subplot(1,1,1) ax.bar(hist.slot_centers(), hist.slots(), width=0.75*hist.slot_width(), color=blue, edgecolor=blue) ax.set_xlabel('Spot count') ax.set_ylabel('Frequency') pdf = PdfPages("spot_count_histogram.pdf") pdf.savefig(fig) pdf.close() #pyplot.show() hist = flex.histogram(n_indexed_lattices.as_double(), n_slots=flex.max(n_indexed_lattices)) hist.show() fig = pyplot.figure() ax = fig.add_subplot(1,1,1) ax.bar(range(int(hist.data_max())), hist.slots(), width=0.75*hist.slot_width(), align='center', color=blue, edgecolor=blue) ax.set_xlim(-0.5, hist.data_max()-0.5) ax.set_xticks(range(0,int(hist.data_max()))) ax.set_xlabel('Number of indexed lattices') ax.set_ylabel('Frequency') pdf = PdfPages("n_indexed_lattices_histogram.pdf") pdf.savefig(fig) pdf.close() #pyplot.show() if flex.max(n_integrated_lattices) > 0: hist = flex.histogram(n_integrated_lattices.as_double(), n_slots=flex.max(n_integrated_lattices)) hist.show() fig = pyplot.figure() ax = fig.add_subplot(1,1,1) ax.bar(range(int(hist.data_max())), hist.slots(), width=0.75*hist.slot_width(), align='center', color=blue, edgecolor=blue) ax.set_xlim(-0.5, hist.data_max()-0.5) ax.set_xticks(range(0,int(hist.data_max()))) ax.set_xlabel('Number of integrated lattices') ax.set_ylabel('Frequency') pdf = PdfPages("n_integrated_lattices_histogram.pdf") pdf.savefig(fig) pdf.close() #pyplot.show() fig, axes = pyplot.subplots(nrows=2, ncols=3, squeeze=False) for i, cell_param in enumerate( (cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma)): ax = axes.flat[i] flex.min_max_mean_double(cell_param).show() print flex.median(cell_param) hist = flex.histogram(cell_param, n_slots=20) hist.show() ax.bar(hist.slot_centers(), hist.slots(), width=0.75*hist.slot_width(), color=blue, edgecolor=blue) ax.set_xlabel('Cell parameter') ax.set_ylabel('Frequency') pyplot.tight_layout() pdf = PdfPages("cell_parameters.pdf") pdf.savefig(fig) pdf.close()
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_experiments from dials.util.options import flatten_datablocks from dials.util.options import flatten_reflections import libtbx.load_env usage = "%s [options] datablock.json | experiments.json | image_*.cbf" %( libtbx.env.dispatcher_name) parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_datablocks=True, read_datablocks_from_images=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 and len(experiments) == 0 and len(reflections) == 0: parser.print_help() exit() for i_expt, expt in enumerate(experiments): print "Experiment %i:" %i_expt print str(expt.detector) print 'Max resolution (at corners): %f' % ( expt.detector.get_max_resolution(expt.beam.get_s0())) print 'Max resolution (inscribed): %f' % ( expt.detector.get_max_inscribed_resolution(expt.beam.get_s0())) if params.show_panel_distance: for ipanel, panel in enumerate(expt.detector): from scitbx import matrix fast = matrix.col(panel.get_fast_axis()) slow = matrix.col(panel.get_slow_axis()) normal = fast.cross(slow) origin = matrix.col(panel.get_origin()) distance = origin.dot(normal) fast_origin = - (origin - distance * normal).dot(fast) slow_origin = - (origin - distance * normal).dot(slow) print 'Panel %d: distance %.2f origin %.2f %.2f' % \ (ipanel, distance, fast_origin, slow_origin) print '' print '' panel_id, (x, y) = beam_centre(expt.detector, expt.beam) if panel_id >= 0 and x is not None and y is not None: if len(expt.detector) > 1: beam_centre_str = "Beam centre: panel %i, (%.2f,%.2f)" %(panel_id, x, y) else: beam_centre_str = "Beam centre: (%.2f,%.2f)" %(x, y) else: beam_centre_str = "" print str(expt.beam) + beam_centre_str + '\n' if expt.scan is not None: print expt.scan if expt.goniometer is not None: print expt.goniometer expt.crystal.show(show_scan_varying=params.show_scan_varying) if expt.crystal.num_scan_points: from scitbx.array_family import flex from cctbx import uctbx abc = flex.vec3_double() angles = flex.vec3_double() for n in range(expt.crystal.num_scan_points): a, b, c, alpha, beta, gamma = expt.crystal.get_unit_cell_at_scan_point(n).parameters() abc.append((a, b, c)) angles.append((alpha, beta, gamma)) a, b, c = abc.mean() alpha, beta, gamma = angles.mean() mean_unit_cell = uctbx.unit_cell((a, b, c, alpha, beta, gamma)) print " Average unit cell: %s" %mean_unit_cell print for datablock in datablocks: if datablock.format_class() is not None: print 'Format: %s' %datablock.format_class() imagesets = datablock.extract_imagesets() for imageset in imagesets: try: print imageset.get_template() except Exception: pass detector = imageset.get_detector() print str(detector) + 'Max resolution: %f\n' %( detector.get_max_resolution(imageset.get_beam().get_s0())) if params.show_panel_distance: for ipanel, panel in enumerate(detector): from scitbx import matrix fast = matrix.col(panel.get_fast_axis()) slow = matrix.col(panel.get_slow_axis()) normal = fast.cross(slow) origin = matrix.col(panel.get_origin()) distance = origin.dot(normal) fast_origin = - (origin - distance * normal).dot(fast) slow_origin = - (origin - distance * normal).dot(slow) print 'Panel %d: distance %.2f origin %.2f %.2f' % \ (ipanel, distance, fast_origin, slow_origin) print '' panel_id, (x, y) = beam_centre(detector, imageset.get_beam()) if panel_id >= 0 and x is not None and y is not None: if len(detector) > 1: beam_centre_str = "Beam centre: panel %i, (%.2f,%.2f)" %(panel_id, x, y) else: beam_centre_str = "Beam centre: (%.2f,%.2f)" %(x, y) else: beam_centre_str = "" print str(imageset.get_beam()) + beam_centre_str + '\n' if imageset.get_scan() is not None: print imageset.get_scan() if imageset.get_goniometer() is not None: print imageset.get_goniometer() from libtbx.containers import OrderedDict, OrderedSet formats = OrderedDict([ ('miller_index', '%i, %i, %i'), ('d','%.2f'), ('dqe','%.3f'), ('id','%i'), ('imageset_id','%i'), ('panel','%i'), ('flags', '%i'), ('background.mean', '%.1f'), ('background.dispersion','%.1f'), ('background.mse', '%.1f'), ('background.sum.value', '%.1f'), ('background.sum.variance', '%.1f'), ('intensity.prf.value','%.1f'), ('intensity.prf.variance','%.1f'), ('intensity.sum.value','%.1f'), ('intensity.sum.variance','%.1f'), ('intensity.cor.value','%.1f'), ('intensity.cor.variance','%.1f'), ('lp','%.3f'), ('num_pixels.background','%i'), ('num_pixels.background_used','%i'), ('num_pixels.foreground','%i'), ('num_pixels.valid','%i'), ('partial_id','%i'), ('partiality','%.4f'), ('profile.correlation','%.3f'), ('profile.rmsd','%.3f'), ('xyzcal.mm','%.2f, %.2f, %.2f'), ('xyzcal.px','%.2f, %.2f, %.2f'), ('delpsical.rad','%.3f'), ('delpsical2','%.3f'), ('xyzobs.mm.value','%.2f, %.2f, %.2f'), ('xyzobs.mm.variance','%.4e, %.4e, %.4e'), ('xyzobs.px.value','%.2f, %.2f, %.2f'), ('xyzobs.px.variance','%.4f, %.4f, %.4f'), ('s1','%.4f, %.4f, %.4f'), ('rlp','%.4f, %.4f, %.4f'), ('zeta','%.3f'), ('x_resid','%.3f'), ('x_resid2','%.3f'), ('y_resid','%.3f'), ('y_resid2','%.3f'), ]) for rlist in reflections: from cctbx.array_family import flex print print "Reflection list contains %i reflections" %(len(rlist)) rows = [["Column", "min", "max", "mean"]] for k, col in rlist.cols(): if type(col) in (flex.double, flex.int, flex.size_t): if type(col) in (flex.int, flex.size_t): col = col.as_double() rows.append([k, formats[k] %flex.min(col), formats[k] %flex.max(col), formats[k]%flex.mean(col)]) elif type(col) in (flex.vec3_double, flex.miller_index): if type(col) == flex.miller_index: col = col.as_vec3_double() rows.append([k, formats[k] %col.min(), formats[k] %col.max(), formats[k]%col.mean()]) from libtbx import table_utils print table_utils.format(rows, has_header=True, prefix="| ", postfix=" |") intensity_keys = ( 'miller_index', 'd', 'intensity.prf.value', 'intensity.prf.variance', 'intensity.sum.value', 'intensity.sum.variance', 'background.mean', 'profile.correlation', 'profile.rmsd' ) profile_fit_keys = ('miller_index', 'd',) centroid_keys = ( 'miller_index', 'd', 'xyzcal.mm', 'xyzcal.px', 'xyzobs.mm.value', 'xyzobs.mm.variance', 'xyzobs.px.value', 'xyzobs.px.variance' ) keys_to_print = OrderedSet() if params.show_intensities: for k in intensity_keys: keys_to_print.add(k) if params.show_profile_fit: for k in profile_fit_keys: keys_to_print.add(k) if params.show_centroids: for k in centroid_keys: keys_to_print.add(k) if params.show_all_reflection_data: for k in formats: keys_to_print.add(k) def format_column(key, data, format_strings=None): if isinstance(data, flex.vec3_double): c_strings = [c.as_string(format_strings[i].strip()) for i, c in enumerate(data.parts())] elif isinstance(data, flex.miller_index): c_strings = [c.as_string(format_strings[i].strip()) for i, c in enumerate(data.as_vec3_double().parts())] elif isinstance(data, flex.size_t): c_strings = [data.as_int().as_string(format_strings[0].strip())] else: c_strings = [data.as_string(format_strings[0].strip())] column = flex.std_string() max_element_lengths = [c.max_element_length() for c in c_strings] for i in range(len(c_strings[0])): column.append(('%%%is' %len(key)) %', '.join( ('%%%is' %max_element_lengths[j]) %c_strings[j][i] for j in range(len(c_strings)))) return column if keys_to_print: keys = [k for k in keys_to_print if k in rlist] rows = [keys] max_reflections = len(rlist) if params.max_reflections is not None: max_reflections = min(len(rlist), params.max_reflections) columns = [] for k in keys: columns.append(format_column(k, rlist[k], format_strings=formats[k].split(','))) print print "Printing %i of %i reflections:" %(max_reflections, len(rlist)) for j in range(len(columns)): key = keys[j] width = max(len(key), columns[j].max_element_length()) print ("%%%is" %width) %key, print for i in range(max_reflections): for j in range(len(columns)): print columns[j][i], print return
def get_table_text(self): '''Produce formatted table text''' table_header = ["","","","","<asu","<obs","<pred","","","","",""] table_header2 = ["Bin","Resolution Range","Completeness","%","multi>","multi>","multi>", "n_meas", "asu_m_meas", "n_pred","<I>","<I/sig(I)>"] use_preds = False # TODO? include_columns = [True, True, True, True, True, True, use_preds, True, True, use_preds, True, True] table_data = [] table_data.append(table_header) table_data.append(table_header2) for bin in self.table: table_row = [] table_row.append("%3d" % bin.i_bin) table_row.append("%-13s" % bin.d_range) table_row.append("%13s" % bin.complete_tag) table_row.append("%5.2f" % (100*bin.completeness)) table_row.append("%6.2f" % bin.redundancy_asu) table_row.append("%6.2f" % bin.redundancy_obs) table_row.append("%6.2f" % (0)) # if redundancy_to_edge is None else bin.redundancy_to_edge)) table_row.append("%6d" % bin.measurements) table_row.append("%6d" % bin.multiply_measured_asu) table_row.append("%6d" % (0)) # if redundancy_to_edge is None else flex.sum(bin.predictions))) table_row.append("%8.0f" % bin.mean_I) table_row.append("%8.3f" % bin.mean_I_sigI) table_data.append(table_row) if len(table_data) <= 2: return ("Intensity statistics table could not be constructed -- no bins accepted.") table_data.append([""] * len(table_header)) total_completeness = 0 total_asu_multiplicity = 0 if self.cumulative_theor_asu_count > 0: total_completeness = 100 * (self.cumulative_observed_asu_count / self.cumulative_theor_asu_count) total_asu_multiplicity = self.cumulative_observed_count / self.cumulative_theor_asu_count total_obs_multiplicity = 0 if self.cumulative_observed_asu_count: total_obs_multiplicity = self.cumulative_observed_count / self.cumulative_observed_asu_count total_mean_I = 0 total_mean_Isigma = 0 if self.cumulative_multiply_observed_asu_count > 0: total_mean_I = self.cumulative_I / self.cumulative_multiply_observed_asu_count total_mean_Isigma = self.cumulative_Isigma / self.cumulative_multiply_observed_asu_count table_data.append( [ format_value("%3s", "All"), format_value("%-13s", " "), format_value("%13s", "[%d/%d]"%(self.cumulative_observed_asu_count, self.cumulative_theor_asu_count)), format_value("%5.2f", total_completeness), format_value("%6.2f", total_asu_multiplicity), format_value("%6.2f", total_obs_multiplicity), format_value("%6.2f", (0)), # if redundancy_to_edge is None else cumulative_n_pred/cumulative_theor)), format_value("%6d", self.cumulative_observed_count), format_value("%6d", self.cumulative_multiply_observed_asu_count), format_value("%6d", (0)), #if redundancy_to_edge is None else flex.sum(redundancy_to_edge))), format_value("%8.0f", total_mean_I), format_value("%8.3f", total_mean_Isigma), ]) table_data = table_utils.manage_columns(table_data, include_columns) return table_utils.format(table_data, has_header = 2, justify ='center', delim = " ")
def run(args): sweep_directories = [] templates = [] n_strong_spots = flex.int() n_strong_spots_dmin_4 = flex.int() d_strong_spots_99th_percentile = flex.double() d_strong_spots_95th_percentile = flex.double() d_strong_spots_50th_percentile = flex.double() n_unindexed_spots = flex.int() n_indexed_lattices = flex.int() n_integrated_lattices = flex.int() sweep_dir_cryst = flex.std_string() orig_dir = os.path.abspath(os.curdir) rmsds = flex.vec3_double() cell_params = flex.sym_mat3_double() n_indexed = flex.double() d_min_indexed = flex.double() rmsds = flex.vec3_double() nproc = easy_mp.get_processes(libtbx.Auto) # nproc = 1 results = easy_mp.parallel_map( func=run_once, iterable=args, processes=nproc, method="multiprocessing", preserve_order=True, asynchronous=True, preserve_exception_message=True, ) for result in results: if result is None: continue sweep_directories.append(result.sweep_dir) templates.append(result.template) n_strong_spots.append(result.n_strong_spots) n_strong_spots_dmin_4.append(result.n_strong_spots_dmin_4) n_unindexed_spots.append(result.n_unindexed_spots) n_indexed_lattices.append(result.n_indexed_lattices) n_integrated_lattices.append(result.n_integrated_lattices) d_strong_spots_50th_percentile.append( result.d_strong_spots_50th_percentile) d_strong_spots_95th_percentile.append( result.d_strong_spots_95th_percentile) d_strong_spots_99th_percentile.append( result.d_strong_spots_99th_percentile) cell_params.extend(result.cell_params) n_indexed.extend(result.n_indexed) d_min_indexed.extend(result.d_min_indexed) rmsds.extend(result.rmsds) sweep_dir_cryst.extend(result.sweep_dir_cryst) table_data = [( "sweep_dir", "template", "#strong_spots", "#unindexed_spots", "#lattices", "d_spacing_50th_percentile", "d_spacing_95th_percentile", "d_spacing_99th_percentile", )] for i in range(len(sweep_directories)): table_data.append(( sweep_directories[i], templates[i], str(n_strong_spots[i]), str(n_unindexed_spots[i]), str(n_indexed_lattices[i]), str(d_strong_spots_50th_percentile[i]), str(d_strong_spots_95th_percentile[i]), str(d_strong_spots_99th_percentile[i]), )) with open("results.txt", "wb") as f: print(table_utils.format(table_data, has_header=True, justify="right"), file=f) table_data = [( "sweep_dir", "cell_a", "cell_b", "cell_c", "alpha", "beta", "gamma", "#indexed_reflections", "d_min_indexed", "rmsd_x", "rmsd_y", "rmsd_phi", )] for i in range(len(cell_params)): table_data.append(( sweep_dir_cryst[i], str(cell_params[i][0]), str(cell_params[i][1]), str(cell_params[i][2]), str(cell_params[i][3]), str(cell_params[i][4]), str(cell_params[i][5]), str(n_indexed[i]), str(d_min_indexed[i]), str(rmsds[i][0]), str(rmsds[i][1]), str(rmsds[i][2]), )) with open("results_indexed.txt", "wb") as f: print(table_utils.format(table_data, has_header=True, justify="right"), file=f) cell_a = flex.double([params[0] for params in cell_params]) cell_b = flex.double([params[1] for params in cell_params]) cell_c = flex.double([params[2] for params in cell_params]) cell_alpha = flex.double([params[3] for params in cell_params]) cell_beta = flex.double([params[4] for params in cell_params]) cell_gamma = flex.double([params[5] for params in cell_params]) from matplotlib import pyplot from matplotlib.backends.backend_pdf import PdfPages pyplot.rc("font", family="serif") pyplot.rc("font", serif="Times New Roman") red, blue = "#B2182B", "#2166AC" hist = flex.histogram(n_strong_spots_dmin_4.as_double(), n_slots=20) hist.show() fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.bar( hist.slot_centers(), hist.slots(), width=0.75 * hist.slot_width(), color=blue, edgecolor=blue, ) ax.set_xlabel("Spot count") ax.set_ylabel("Frequency") pdf = PdfPages("spot_count_histogram.pdf") pdf.savefig(fig) pdf.close() # pyplot.show() hist = flex.histogram(n_indexed_lattices.as_double(), n_slots=flex.max(n_indexed_lattices)) hist.show() fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.bar( range(int(hist.data_max())), hist.slots(), width=0.75 * hist.slot_width(), align="center", color=blue, edgecolor=blue, ) ax.set_xlim(-0.5, hist.data_max() - 0.5) ax.set_xticks(range(0, int(hist.data_max()))) ax.set_xlabel("Number of indexed lattices") ax.set_ylabel("Frequency") pdf = PdfPages("n_indexed_lattices_histogram.pdf") pdf.savefig(fig) pdf.close() # pyplot.show() if flex.max(n_integrated_lattices) > 0: hist = flex.histogram(n_integrated_lattices.as_double(), n_slots=flex.max(n_integrated_lattices)) hist.show() fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.bar( range(int(hist.data_max())), hist.slots(), width=0.75 * hist.slot_width(), align="center", color=blue, edgecolor=blue, ) ax.set_xlim(-0.5, hist.data_max() - 0.5) ax.set_xticks(range(0, int(hist.data_max()))) ax.set_xlabel("Number of integrated lattices") ax.set_ylabel("Frequency") pdf = PdfPages("n_integrated_lattices_histogram.pdf") pdf.savefig(fig) pdf.close() # pyplot.show() fig, axes = pyplot.subplots(nrows=2, ncols=3, squeeze=False) for i, cell_param in enumerate( (cell_a, cell_b, cell_c, cell_alpha, cell_beta, cell_gamma)): ax = axes.flat[i] flex.min_max_mean_double(cell_param).show() print(flex.median(cell_param)) hist = flex.histogram(cell_param, n_slots=20) hist.show() ax.bar( hist.slot_centers(), hist.slots(), width=0.75 * hist.slot_width(), color=blue, edgecolor=blue, ) ax.set_xlabel("Cell parameter") ax.set_ylabel("Frequency") pyplot.tight_layout() pdf = PdfPages("cell_parameters.pdf") pdf.savefig(fig) pdf.close()
def show_overall_observations(Fit_I,Fit_I_stddev,I_visited,ordered,sim, out = None,title = None,work_params = None): # at minimum we need the Miller set of merged intensities and sigmas # assert len(ordered.indices())==len(Fit_I) # have to assert this because the "ordered" indices may not in fact be ordered. # but can not assert this in the test1.py test suite. model_subset = ordered[0:len(Fit_I)] uc = ordered.unit_cell() model_subset.crystal_symmetry().show_summary() if work_params is not None: d_min = work_params.d_min d_max = work_params.d_max n_bins = work_params.output.n_bins else: d_min = flex.min(uc.d(model_subset.indices())) # extent of data d_max = None n_bins = min( len(Fit_I)//20, 15 ) #obs, redundancy, summed_wt_I, summed_weight, ISIGI, n_bins=15, out=None, title=None, work_params=None): if out is None: out = sys.stdout model_subset.setup_binner(d_max=(d_max or 100000), d_min=d_min, n_bins=n_bins) result = [] multiplicity = flex.int(len(Fit_I)) for iraw in xrange(len(sim.miller)): multiplicity[sim.miller[iraw]] += 1 cumulative_unique = 0 cumulative_meas = 0 cumulative_theor = 0 cumulative_In = 0 cumulative_I = 0.0 cumulative_Isigma = 0.0 frame_worker = n_frames_worker(obs_arrays=sim, unique_set=model_subset, d_max=(d_max or 100000),d_min=d_min,n_bins=n_bins) for i_bin in model_subset.binner().range_used(): sel_w = model_subset.binner().selection(i_bin) sel_fo_all = model_subset.select(sel_w) d_range = model_subset.binner().bin_legend( i_bin=i_bin, show_bin_number=False, show_counts=False) sel_multiplicity = multiplicity.select(sel_w) sel_absent = sel_multiplicity.count(0) n_present = sel_multiplicity.size() - sel_absent sel_complete_tag = "[%d/%d]" % (n_present, sel_multiplicity.size()) sel_measurements = flex.sum(sel_multiplicity) # Alternatively, redundancy (or multiplicity) is calculated as the # average number of observations for the observed # reflections--missing reflections do not affect the redundancy # adversely, and the reported value becomes # completeness-independent. val_multiplicity_obs = 0 if n_present > 0: val_multiplicity_obs = flex.sum(sel_multiplicity) / n_present sel_frames = frame_worker.per_bin_frames(i_bin) # Per-bin sum of I and I/sig(I). For any reflection, the weight # of the merged intensity must be positive for this to make sense. sel_o = (sel_w & (Fit_I_stddev > 0.)) selected_intensity = Fit_I.select(sel_o) selected_stddev = Fit_I_stddev.select(sel_o) I_sum = flex.sum(selected_intensity) assert selected_stddev.count(0.) == 0 I_sigI_sum = flex.sum(selected_intensity / selected_stddev) I_n = sel_o.count(True) assert sel_measurements == frame_worker.per_bin_meas(i_bin) if sel_measurements > 0: mean_I = mean_I_sigI = 0 if I_n > 0: mean_I = I_sum / I_n mean_I_sigI = I_sigI_sum / I_n bin = resolution_bin( i_bin=i_bin, d_range=d_range, d_min=model_subset.binner().bin_d_min(i_bin), redundancy_asu=flex.mean(sel_multiplicity.as_double()), redundancy_obs=val_multiplicity_obs, frames=sel_frames, complete_tag=sel_complete_tag, completeness=n_present / sel_multiplicity.size(), measurements=sel_measurements, negative=frame_worker.per_bin_neg(i_bin), percent_neg=100.*frame_worker.per_bin_neg(i_bin)/frame_worker.per_bin_meas(i_bin), mean_I=mean_I, mean_I_sigI=mean_I_sigI ) result.append(bin) cumulative_unique += n_present cumulative_meas += sel_measurements cumulative_theor += sel_multiplicity.size() cumulative_In += I_n cumulative_I += I_sum cumulative_Isigma += I_sigI_sum if (title is not None) : print >> out, title from libtbx import table_utils table_header = ["","","","<asu","<obs",""," #"," %","","",""] table_header2 = ["Bin","Resolution Range","Completeness","multi>","multi>","n_meas"," neg"," neg","n_xtal","<I>","<I/sig(I)>"] table_data = [] table_data.append(table_header) table_data.append(table_header2) for bin in result: table_row = [] table_row.append("%3d" % bin.i_bin) table_row.append("%-13s" % bin.d_range) table_row.append("%13s" % bin.complete_tag) table_row.append("%6.2f" % bin.redundancy_asu) table_row.append("%6.2f" % bin.redundancy_obs) table_row.append("%6d" % bin.measurements) table_row.append("%4d" % bin.negative) table_row.append("%5.2f"%bin.percent_neg) table_row.append("%6d" % bin.frames) table_row.append("%8.3f" % bin.mean_I) table_row.append("%8.3f" % bin.mean_I_sigI) table_data.append(table_row) table_data.append([""]*len(table_header)) table_data.append( [ format_value("%3s", "All"), format_value("%-13s", " "), format_value("%13s", "[%d/%d]"%(cumulative_unique,cumulative_theor)), format_value("%6.2f", cumulative_meas/cumulative_theor), format_value("%6.2f", cumulative_meas/cumulative_unique), format_value("%6d", cumulative_meas), format_value("%4d", frame_worker.all_neg()), format_value("%5.2f", 100.*frame_worker.all_neg()/frame_worker.all_meas()), format_value("%6d", frame_worker.all_frames()), format_value("%8.3f", cumulative_I/cumulative_In), format_value("%8.3f", cumulative_Isigma/cumulative_In), ]) print print >>out,table_utils.format(table_data,has_header=2,justify='center',delim=" ") # XXX generate table object for displaying plots if (title is None) : title = "Data statistics by resolution" table = data_plots.table_data( title=title, x_is_inverse_d_min=True, force_exact_x_labels=True) table.add_column( column=[1 / bin.d_min**2 for bin in result], column_name="d_min", column_label="Resolution") table.add_column( column=[bin.redundancy_asu for bin in result], column_name="redundancy", column_label="Redundancy") table.add_column( column=[bin.completeness for bin in result], column_name="completeness", column_label="Completeness") table.add_column( column=[bin.mean_I_sigI for bin in result], column_name="mean_i_over_sigI", column_label="<I/sig(I)>") table.add_graph( name="Redundancy vs. resolution", type="GRAPH", columns=[0,1]) table.add_graph( name="Completeness vs. resolution", type="GRAPH", columns=[0,2]) table.add_graph( name="<I/sig(I)> vs. resolution", type="GRAPH", columns=[0,3]) return table,n_bins,d_min
def run(self): '''Execute the script.''' import os, math from cctbx.crystal import symmetry from scitbx.array_family import flex from libtbx import table_utils, easy_pickle from xfel.command_line.cspad_cbf_metrology import find_files from dxtbx.model.experiment.experiment_list import ExperimentListFactory table_header = ["","","","I","IsigI","N >","RMSD","Cutoff"] table_header2 = ["Bin","Resolution Range","Completeness","","","cutoff","(um)",""] # Parse the command line params, options, all_paths = self.parser.parse_args(show_diff_phil=False, return_unhandled=True) exp_paths = [] refl_paths = [] for path in all_paths: exps, refs = find_files(path, "integrated") exp_paths.extend(exps) refl_paths.extend(refs) assert len(exp_paths) == len(refl_paths) best_data = {} best_limits = flex.double() for exp_path, refl_path in zip(exp_paths, refl_paths): experiments = ExperimentListFactory.from_json_file(exp_path) reflections = easy_pickle.load(refl_path) exp_name = os.path.basename(exp_path) if exp_name.startswith("idx-") and exp_name.endswith("_refined_experiments.json"): tag = exp_name.lstrip("idx-").rstrip("_refined_experiments.json") else: tag = "%s, %s"%(exp_path, refl_path) for exp_id, experiment in enumerate(experiments): print "*"*80 print "Data table for", tag table_data = [] table_data.append(table_header) table_data.append(table_header2) crystal = experiment.crystal refls = reflections.select(reflections['id'] == exp_id) sym = symmetry(unit_cell = crystal.get_unit_cell(), space_group = crystal.get_space_group()) d = crystal.get_unit_cell().d(refls['miller_index']) mset = sym.miller_set(indices = refls['miller_index'].select(d>=params.d_min), anomalous_flag=False) binner = mset.setup_binner(n_bins=params.n_bins) acceptable_resolution_bins = [] for i in binner.range_used(): d_max, d_min = binner.bin_d_range(i) sel = (d <= d_max) & (d > d_min) sel &= refls['intensity.sum.value'] > 0 bin_refls = refls.select(sel) n_refls = len(bin_refls) avg_i = flex.mean(bin_refls['intensity.sum.value']) if n_refls > 0 else 0 avg_i_sigi = flex.mean(bin_refls['intensity.sum.value'] / flex.sqrt(bin_refls['intensity.sum.variance'])) if n_refls > 0 else 0 acceptable_resolution_bins.append(avg_i_sigi >= params.sig_filter_sigma) bright_refls = bin_refls.select((bin_refls['intensity.sum.value']/flex.sqrt(bin_refls['intensity.sum.variance'])) >= params.sig_filter_sigma) n_bright = len(bright_refls) rmsd_obs = 1000*math.sqrt((bright_refls['xyzcal.mm']-bright_refls['xyzobs.mm.value']).sum_sq()/n_bright) if n_bright > 0 else 0 table_row = [] table_row.append("%3d"%i) table_row.append("%-13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False, show_d_range=True, show_counts=False)) table_row.append("%13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False, show_d_range=False, show_counts=True)) table_row.append("%.1f"%(avg_i)) table_row.append("%.1f"%(avg_i_sigi)) table_row.append("%3d"%n_bright) table_row.append("%.1f"%(rmsd_obs)) table_data.append(table_row) acceptable_resolution_bins = [acceptable_resolution_bins[i] for i in xrange(len(acceptable_resolution_bins)) if False not in acceptable_resolution_bins[:i+1]] for b, row in zip(acceptable_resolution_bins, table_data[2:]): if b: row.append("X") print table_utils.format(table_data,has_header=2,justify='center',delim=" ") if any(acceptable_resolution_bins): best_index = acceptable_resolution_bins.count(True)-1 best_row = table_data[best_index+2] d_min = binner.bin_d_range(binner.range_used()[best_index])[1] if len(best_limits) < params.best_count: best_limits.append(d_min) best_data[tag] = d_min, best_row elif (d_min < best_limits).count(True) > 0: worst_d_min = flex.max(best_limits) for tag, data in best_data.iteritems(): if worst_d_min == data[0]: best_data[tag] = d_min, best_row best_limits[flex.first_index(best_limits, worst_d_min)] = d_min break print tag, "best row:", " ".join(best_row) else: print "Data didn't pass cutoff" if len(best_limits) > 0: print "*"*80 print "Top", len(best_limits) for tag, data in best_data.iteritems(): print tag, " ".join(data[1])
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_datablocks, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) # Find all detector objects detectors = [] detectors.extend(experiments.detectors()) dbs = [] for datablock in datablocks: dbs.extend(datablock.unique_detectors()) detectors.extend(dbs) # Verify inputs if len(detectors) != 2: raise Sorry("Please provide a reference and a moving set of experiments and or datablocks") reflections = reflections[1] detector = detectors[1] if not hasattr(detector, 'hierarchy'): raise Sorry("Script intended for hierarchical detectors") if params.max_hierarchy_level is None or str(params.max_hierarchy_level).lower() == 'auto': params.max_hierarchy_level = 0 root = detector.hierarchy() while root.is_group(): root = root[0] params.max_hierarchy_level += 1 print "Found", params.max_hierarchy_level+1, "hierarchy levels" reference_root = detectors[0].hierarchy() moving_root = detector.hierarchy() rori = get_center(reference_root) rf = col(reference_root.get_fast_axis()) rs = col(reference_root.get_slow_axis()) r_norm = col(reference_root.get_normal()) s0 = col(flex.vec3_double([col(b.get_s0()) for b in experiments.beams()]).mean()) summary_table_header = ["Hierarchy","Delta XY","Delta XY","R Offsets","R Offsets","T Offsets","T Offsets","Z Offsets","Z Offsets","dR Norm","dR Norm","dT Norm","dT Norm","Local dNorm", "Local dNorm", "Rot Z","Rot Z"] summary_table_header2 = ["Level","","Sigma","","Sigma","","Sigma","","Sigma","","Sigma","","Sigma","","Sigma","","Sigma"] summary_table_header3 = ["","(microns)","(microns)","(microns)","(microns)","(microns)","(microns)","(microns)","(microns)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)"] summary_table_data = [] summary_table_data.append(summary_table_header) summary_table_data.append(summary_table_header2) summary_table_data.append(summary_table_header3) table_header = ["PanelG","BC dist","Delta XY","R Offsets","T Offsets","Z Offsets","dR Norm","dT Norm","Local dNorm","Rot Z","N Refls"] table_header2 = ["ID","(mm)","(microns)","(microns)","(microns)","(microns)","(deg)","(deg)","(deg)","(deg)",""] from xfel.cftbx.detector.cspad_cbf_tbx import basis def get_full_basis_shift(pg): """Compute basis shift from pg to lab space""" shift = basis(panelgroup=pg) while True: parent = pg.parent() if parent is None: break shift = basis(panelgroup=parent) * shift pg = parent return shift # Iterate through the hierarchy levels for level in xrange(params.max_hierarchy_level+1): delta_xy = flex.double() r_offsets = flex.double() t_offsets = flex.double() z_offsets = flex.double() rot_z = flex.double() delta_r_norm = flex.double() delta_t_norm = flex.double() local_dnorm = flex.double() bc_dists = flex.double() weights = flex.double() rows = [] for pg_id, (pg1, pg2) in enumerate(zip(iterate_detector_at_level(reference_root, 0, level), iterate_detector_at_level(moving_root, 0, level))): weight = 0 for panel_id, p in enumerate(iterate_panels(pg2)): weight += len(reflections.select(reflections['panel'] == id_from_name(detector, p.get_name()))) weights.append(weight) bc = col(pg1.get_beam_centre_lab(s0)) ori = get_center(pg1) bc_dist = (ori-bc).length() bc_dists.append(bc_dist) z_dists = [] ori_xy = [] for pg in [pg1,pg2]: ori = pg.get_local_origin() ori_xy.append(col((ori[0], ori[1]))) z_dists.append(ori[2]*1000) dxy = (ori_xy[1]-ori_xy[0]).length()*1000 delta_xy.append(dxy) z_off = z_dists[1]-z_dists[0] z_offsets.append(z_off) pgo1 = col(pg1.get_origin()) ro_pgo = pgo1 - rori # vector from the detector origin to the panel group origin if ro_pgo.length() == 0: radial = col((0,0,0)) transverse = col((0,0,0)) else: radial = ((rf.dot(ro_pgo) * rf) + (rs.dot(ro_pgo) * rs)).normalize() # component of ro_pgo in rf rs plane transverse = r_norm.cross(radial).normalize() # now radial and transverse are vectors othogonal to each other and the detector normal, such that # radial points at the panel group origin # compute shift in local frame, then convert that shift to lab space, then make it relative to the reference's origin, in lab space lpgo1 = col(pg1.get_local_origin()) lpgo2 = col(pg2.get_local_origin()) delta_pgo = (get_full_basis_shift(pg1) * (lpgo2-lpgo1)) - pgo1 # v is the component of delta_pgo along the radial vector v = (radial.dot(delta_pgo) * radial) r_offset = v.length() * 1000 angle = r_norm.angle(v, deg=True) if r_norm.cross(v).dot(transverse) < 0: r_offset = -r_offset r_offsets.append(r_offset) # v is the component of delta_pgo along the transverse vector v = (transverse.dot(delta_pgo) * transverse) t_offset = v.length() * 1000 angle = r_norm.angle(v, deg=True) if r_norm.cross(v).dot(radial) < 0: t_offset = -t_offset t_offsets.append(t_offset) pgn1 = col(pg1.get_normal()) pgf1 = col(pg1.get_fast_axis()) pgs1 = col(pg1.get_slow_axis()) pgn2 = col(pg2.get_normal()) pgf2 = col(pg2.get_fast_axis()) # v1 and v2 are the component of pgf1 and pgf2 in the rf rs plane v1 = (rf.dot(pgf1) * rf) + (rs.dot(pgf1) * rs) v2 = (rf.dot(pgf2) * rf) + (rs.dot(pgf2) * rs) rz = v1.angle(v2, deg=True) rot_z.append(rz) # v1 and v2 are the components of pgn1 and pgn2 in the r_norm radial plane v1 = (r_norm.dot(pgn1) * r_norm) + (radial.dot(pgn1) * radial) v2 = (r_norm.dot(pgn2) * r_norm) + (radial.dot(pgn2) * radial) drn = v1.angle(v2, deg=True) if v2.cross(v1).dot(transverse) < 0: drn = -drn delta_r_norm.append(drn) # v1 and v2 are the components of pgn1 and pgn2 in the r_norm transverse plane v1 = (r_norm.dot(pgn1) * r_norm) + (transverse.dot(pgn1) * transverse) v2 = (r_norm.dot(pgn2) * r_norm) + (transverse.dot(pgn2) * transverse) dtn = v1.angle(v2, deg=True) if v2.cross(v1).dot(radial) < 0: dtn = -dtn delta_t_norm.append(dtn) # Determine angle between normals in local space lpgf1 = col(pg1.get_local_fast_axis()) lpgs1 = col(pg1.get_local_slow_axis()) lpgn1 = lpgf1.cross(lpgs1) lpgf2 = col(pg2.get_local_fast_axis()) lpgs2 = col(pg2.get_local_slow_axis()) lpgn2 = lpgf2.cross(lpgs2) ldn = lpgn1.angle(lpgn2, deg=True) local_dnorm.append(ldn) row = ["%3d"%pg_id, "%6.1f"%bc_dist, "%6.1f"%dxy, "%6.1f"%r_offset, "%6.1f"%t_offset, "%6.1f"%z_off, "%.4f"%drn, "%.4f"%dtn, "%.4f"%ldn, "%.4f"%rz, "%8d"%weight] rows.append(row) wm_row = ["Weighted mean", ""] ws_row = ["Weighted stddev", ""] s_row = ["%d"%level] iterable = zip([delta_xy, r_offsets, t_offsets, z_offsets, delta_r_norm, delta_t_norm, local_dnorm, rot_z], ["%6.1f","%6.1f","%6.1f","%6.1f","%.4f","%.4f","%.4f","%.4f"]) if len(z_offsets) == 0: wm_row.extend(["%6.1f"%0]*8) ws_row.extend(["%6.1f"%0]*8) s_row.extend(["%6.1f"%0]*8) elif len(z_offsets) == 1: for data, fmt in iterable: wm_row.append(fmt%data[0]) ws_row.append(fmt%0) s_row.append(fmt%data[0]) s_row.append(fmt%0) else: for data, fmt in iterable: stats = flex.mean_and_variance(data, weights) wm_row.append(fmt%stats.mean()) ws_row.append(fmt%stats.gsl_stats_wsd()) s_row.append(fmt%stats.mean()) s_row.append(fmt%stats.gsl_stats_wsd()) wm_row.append("") ws_row.append("") summary_table_data.append(s_row) table_data = [table_header, table_header2] table_d = {d:row for d, row in zip(bc_dists, rows)} table_data.extend([table_d[key] for key in sorted(table_d)]) table_data.append(wm_row) table_data.append(ws_row) from libtbx import table_utils print "Hierarchy level %d Detector shifts"%level print table_utils.format(table_data,has_header=2,justify='center',delim=" ") print "Detector shifts summary" print table_utils.format(summary_table_data,has_header=3,justify='center',delim=" ") print print """
def __str__(self): from libtbx import table_utils U = matrix.sqr(self.experiment.crystal.get_U()) B = matrix.sqr(self.experiment.crystal.get_B()) a_star_ = U * B * a_star b_star_ = U * B * b_star c_star_ = U * B * c_star Binvt = B.inverse().transpose() a_ = U * Binvt * a b_ = U * Binvt * b c_ = U * Binvt * c names = self.experiment.goniometer.get_names() axes = self.experiment.goniometer.get_axes() rows = [["Experimental axis", "a*", "b*", "c*"]] rows.append([names[0]] + [ "%.3f" % smallest_angle(axis.angle(matrix.col(axes[0]), deg=True)) for axis in (a_star_, b_star_, c_star_) ]) rows.append(["Beam"] + [ "%.3f" % smallest_angle(axis.angle(self.s0, deg=True)) for axis in (a_star_, b_star_, c_star_) ]) rows.append([names[2]] + [ "%.3f" % smallest_angle(axis.angle(matrix.col(axes[2]), deg=True)) for axis in (a_star_, b_star_, c_star_) ]) output = [] output.append( "Angles between reciprocal cell axes and principal experimental axes:" ) output.append(table_utils.format(rows=rows, has_header=True)) output.append("") rows = [["Experimental axis", "a", "b", "c"]] rows.append([names[0]] + [ "%.3f" % smallest_angle(axis.angle(matrix.col(axes[0]), deg=True)) for axis in (a_, b_, c_) ]) rows.append(["Beam"] + [ "%.3f" % smallest_angle(axis.angle(self.s0, deg=True)) for axis in (a_, b_, c_) ]) rows.append([names[2]] + [ "%.3f" % smallest_angle(axis.angle(matrix.col(axes[2]), deg=True)) for axis in (a_, b_, c_) ]) output.append( "Angles between unit cell axes and principal experimental axes:") output.append(table_utils.format(rows=rows, has_header=True)) output.append("") names = self.experiment.goniometer.get_names() space_group = self.experiment.crystal.get_space_group() reciprocal = self.frame == "reciprocal" rows = [] for angles, vector_pairs in self.unique_solutions.items(): v1, v2 = list(vector_pairs)[0] rows.append(( describe(v1, space_group, reciprocal=reciprocal), describe(v2, space_group, reciprocal=reciprocal), "% 7.3f" % angles[0], "% 7.3f" % angles[1], )) rows = [("Primary axis", "Secondary axis", names[1], names[0]) ] + sorted(rows) output.append("Independent solutions:") output.append(table_utils.format(rows=rows, has_header=True)) return "\n".join(output)
def run_cc(params, reindexing_op, output): uniform, selected_uniform, have_iso_ref = load_cc_data(params, reindexing_op, output) NBIN = params.output.n_bins if have_iso_ref: slope, offset, corr_iso, N_iso = correlation(selected_uniform[1], selected_uniform[0], params.include_negatives) print >> output, "C.C. iso is %.1f%% on %d indices" % (100 * corr_iso, N_iso) slope, offset, corr_int, N_int = correlation(selected_uniform[2], selected_uniform[3], params.include_negatives) print >> output, "C.C. int is %.1f%% on %d indices" % (100.0 * corr_int, N_int) if have_iso_ref: binned_cc_ref, binned_cc_ref_N = binned_correlation( selected_uniform[1], selected_uniform[0], params.include_negatives ) # binned_cc_ref.show(f=output) ref_scale = scale_factor( selected_uniform[1], selected_uniform[0], weights=flex.pow(selected_uniform[1].sigmas(), -2), use_binning=True, ) # ref_scale.show(f=output) ref_riso = r1_factor(selected_uniform[1], selected_uniform[0], scale_factor=ref_scale, use_binning=True) # ref_riso.show(f=output) ref_scale_all = scale_factor( selected_uniform[1], selected_uniform[0], weights=flex.pow(selected_uniform[1].sigmas(), -2) ) ref_riso_all = r1_factor(selected_uniform[1], selected_uniform[0], scale_factor=ref_scale_all) binned_cc_int, binned_cc_int_N = binned_correlation( selected_uniform[2], selected_uniform[3], params.include_negatives ) # binned_cc_int.show(f=output) oe_scale = scale_factor( selected_uniform[2], selected_uniform[3], weights=flex.pow(selected_uniform[2].sigmas(), -2) + flex.pow(selected_uniform[3].sigmas(), -2), use_binning=True, ) # oe_scale.show(f=output) oe_rint = r1_factor(selected_uniform[2], selected_uniform[3], scale_factor=oe_scale, use_binning=True) # oe_rint.show(f=output) oe_rsplit = r_split(selected_uniform[2], selected_uniform[3], use_binning=True) oe_scale_all = scale_factor( selected_uniform[2], selected_uniform[3], weights=flex.pow(selected_uniform[2].sigmas(), -2) + flex.pow(selected_uniform[3].sigmas(), -2), ) oe_rint_all = r1_factor(selected_uniform[2], selected_uniform[3], scale_factor=oe_scale_all) oe_rsplit_all = r_split(selected_uniform[2], selected_uniform[3]) if have_iso_ref: print >> output, "R factors Riso = %.1f%%, Rint = %.1f%%" % (100.0 * ref_riso_all, 100.0 * oe_rint_all) else: print >> output, "R factor Rint = %.1f%%" % (100.0 * oe_rint_all) split_sigma_data = split_sigma_test( selected_uniform[2], selected_uniform[3], scale=oe_scale, use_binning=True, show_plot=False ) split_sigma_data_all = split_sigma_test( selected_uniform[2], selected_uniform[3], scale=oe_scale_all, use_binning=False, show_plot=False ) print >> output if reindexing_op == "h,k,l": print >> output, "Table of Scaling Results:" else: print >> output, "Table of Scaling Results Reindexing as %s:" % reindexing_op from libtbx import table_utils table_header = ["", "", "", "CC", " N", "CC", " N", "R", "R", "R", "Scale", "Scale", "SpSig"] table_header2 = [ "Bin", "Resolution Range", "Completeness", "int", "int", "iso", "iso", "int", "split", "iso", "int", "iso", "Test", ] table_data = [] table_data.append(table_header) table_data.append(table_header2) items = binned_cc_int.binner.range_used() # XXX Make it clear what the completeness here actually is! cumulative_counts_given = 0 cumulative_counts_complete = 0 for bin in items: table_row = [] table_row.append("%3d" % bin) table_row.append( "%-13s" % binned_cc_int.binner.bin_legend( i_bin=bin, show_bin_number=False, show_bin_range=False, show_d_range=True, show_counts=False ) ) table_row.append( "%13s" % binned_cc_int.binner.bin_legend( i_bin=bin, show_bin_number=False, show_bin_range=False, show_d_range=False, show_counts=True ) ) cumulative_counts_given += binned_cc_int.binner._counts_given[bin] cumulative_counts_complete += binned_cc_int.binner._counts_complete[bin] table_row.append("%.1f%%" % (100.0 * binned_cc_int.data[bin])) table_row.append("%7d" % (binned_cc_int_N.data[bin])) if have_iso_ref and binned_cc_ref.data[bin] is not None: table_row.append("%.1f%%" % (100 * binned_cc_ref.data[bin])) else: table_row.append("--") if have_iso_ref and binned_cc_ref_N.data[bin] is not None: table_row.append("%6d" % (binned_cc_ref_N.data[bin])) else: table_row.append("--") if oe_rint.data[bin] is not None: table_row.append("%.1f%%" % (100.0 * oe_rint.data[bin])) else: table_row.append("--") if oe_rsplit.data[bin] is not None: table_row.append("%.1f%%" % (100 * oe_rsplit.data[bin])) else: table_row.append("--") if have_iso_ref and ref_riso.data[bin] is not None: table_row.append("%.1f%%" % (100 * ref_riso.data[bin])) else: table_row.append("--") if oe_scale.data[bin] is not None: table_row.append("%.3f" % oe_scale.data[bin]) else: table_row.append("--") if have_iso_ref and ref_scale.data[bin] is not None: table_row.append("%.3f" % ref_scale.data[bin]) else: table_row.append("--") if split_sigma_data.data[bin] is not None: table_row.append("%.4f" % split_sigma_data.data[bin]) else: table_row.append("--") table_data.append(table_row) table_data.append([""] * len(table_header)) table_row = [ format_value("%3s", "All"), format_value("%-13s", " "), format_value("%13s", "[%d/%d]" % (cumulative_counts_given, cumulative_counts_complete)), format_value("%.1f%%", 100 * corr_int), format_value("%7d", N_int), ] if have_iso_ref: table_row.extend((format_value("%.1f%%", 100 * corr_iso), format_value("%6d", N_iso))) else: table_row.extend(("--", "--")) table_row.extend((format_value("%.1f%%", 100 * oe_rint_all), format_value("%.1f%%", 100 * oe_rsplit_all))) if have_iso_ref: table_row.append(format_value("%.1f%%", 100 * ref_riso_all)) else: table_row.append("--") table_row.append(format_value("%.3f", oe_scale_all)) if have_iso_ref: table_row.append(format_value("%.3f", ref_scale_all)) else: table_row.append("--") if split_sigma_data_all is not None: table_row.append("%.1f" % split_sigma_data_all) else: table_row.append("--") table_data.append(table_row) print >> output print >> output, table_utils.format(table_data, has_header=2, justify="center", delim=" ") print >> output, """CCint is the CC-1/2 defined by Diederichs; correlation between odd/even images. Similarly, Scale int and R int are the scaling factor and scaling R factor between odd/even images. "iso" columns compare the whole XFEL dataset to the isomorphous reference.""" print >> output, """Niso: result vs. reference common set""", if params.include_negatives: print >> output, """including negative merged intensities (set by phil parameter).""" elif params.scaling.log_cutoff is None: print >> output else: print >> output, """with intensites < %7.2g filtered out (controlled by scaling.log_cutoff phil parameter set to %5.1f)""" % ( math.exp(params.scaling.log_cutoff), params.scaling.log_cutoff, ) if have_iso_ref: assert N_iso == flex.sum(flex.double([x for x in binned_cc_ref_N.data if x is not None])) assert N_int == flex.sum(flex.double([x for x in binned_cc_int_N.data if x is not None])) if params.scaling.show_plots: from matplotlib import pyplot as plt plt.plot(flex.log(selected_uniform[-2].data()), flex.log(selected_uniform[-1].data()), "r.") plt.show() if have_iso_ref: plt.plot(flex.log(selected_uniform[0].data()), flex.log(selected_uniform[1].data()), "r.") plt.show() print >> output
def __call__(self, experiments, reflections): results = flex.reflection_table() table_header = ["","","","I","IsigI","N >","RMSD","Cutoff"] table_header2 = ["Bin","Resolution Range","Completeness","","","cutoff","(um)",""] for exp_id in xrange(len(experiments)): print("*"*80) print("Significance filtering experiment", exp_id) table_data = [] table_data.append(table_header) table_data.append(table_header2) experiment = experiments[exp_id] # Find the bins for this experiment crystal = experiment.crystal refls = reflections.select(reflections['id'] == exp_id) sym = symmetry(unit_cell = crystal.get_unit_cell(), space_group = crystal.get_space_group()) d = crystal.get_unit_cell().d(refls['miller_index']) mset = sym.miller_set(indices = refls['miller_index'], anomalous_flag=False) binner = mset.setup_binner(n_bins=self.params.n_bins) acceptable_resolution_bins = [] # Iterate through the bins, examining I/sigI at each bin for i in binner.range_used(): d_max, d_min = binner.bin_d_range(i) sel = (d <= d_max) & (d > d_min) sel &= refls['intensity.sum.value'] > 0 bin_refls = refls.select(sel) n_refls = len(bin_refls) avg_i = flex.mean(bin_refls['intensity.sum.value']) if n_refls > 0 else 0 avg_i_sigi = flex.mean(bin_refls['intensity.sum.value'] / flex.sqrt(bin_refls['intensity.sum.variance'])) if n_refls > 0 else 0 acceptable_resolution_bins.append(avg_i_sigi >= self.params.isigi_cutoff) bright_refls = bin_refls.select((bin_refls['intensity.sum.value']/flex.sqrt(bin_refls['intensity.sum.variance'])) >= self.params.isigi_cutoff) n_bright = len(bright_refls) rmsd_obs = 1000*math.sqrt((bright_refls['xyzcal.mm']-bright_refls['xyzobs.mm.value']).sum_sq()/n_bright) if n_bright > 0 else 0 table_row = [] table_row.append("%3d"%i) table_row.append("%-13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False, show_d_range=True, show_counts=False)) table_row.append("%13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False, show_d_range=False, show_counts=True)) table_row.append("%.1f"%(avg_i)) table_row.append("%.1f"%(avg_i_sigi)) table_row.append("%3d"%n_bright) table_row.append("%.1f"%(rmsd_obs)) table_data.append(table_row) # Throw out bins that go back above the cutoff after the first non-passing bin is found acceptable_resolution_bins = [acceptable_resolution_bins[i] for i in xrange(len(acceptable_resolution_bins)) if False not in acceptable_resolution_bins[:i+1]] for b, row in zip(acceptable_resolution_bins, table_data[2:]): if b: row.append("X") print(table_utils.format(table_data,has_header=2,justify='center',delim=" ")) # Save the results if any(acceptable_resolution_bins): best_index = acceptable_resolution_bins.count(True)-1 best_row = table_data[best_index+2] d_min = binner.bin_d_range(binner.range_used()[best_index])[1] print("best row:", " ".join(best_row)) if self.params.enable: results.extend(refls.select(d >= d_min)) else: print("Data didn't pass cutoff") if self.params.enable: return results else: return reflections
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = flatten_experiments(params.input.experiments) # Find all detector objects detectors = experiments.detectors() # Verify inputs if len(params.input.reflections) == len(detectors) and len(detectors) > 1: # case for passing in multiple images on the command line assert len(params.input.reflections) == len(detectors) reflections = flex.reflection_table() for expt_id in xrange(len(detectors)): subset = params.input.reflections[expt_id].data subset['id'] = flex.int(len(subset), expt_id) reflections.extend(subset) else: # case for passing in combined experiments and reflections reflections = flatten_reflections(params.input.reflections)[0] detector = detectors[0] #from dials.algorithms.refinement.prediction import ExperimentsPredictor #ref_predictor = ExperimentsPredictor(experiments, force_stills=experiments.all_stills()) print "N reflections total:", len(reflections) if params.residuals.exclude_outliers: reflections = reflections.select(reflections.get_flags(reflections.flags.used_in_refinement)) print "N reflections used in refinement:", len(reflections) print "Reporting only on those reflections used in refinement" if self.params.residuals.i_sigi_cutoff is not None: sel = (reflections['intensity.sum.value']/flex.sqrt(reflections['intensity.sum.variance'])) >= self.params.residuals.i_sigi_cutoff reflections = reflections.select(sel) print "After filtering by I/sigi cutoff of %f, there are %d reflections left"%(self.params.residuals.i_sigi_cutoff,len(reflections)) reflections['difference_vector_norms'] = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).norms() n = len(reflections) rmsd = self.get_weighted_rmsd(reflections) print "Dataset RMSD (microns)", rmsd * 1000 if params.tag is None: tag = '' else: tag = '%s '%params.tag # set up delta-psi ratio heatmap p = flex.int() # positive n = flex.int() # negative for i in set(reflections['id']): exprefls = reflections.select(reflections['id']==i) p.append(len(exprefls.select(exprefls['delpsical.rad']>0))) n.append(len(exprefls.select(exprefls['delpsical.rad']<0))) plt.hist2d(p, n, bins=30) cb = plt.colorbar() cb.set_label("N images") plt.title(r"%s2D histogram of pos vs. neg $\Delta\Psi$ per image"%tag) plt.xlabel(r"N reflections with $\Delta\Psi$ > 0") plt.ylabel(r"N reflections with $\Delta\Psi$ < 0") self.delta_scalar = 50 # Iterate through the detectors, computing detector statistics at the per-panel level (IE one statistic per panel) # Per panel dictionaries rmsds = {} refl_counts = {} transverse_rmsds = {} radial_rmsds = {} ttdpcorr = {} pg_bc_dists = {} mean_delta_two_theta = {} # per panelgroup flex arrays pg_rmsds = flex.double() pg_r_rmsds = flex.double() pg_t_rmsds = flex.double() pg_refls_count = flex.int() pg_refls_count_d = {} table_header = ["PG id", "RMSD","Radial", "Transverse", "N refls"] table_header2 = ["","(um)","RMSD (um)","RMSD (um)",""] table_data = [] table_data.append(table_header) table_data.append(table_header2) # Compute a set of radial and transverse displacements for each reflection print "Setting up stats..." tmp = flex.reflection_table() # Need to construct a variety of vectors for panel_id, panel in enumerate(detector): panel_refls = reflections.select(reflections['panel'] == panel_id) bcl = flex.vec3_double() tto = flex.double() ttc = flex.double() # Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect # the panel, if it did intersect the panel) for expt_id in set(panel_refls['id']): beam = experiments[expt_id].beam s0 = beam.get_s0() expt_refls = panel_refls.select(panel_refls['id'] == expt_id) beam_centre = panel.get_beam_centre_lab(s0) bcl.extend(flex.vec3_double(len(expt_refls), beam_centre)) obs_x, obs_y, _ = expt_refls['xyzobs.px.value'].parts() cal_x, cal_y, _ = expt_refls['xyzcal.px'].parts() tto.extend(flex.double([panel.get_two_theta_at_pixel(s0, (obs_x[i], obs_y[i])) for i in xrange(len(expt_refls))])) ttc.extend(flex.double([panel.get_two_theta_at_pixel(s0, (cal_x[i], cal_y[i])) for i in xrange(len(expt_refls))])) panel_refls['beam_centre_lab'] = bcl panel_refls['two_theta_obs'] = tto * (180/math.pi) panel_refls['two_theta_cal'] = ttc * (180/math.pi) #+ (0.5*panel_refls['delpsical.rad']*panel_refls['two_theta_obs']) # Compute obs in lab space x, y, _ = panel_refls['xyzobs.mm.value'].parts() c = flex.vec2_double(x, y) panel_refls['obs_lab_coords'] = panel.get_lab_coord(c) # Compute deltaXY in panel space. This vector is relative to the panel origin x, y, _ = (panel_refls['xyzcal.mm'] - panel_refls['xyzobs.mm.value']).parts() # Convert deltaXY to lab space, subtracting off of the panel origin panel_refls['delta_lab_coords'] = panel.get_lab_coord(flex.vec2_double(x,y)) - panel.get_origin() tmp.extend(panel_refls) reflections = tmp # The radial vector points from the center of the reflection to the beam center radial_vectors = (reflections['obs_lab_coords'] - reflections['beam_centre_lab']).each_normalize() # The transverse vector is orthogonal to the radial vector and the beam vector transverse_vectors = radial_vectors.cross(reflections['beam_centre_lab']).each_normalize() # Compute the raidal and transverse components of each deltaXY reflections['radial_displacements'] = reflections['delta_lab_coords'].dot(radial_vectors) reflections['transverse_displacements'] = reflections['delta_lab_coords'].dot(transverse_vectors) # Iterate through the detector at the specified hierarchy level for pg_id, pg in enumerate(iterate_detector_at_level(detector.hierarchy(), 0, params.hierarchy_level)): pg_msd_sum = 0 pg_r_msd_sum = 0 pg_t_msd_sum = 0 pg_refls = 0 pg_delpsi = flex.double() pg_deltwotheta = flex.double() for p in iterate_panels(pg): panel_id = id_from_name(detector, p.get_name()) panel_refls = reflections.select(reflections['panel'] == panel_id) n = len(panel_refls) pg_refls += n delta_x = panel_refls['xyzcal.mm'].parts()[0] - panel_refls['xyzobs.mm.value'].parts()[0] delta_y = panel_refls['xyzcal.mm'].parts()[1] - panel_refls['xyzobs.mm.value'].parts()[1] tmp = flex.sum((delta_x**2)+(delta_y**2)) pg_msd_sum += tmp r = panel_refls['radial_displacements'] t = panel_refls['transverse_displacements'] pg_r_msd_sum += flex.sum_sq(r) pg_t_msd_sum += flex.sum_sq(t) pg_delpsi.extend(panel_refls['delpsical.rad']*180/math.pi) pg_deltwotheta.extend(panel_refls['two_theta_obs'] - panel_refls['two_theta_cal']) bc = col(pg.get_beam_centre_lab(s0)) ori = get_center(pg) pg_bc_dists[pg.get_name()] = (ori-bc).length() if len(pg_deltwotheta) > 0: mean_delta_two_theta[pg.get_name()] = flex.mean(pg_deltwotheta) else: mean_delta_two_theta[pg.get_name()] = 0 if pg_refls == 0: pg_rmsd = pg_r_rmsd = pg_t_rmsd = 0 else: pg_rmsd = math.sqrt(pg_msd_sum/pg_refls) * 1000 pg_r_rmsd = math.sqrt(pg_r_msd_sum/pg_refls) * 1000 pg_t_rmsd = math.sqrt(pg_t_msd_sum/pg_refls) * 1000 pg_rmsds.append(pg_rmsd) pg_r_rmsds.append(pg_r_rmsd) pg_t_rmsds.append(pg_t_rmsd) pg_refls_count.append(pg_refls) pg_refls_count_d[pg.get_name()] = pg_refls table_data.append(["%d"%pg_id, "%.1f"%pg_rmsd, "%.1f"%pg_r_rmsd, "%.1f"%pg_t_rmsd, "%6d"%pg_refls]) refl_counts[pg.get_name()] = pg_refls if pg_refls == 0: rmsds[p.get_name()] = -1 radial_rmsds[p.get_name()] = -1 transverse_rmsds[p.get_name()] = -1 ttdpcorr[pg.get_name()] = -1 else: rmsds[pg.get_name()] = pg_rmsd radial_rmsds[pg.get_name()] = pg_r_rmsd transverse_rmsds[pg.get_name()] = pg_t_rmsd lc = flex.linear_correlation(pg_delpsi, pg_deltwotheta) ttdpcorr[pg.get_name()] = lc.coefficient() r1 = ["Weighted mean"] r2 = ["Weighted stddev"] if len(pg_rmsds) > 1: stats = flex.mean_and_variance(pg_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(pg_r_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(pg_t_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) else: r1.extend([""]*3) r2.extend([""]*3) r1.append("") r2.append("") table_data.append(r1) table_data.append(r2) table_data.append(["Mean", "", "", "", "%8.1f"%flex.mean(pg_refls_count.as_double())]) from libtbx import table_utils print "Detector statistics. Angles in degrees, RMSDs in microns" print table_utils.format(table_data,has_header=2,justify='center',delim=" ") self.histogram(reflections, '%sDifference vector norms (mm)'%tag) if params.show_plots: if self.params.tag is None: t = "" else: t = "%s "%self.params.tag self.image_rmsd_histogram(reflections, tag) # Plots! these are plots with callbacks to draw on individual panels self.detector_plot_refls(detector, reflections, '%sOverall positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltas) self.detector_plot_refls(detector, reflections, '%sRadial positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_radial_deltas) self.detector_plot_refls(detector, reflections, '%sTransverse positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_transverse_deltas) self.detector_plot_refls(detector, reflections, r'%s$\Delta\Psi$'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltapsi, colorbar_units=r"$\circ$") self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY*%s'%(tag, self.delta_scalar), show=False, plot_callback=self.plot_deltas) self.detector_plot_refls(detector, reflections, '%sSP Manual CDF'%tag, show=False, plot_callback=self.plot_cdf_manually) self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY Histograms'%tag, show=False, plot_callback=self.plot_histograms) self.detector_plot_refls(detector, reflections, r'%sRadial displacements vs. $\Delta\Psi$, colored by $\Delta$XY'%tag, show=False, plot_callback=self.plot_radial_displacements_vs_deltapsi) self.detector_plot_refls(detector, reflections, r'%sDistance vector norms'%tag, show=False, plot_callback=self.plot_difference_vector_norms_histograms) # Plot intensity vs. radial_displacement fig = plt.figure() panel_id = 15 panel_refls = reflections.select(reflections['panel'] == panel_id) a = panel_refls['radial_displacements'] b = panel_refls['intensity.sum.value'] sel = (a > -0.2) & (a < 0.2) & (b < 50000) plt.hist2d(a.select(sel), b.select(sel), bins=100) plt.title("%s2D histogram of intensity vs. radial displacement for panel %d"%(tag, panel_id)) plt.xlabel("Radial displacement (mm)") plt.ylabel("Intensity") ax = plt.colorbar() ax.set_label("Counts") # Plot delta 2theta vs. deltapsi n_bins = 10 bin_size = len(reflections)//n_bins bin_low = [] bin_high = [] data = flex.sorted(reflections['two_theta_obs']) for i in xrange(n_bins): bin_low = data[i*bin_size] if (i+1)*bin_size >= len(reflections): bin_high = data[-1] else: bin_high = data[(i+1)*bin_size] refls = reflections.select((reflections['two_theta_obs'] >= bin_low) & (reflections['two_theta_obs'] <= bin_high)) a = refls['delpsical.rad']*180/math.pi b = refls['two_theta_obs'] - refls['two_theta_cal'] fig = plt.figure() sel = (a > -0.2) & (a < 0.2) & (b > -0.05) & (b < 0.05) plt.hist2d(a.select(sel), b.select(sel), bins=50, range = [[-0.2, 0.2], [-0.05, 0.05]]) cb = plt.colorbar() cb.set_label("N reflections") plt.title(r'%sBin %d (%.02f, %.02f 2$\Theta$) $\Delta2\Theta$ vs. $\Delta\Psi$. Showing %d of %d refls'%(tag,i,bin_low,bin_high,len(a.select(sel)),len(a))) plt.xlabel(r'$\Delta\Psi \circ$') plt.ylabel(r'$\Delta2\Theta \circ$') # Plot delta 2theta vs. 2theta a = reflections['two_theta_obs']#[:71610] b = reflections['two_theta_obs'] - reflections['two_theta_cal'] fig = plt.figure() limits = -0.05, 0.05 sel = (b > limits[0]) & (b < limits[1]) plt.hist2d(a.select(sel), b.select(sel), bins=100, range=((0,50), limits)) plt.clim((0,100)) cb = plt.colorbar() cb.set_label("N reflections") plt.title(r'%s$\Delta2\Theta$ vs. 2$\Theta$. Showing %d of %d refls'%(tag,len(a.select(sel)),len(a))) plt.xlabel(r'2$\Theta \circ$') plt.ylabel(r'$\Delta2\Theta \circ$') # calc the trendline z = np.polyfit(a.select(sel), b.select(sel), 1) print 'y=%.7fx+(%.7f)'%(z[0],z[1]) # Plots with single values per panel self.detector_plot_dict(detector, refl_counts, u"%s N reflections"%t, u"%6d", show=False) self.detector_plot_dict(detector, rmsds, "%s Positional RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, radial_rmsds, "%s Radial RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, transverse_rmsds, "%s Transverse RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, ttdpcorr, r"%s $\Delta2\Theta$ vs. $\Delta\Psi$ CC"%t, u"%5.3f", show=False) self.plot_unitcells(experiments) self.plot_data_by_two_theta(reflections, tag) # Plot data by panel group sorted_values = sorted(pg_bc_dists.values()) vdict = {} for k in pg_bc_dists: vdict[pg_bc_dists[k]] = k sorted_keys = [vdict[v] for v in sorted_values if vdict[v] in rmsds] x = [sorted_values[i] for i in xrange(len(sorted_values)) if pg_bc_dists.keys()[i] in rmsds] self.plot_multi_data(x, [[pg_refls_count_d[k] for k in sorted_keys], ([rmsds[k] for k in sorted_keys], [radial_rmsds[k] for k in sorted_keys], [transverse_rmsds[k] for k in sorted_keys]), [radial_rmsds[k]/transverse_rmsds[k] for k in sorted_keys], [mean_delta_two_theta[k] for k in sorted_keys]], "Panel group distance from beam center (mm)", ["N reflections", ("Overall RMSD", "Radial RMSD", "Transverse RMSD"), "R/T RMSD ratio", "Delta two theta"], ["N reflections", "RMSD (microns)", "R/T RMSD ratio", "Delta two theta (degrees)"], "%sData by panelgroup"%tag) if self.params.save_pdf: pp = PdfPages('residuals_%s.pdf'%(tag.strip())) for i in plt.get_fignums(): pp.savefig(plt.figure(i)) pp.close() else: plt.show()