def test_plot_stats(centroid_test_data, tmpdir): experiments, reflections = centroid_test_data stats = per_image_analysis.stats_per_image( experiments[0], reflections, resolution_analysis=False ) image_file = tmpdir.join("pia.png") per_image_analysis.plot_stats(stats, filename=image_file.strpath) assert image_file.check()
def run(args): from dials.util.options import OptionParser import libtbx.load_env usage = "%s [options] find_spots.expt" % (libtbx.env.dispatcher_name) parser = OptionParser(usage=usage, phil=phil_scope, epilog=help_message) params, options, args = parser.parse_args( show_diff_phil=True, return_unhandled=True ) positions = None if params.positions is not None: with open(params.positions, "rb") as f: positions = flex.vec2_double() for line in f.readlines(): line = ( line.replace("(", " ") .replace(")", "") .replace(",", " ") .strip() .split() ) assert len(line) == 3 i, x, y = [float(l) for l in line] positions.append((x, y)) assert len(args) == 1 json_file = args[0] import json with open(json_file, "rb") as f: results = json.load(f) n_indexed = flex.double() fraction_indexed = flex.double() n_spots = flex.double() n_lattices = flex.double() crystals = [] image_names = flex.std_string() for r in results: n_spots.append(r["n_spots_total"]) image_names.append(str(r["image"])) if "n_indexed" in r: n_indexed.append(r["n_indexed"]) n_lattices.append(len(r["lattices"])) for d in r["lattices"]: from dxtbx.model.crystal import CrystalFactory crystals.append(CrystalFactory.from_dict(d["crystal"])) else: n_indexed.append(0) n_lattices.append(0) if n_indexed.size(): sel = n_spots > 0 fraction_indexed = flex.double(n_indexed.size(), 0) fraction_indexed.set_selected(sel, n_indexed.select(sel) / n_spots.select(sel)) import matplotlib matplotlib.use("Agg") from matplotlib import pyplot red = "#e74c3c" plot = True table = True grid = params.grid from libtbx import group_args from dials.algorithms.spot_finding.per_image_analysis import plot_stats, print_table estimated_d_min = flex.double() d_min_distl_method_1 = flex.double() d_min_distl_method_2 = flex.double() n_spots_total = flex.int() n_spots_no_ice = flex.int() total_intensity = flex.double() for d in results: estimated_d_min.append(d["estimated_d_min"]) d_min_distl_method_1.append(d["d_min_distl_method_1"]) d_min_distl_method_2.append(d["d_min_distl_method_2"]) n_spots_total.append(d["n_spots_total"]) n_spots_no_ice.append(d["n_spots_no_ice"]) total_intensity.append(d["total_intensity"]) stats = group_args( image=image_names, n_spots_total=n_spots_total, n_spots_no_ice=n_spots_no_ice, n_spots_4A=None, n_indexed=n_indexed, fraction_indexed=fraction_indexed, total_intensity=total_intensity, estimated_d_min=estimated_d_min, d_min_distl_method_1=d_min_distl_method_1, d_min_distl_method_2=d_min_distl_method_2, noisiness_method_1=None, noisiness_method_2=None, ) if plot: plot_stats(stats) pyplot.clf() if table: print_table(stats) n_rows = 10 n_rows = min(n_rows, len(n_spots_total)) perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True) print("Top %i images sorted by number of spots:" % n_rows) print_table(stats, perm=perm_n_spots_total, n_rows=n_rows) if flex.max(n_indexed) > 0: perm_n_indexed = flex.sort_permutation(n_indexed, reverse=True) print("Top %i images sorted by number of indexed reflections:" % n_rows) print_table(stats, perm=perm_n_indexed, n_rows=n_rows) print("Number of indexed lattices: ", (n_indexed > 0).count(True)) print( "Number with valid d_min but failed indexing: ", ( (d_min_distl_method_1 > 0) & (d_min_distl_method_2 > 0) & (estimated_d_min > 0) & (n_indexed == 0) ).count(True), ) n_bins = 20 spot_count_histogram( n_spots_total, n_bins=n_bins, filename="hist_n_spots_total.png", log=True ) spot_count_histogram( n_spots_no_ice, n_bins=n_bins, filename="hist_n_spots_no_ice.png", log=True ) spot_count_histogram( n_indexed.select(n_indexed > 0), n_bins=n_bins, filename="hist_n_indexed.png", log=False, ) if len(crystals): plot_unit_cell_histograms(crystals) if params.stereographic_projections and len(crystals): from dxtbx.model.experiment_list import ExperimentListFactory experiments = ExperimentListFactory.from_filenames( [image_names[0]], verbose=False ) assert len(experiments) == 1 imageset = experiments.imagesets()[0] s0 = imageset.get_beam().get_s0() # XXX what if no goniometer? rotation_axis = imageset.get_goniometer().get_rotation_axis() indices = ((1, 0, 0), (0, 1, 0), (0, 0, 1)) for i, index in enumerate(indices): from cctbx import crystal, miller from scitbx import matrix miller_indices = flex.miller_index([index]) symmetry = crystal.symmetry( unit_cell=crystals[0].get_unit_cell(), space_group=crystals[0].get_space_group(), ) miller_set = miller.set(symmetry, miller_indices) d_spacings = miller_set.d_spacings() d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1() d_spacings = d_spacings.generate_bijvoet_mates() miller_indices = d_spacings.indices() # plane normal d0 = matrix.col(s0).normalize() d1 = d0.cross(matrix.col(rotation_axis)).normalize() d2 = d1.cross(d0).normalize() reference_poles = (d0, d1, d2) from dials.command_line.stereographic_projection import ( stereographic_projection, ) projections = [] for cryst in crystals: reciprocal_space_points = ( list(cryst.get_U() * cryst.get_B()) * miller_indices.as_vec3_double() ) projections.append( stereographic_projection(reciprocal_space_points, reference_poles) ) # from dials.algorithms.indexing.compare_orientation_matrices import \ # difference_rotation_matrix_and_euler_angles # R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles( # crystals[0], cryst) # print max(euler_angles) from dials.command_line.stereographic_projection import plot_projections plot_projections(projections, filename="projections_%s.png" % ("hkl"[i])) pyplot.clf() def plot_grid( values, grid, file_name, cmap=pyplot.cm.Reds, vmin=None, vmax=None, invalid="white", ): values = values.as_double() # At DLS, fast direction appears to be largest direction if grid[0] > grid[1]: values.reshape(flex.grid(reversed(grid))) values = values.matrix_transpose() else: values.reshape(flex.grid(grid)) f, ax1 = pyplot.subplots(1) mesh1 = ax1.pcolormesh(values.as_numpy_array(), cmap=cmap, vmin=vmin, vmax=vmax) mesh1.cmap.set_under(color=invalid, alpha=None) mesh1.cmap.set_over(color=invalid, alpha=None) ax1.set_aspect("equal") ax1.invert_yaxis() pyplot.colorbar(mesh1, ax=ax1) pyplot.savefig(file_name, dpi=600) pyplot.clf() def plot_positions( values, positions, file_name, cmap=pyplot.cm.Reds, vmin=None, vmax=None, invalid="white", ): values = values.as_double() assert positions.size() >= values.size() positions = positions[: values.size()] if vmin is None: vmin = flex.min(values) if vmax is None: vmax = flex.max(values) x, y = positions.parts() dx = flex.abs(x[1:] - x[:-1]) dy = flex.abs(y[1:] - y[:-1]) dx = dx.select(dx > 0) dy = dy.select(dy > 0) scale = 1 / flex.min(dx) # print scale x = (x * scale).iround() y = (y * scale).iround() from libtbx.math_utils import iceil z = flex.double(flex.grid(iceil(flex.max(y)) + 1, iceil(flex.max(x)) + 1), -2) # print z.all() for x_, y_, z_ in zip(x, y, values): z[y_, x_] = z_ plot_grid( z.as_1d(), z.all(), file_name, cmap=cmap, vmin=vmin, vmax=vmax, invalid=invalid, ) return if grid is not None or positions is not None: if grid is not None: positions = tuple(reversed(grid)) plotter = plot_grid else: plotter = plot_positions cmap = pyplot.get_cmap(params.cmap) plotter( n_spots_total, positions, "grid_spot_count_total.png", cmap=cmap, invalid=params.invalid, ) plotter( n_spots_no_ice, positions, "grid_spot_count_no_ice.png", cmap=cmap, invalid=params.invalid, ) plotter( total_intensity, positions, "grid_total_intensity.png", cmap=cmap, invalid=params.invalid, ) if flex.max(n_indexed) > 0: plotter( n_indexed, positions, "grid_n_indexed.png", cmap=cmap, invalid=params.invalid, ) plotter( fraction_indexed, positions, "grid_fraction_indexed.png", cmap=cmap, vmin=0, vmax=1, invalid=params.invalid, ) for i, d_min in enumerate( (estimated_d_min, d_min_distl_method_1, d_min_distl_method_2) ): vmin = flex.min(d_min.select(d_min > 0)) vmax = flex.max(d_min) cmap = pyplot.get_cmap("%s_r" % params.cmap) d_min.set_selected(d_min <= 0, vmax) if i == 0: plotter( d_min, positions, "grid_d_min.png", cmap=cmap, vmin=vmin, vmax=vmax, invalid=params.invalid, ) else: plotter( d_min, positions, "grid_d_min_method_%i.png" % i, cmap=cmap, vmin=vmin, vmax=vmax, invalid=params.invalid, ) if flex.max(n_indexed) > 0: pyplot.hexbin(n_spots, n_indexed, bins="log", cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() xlim = pyplot.xlim() ylim = pyplot.ylim() pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel("# spots") pyplot.ylabel("# indexed") pyplot.savefig("n_spots_vs_n_indexed.png") pyplot.clf() pyplot.hexbin( n_spots, fraction_indexed, bins="log", cmap=pyplot.cm.jet, gridsize=50 ) pyplot.colorbar() pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel("# spots") pyplot.ylabel("Fraction indexed") pyplot.savefig("n_spots_vs_fraction_indexed.png") pyplot.clf() pyplot.hexbin( n_indexed, fraction_indexed, bins="log", cmap=pyplot.cm.jet, gridsize=50 ) pyplot.colorbar() pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel("# indexed") pyplot.ylabel("Fraction indexed") pyplot.savefig("n_indexed_vs_fraction_indexed.png") pyplot.clf() pyplot.hexbin(n_spots, n_lattices, bins="log", cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel("# spots") pyplot.ylabel("# lattices") pyplot.savefig("n_spots_vs_n_lattices.png") pyplot.clf() pyplot.hexbin( estimated_d_min, d_min_distl_method_1, bins="log", cmap=pyplot.cm.jet, gridsize=50, ) pyplot.colorbar() # pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(estimated_d_min), max(d_min_distl_method_1)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel("estimated_d_min") pyplot.ylabel("d_min_distl_method_1") pyplot.savefig("d_min_vs_distl_method_1.png") pyplot.clf() pyplot.hexbin( estimated_d_min, d_min_distl_method_2, bins="log", cmap=pyplot.cm.jet, gridsize=50, ) pyplot.colorbar() # pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(estimated_d_min), max(d_min_distl_method_2)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel("estimated_d_min") pyplot.ylabel("d_min_distl_method_2") pyplot.savefig("d_min_vs_distl_method_2.png") pyplot.clf() pyplot.hexbin( d_min_distl_method_1, d_min_distl_method_2, bins="log", cmap=pyplot.cm.jet, gridsize=50, ) pyplot.colorbar() # pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(d_min_distl_method_1), max(d_min_distl_method_2)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel("d_min_distl_method_1") pyplot.ylabel("d_min_distl_method_2") pyplot.savefig("distl_method_1_vs_distl_method_2.png") pyplot.clf() pyplot.hexbin(n_spots, estimated_d_min, bins="log", cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel("# spots") pyplot.ylabel("estimated_d_min") pyplot.savefig("n_spots_vs_d_min.png") pyplot.clf() pyplot.hexbin( n_spots, d_min_distl_method_1, bins="log", cmap=pyplot.cm.jet, gridsize=50 ) pyplot.colorbar() pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel("# spots") pyplot.ylabel("d_min_distl_method_1") pyplot.savefig("n_spots_vs_distl_method_1.png") pyplot.clf() pyplot.hexbin( n_spots, d_min_distl_method_2, bins="log", cmap=pyplot.cm.jet, gridsize=50 ) pyplot.colorbar() pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel("# spots") pyplot.ylabel("d_min_distl_method_2") pyplot.savefig("n_spots_vs_distl_method_2.png") pyplot.clf()
def work_all(host, port, filenames, params, plot=False, table=False, json_file=None, grid=None, nproc=None): import json from multiprocessing.pool import ThreadPool as thread_pool if nproc is None: nproc = _nproc() pool = thread_pool(processes=nproc) threads = {} for filename in filenames: threads[filename] = pool.apply_async(work, (host, port, filename, params)) results = [] for filename in filenames: response = threads[filename].get() d = json.loads(response) results.append(d) print response_to_xml(d) if json_file is not None: 'Writing results to %s' % json_file with open(json_file, 'wb') as f: json.dump(results, f) if plot or table: from scitbx.array_family import flex from libtbx import group_args from dials.algorithms.spot_finding.per_image_analysis \ import plot_stats, print_table estimated_d_min = flex.double() d_min_distl_method_1 = flex.double() d_min_distl_method_2 = flex.double() n_spots_total = flex.int() n_spots_no_ice = flex.int() total_intensity = flex.double() for d in results: estimated_d_min.append(d['estimated_d_min']) d_min_distl_method_1.append(d['d_min_distl_method_1']) d_min_distl_method_2.append(d['d_min_distl_method_2']) n_spots_total.append(d['n_spots_total']) n_spots_no_ice.append(d['n_spots_no_ice']) total_intensity.append(d['total_intensity']) stats = group_args(n_spots_total=n_spots_total, n_spots_no_ice=n_spots_no_ice, n_spots_4A=None, total_intensity=total_intensity, estimated_d_min=estimated_d_min, d_min_distl_method_1=d_min_distl_method_1, d_min_distl_method_2=d_min_distl_method_2, noisiness_method_1=None, noisiness_method_2=None) if plot: plot_stats(stats) if table: print_table(stats) if grid is not None: from matplotlib import pyplot n_spots_no_ice.reshape(flex.grid(grid)) print n_spots_no_ice.size() from matplotlib import pyplot fig = pyplot.figure() pyplot.pcolormesh(n_spots_no_ice.as_numpy_array(), cmap=pyplot.cm.Reds) pyplot.savefig("spot_count.png") return
def run(args=None): usage = "dials.spot_counts_per_image [options] imported.expt strong.refl" parser = OptionParser( usage=usage, read_reflections=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message, ) params, options = parser.parse_args(args, show_diff_phil=False) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if not reflections and not experiments: parser.print_help() return # FIXME may want to change this to allow many to be passed i.e. # from parallel runs if len(reflections) != 1: sys.exit("Only one reflection list may be passed") reflections = reflections[0] if "miller_index" in reflections: sys.exit("Only unindexed reflections are currently supported") if any(experiments.crystals()): sys.exit("Only unindexed experiments are currently supported") reflections.centroid_px_to_mm(experiments) reflections.map_centroids_to_reciprocal_space(experiments) if params.id is not None: reflections = reflections.select(reflections["id"] == params.id) all_stats = [] for i, expt in enumerate(experiments): refl = reflections.select(reflections["id"] == i) stats = per_image_analysis.stats_per_image( expt, refl, resolution_analysis=params.resolution_analysis) all_stats.append(stats) # transpose stats summary_table = {} for s in all_stats: for k, value in s._asdict().items(): summary_table.setdefault(k, []) summary_table[k].extend(value) stats = per_image_analysis.StatsMultiImage(**summary_table) print(stats) overall_stats = per_image_analysis.stats_for_reflection_table( reflections, resolution_analysis=params.resolution_analysis) rows = [ ("Overall statistics", ""), ("#spots", "%i" % overall_stats.n_spots_total), ("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice), ("d_min", f"{overall_stats.estimated_d_min:.2f}"), ( "d_min (distl method 1)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1), ), ( "d_min (distl method 2)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_2, overall_stats.noisiness_method_2), ), ] print(tabulate(rows, headers="firstrow")) if params.json: if params.split_json: for k, v in stats._asdict().items(): start, end = params.json.split(".") with open(f"{start}_{k}.{end}", "w") as fp: json.dump(v, fp) if params.joint_json: with open(params.json, "w") as fp: json.dump(stats._asdict(), fp) if params.plot: import matplotlib matplotlib.use("Agg") per_image_analysis.plot_stats(stats, filename=params.plot)
def work_all( host, port, filenames, params, plot=False, table=False, json_file=None, grid=None, nproc=None, ): if nproc is None: nproc = _nproc() with ThreadPool(processes=nproc) as pool: threads = {} for filename in filenames: threads[filename] = pool.apply_async(work, (host, port, filename, params)) results = [] for filename in filenames: response = threads[filename].get() d = json.loads(response) results.append(d) print(response_to_xml(d)) if json_file is not None: with open(json_file, "wb") as f: json.dump(results, f) if plot or table: from dials.algorithms.spot_finding.per_image_analysis import ( StatsMultiImage, plot_stats, ) estimated_d_min = flex.double() d_min_distl_method_1 = flex.double() d_min_distl_method_2 = flex.double() n_spots_total = flex.int() n_spots_no_ice = flex.int() total_intensity = flex.double() for d in results: estimated_d_min.append(d["estimated_d_min"]) d_min_distl_method_1.append(d["d_min_distl_method_1"]) d_min_distl_method_2.append(d["d_min_distl_method_2"]) n_spots_total.append(d["n_spots_total"]) n_spots_no_ice.append(d["n_spots_no_ice"]) total_intensity.append(d["total_intensity"]) stats = StatsMultiImage( n_spots_total=n_spots_total, n_spots_no_ice=n_spots_no_ice, n_spots_4A=None, total_intensity=total_intensity, estimated_d_min=estimated_d_min, d_min_distl_method_1=d_min_distl_method_1, d_min_distl_method_2=d_min_distl_method_2, noisiness_method_1=None, noisiness_method_2=None, ) if plot: plot_stats(stats) if table: print(stats) if grid is not None: from matplotlib import pyplot n_spots_no_ice.reshape(flex.grid(grid)) print(n_spots_no_ice.size()) pyplot.figure() pyplot.pcolormesh(n_spots_no_ice.as_numpy_array(), cmap=pyplot.cm.Reds) pyplot.savefig("spot_count.png")
def work_all(host, port, filenames, params, plot=False, table=False, json_file=None, grid=None, nproc=None): import json from multiprocessing.pool import ThreadPool as thread_pool if nproc is None: nproc=_nproc() pool = thread_pool(processes=nproc) threads = { } for filename in filenames: threads[filename] = pool.apply_async(work, (host, port, filename, params)) results = [] for filename in filenames: response = threads[filename].get() d = json.loads(response) results.append(d) print response_to_xml(d) if json_file is not None: 'Writing results to %s' %json_file with open(json_file, 'wb') as f: json.dump(results, f) if plot or table: from scitbx.array_family import flex from libtbx import group_args from dials.algorithms.spot_finding.per_image_analysis \ import plot_stats, print_table estimated_d_min = flex.double() d_min_distl_method_1 = flex.double() d_min_distl_method_2 = flex.double() n_spots_total = flex.int() n_spots_no_ice = flex.int() total_intensity = flex.double() for d in results: estimated_d_min.append(d['estimated_d_min']) d_min_distl_method_1.append(d['d_min_distl_method_1']) d_min_distl_method_2.append(d['d_min_distl_method_2']) n_spots_total.append(d['n_spots_total']) n_spots_no_ice.append(d['n_spots_no_ice']) total_intensity.append(d['total_intensity']) stats = group_args(n_spots_total=n_spots_total, n_spots_no_ice=n_spots_no_ice, n_spots_4A=None, total_intensity=total_intensity, estimated_d_min=estimated_d_min, d_min_distl_method_1=d_min_distl_method_1, d_min_distl_method_2=d_min_distl_method_2, noisiness_method_1=None, noisiness_method_2=None) if plot: plot_stats(stats) if table: print_table(stats) if grid is not None: from matplotlib import pyplot n_spots_no_ice.reshape(flex.grid(grid)) print n_spots_no_ice.size() from matplotlib import pyplot fig = pyplot.figure() pyplot.pcolormesh(n_spots_no_ice.as_numpy_array(), cmap=pyplot.cm.Reds) pyplot.savefig("spot_count.png") return
def run(args): usage = "dials.spot_counts_per_image [options] imported.expt strong.refl" parser = OptionParser( usage=usage, read_reflections=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if not any([reflections, experiments]): parser.print_help() return # FIXME may want to change this to allow many to be passed i.e. # from parallel runs if len(reflections) != 1: sys.exit("Only one reflection list may be passed") reflections = reflections[0] expts = set(reflections["id"]) if max(expts) >= len(experiments.imagesets()): sys.exit("Unknown experiments in reflection list") if params.id is not None: reflections = reflections.select(reflections["id"] == params.id) all_stats = [] for j, imageset in enumerate(experiments.imagesets()): refl = reflections.select(reflections["id"] == j) stats = per_image_analysis.stats_imageset( imageset, refl, resolution_analysis=params.resolution_analysis, plot=params.individual_plots, ) all_stats.append(stats) # transpose stats class empty(object): pass e = empty() for s in all_stats: for k in dir(s): if k.startswith("_") or k in ["merge", "next"]: continue if not hasattr(e, k): setattr(e, k, []) getattr(e, k).extend(getattr(s, k)) per_image_analysis.print_table(e) from libtbx import table_utils # FIXME this is now probably nonsense... overall_stats = per_image_analysis.stats_single_image( imageset, reflections, resolution_analysis=params.resolution_analysis ) rows = [ ("Overall statistics", ""), ("#spots", "%i" % overall_stats.n_spots_total), ("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice), ("d_min", "%.2f" % overall_stats.estimated_d_min), ( "d_min (distl method 1)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1), ), ( "d_min (distl method 2)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1), ), ] print(table_utils.format(rows, has_header=True, prefix="| ", postfix=" |")) if params.json: import json if params.split_json: for k in stats.__dict__: start, end = params.json.split(".") with open("%s_%s.%s" % (start, k, end), "wb") as fp: json.dump(stats.__dict__[k], fp) if params.joint_json: with open(params.json, "wb") as fp: json.dump(stats.__dict__, fp) if params.plot: import matplotlib matplotlib.use("Agg") per_image_analysis.plot_stats(stats, filename=params.plot)
def run(args): import libtbx.load_env usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, read_reflections=True, read_datablocks=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message) from libtbx.utils import Sorry params, options = parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) if not any([reflections, experiments, datablocks]): parser.print_help() return if len(reflections) != 1: raise Sorry('exactly 1 reflection table must be specified') if len(datablocks) != 1: if experiments: if len(experiments.imagesets()) != 1: raise Sorry('exactly 1 datablock must be specified') imageset = experiments.imagesets()[0] else: raise Sorry('exactly 1 datablock must be specified') else: imageset = datablocks[0].extract_imagesets()[0] reflections = reflections[0] if params.id is not None: reflections = reflections.select(reflections['id'] == params.id) stats = per_image_analysis.stats_imageset( imageset, reflections, resolution_analysis=params.resolution_analysis, plot=params.individual_plots) per_image_analysis.print_table(stats) from libtbx import table_utils overall_stats = per_image_analysis.stats_single_image( imageset, reflections, resolution_analysis=params.resolution_analysis) rows = [ ("Overall statistics", ""), ("#spots", "%i" % overall_stats.n_spots_total), ("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice), #("total_intensity", "%.0f" %overall_stats.total_intensity), ("d_min", "%.2f" % overall_stats.estimated_d_min), ("d_min (distl method 1)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ("d_min (distl method 2)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ] print table_utils.format(rows, has_header=True, prefix="| ", postfix=" |") if params.json is not None: import json with open(params.json, 'wb') as fp: json.dump(stats.__dict__, fp) if params.plot is not None: per_image_analysis.plot_stats(stats, filename=params.plot)
def run(args): import libtbx.load_env usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, read_reflections=True, read_datablocks=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message) from libtbx.utils import Sorry params, options = parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) if not any([reflections, experiments, datablocks]): parser.print_help() return if len(reflections) != 1: raise Sorry('exactly 1 reflection table must be specified') if len(datablocks) != 1: if experiments: if len(experiments.imagesets()) != 1: raise Sorry('exactly 1 datablock must be specified') imageset = experiments.imagesets()[0] else: raise Sorry('exactly 1 datablock must be specified') else: imageset = datablocks[0].extract_imagesets()[0] reflections = reflections[0] if params.id is not None: reflections = reflections.select(reflections['id'] == params.id) stats = per_image_analysis.stats_imageset( imageset, reflections, resolution_analysis=params.resolution_analysis, plot=params.individual_plots) per_image_analysis.print_table(stats) from libtbx import table_utils overall_stats = per_image_analysis.stats_single_image( imageset, reflections, resolution_analysis=params.resolution_analysis) rows = [ ("Overall statistics", ""), ("#spots", "%i" %overall_stats.n_spots_total), ("#spots_no_ice", "%i" %overall_stats.n_spots_no_ice), #("total_intensity", "%.0f" %overall_stats.total_intensity), ("d_min", "%.2f" %overall_stats.estimated_d_min), ("d_min (distl method 1)", "%.2f (%.2f)" %( overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ("d_min (distl method 2)", "%.2f (%.2f)" %( overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ] print table_utils.format(rows, has_header=True, prefix="| ", postfix=" |") if params.json is not None: import json with open(params.json, 'wb') as fp: json.dump(stats.__dict__, fp) if params.plot is not None: per_image_analysis.plot_stats(stats, filename=params.plot)
def run(args): from dials.util.options import OptionParser import libtbx.load_env usage = "%s [options] find_spots.json" % (libtbx.env.dispatcher_name) parser = OptionParser(usage=usage, phil=phil_scope, epilog=help_message) params, options, args = parser.parse_args(show_diff_phil=True, return_unhandled=True) positions = None if params.positions is not None: with open(params.positions, 'rb') as f: positions = flex.vec2_double() for line in f.readlines(): line = line.replace('(', ' ').replace(')', '').replace( ',', ' ').strip().split() assert len(line) == 3 i, x, y = [float(l) for l in line] positions.append((x, y)) assert len(args) == 1 json_file = args[0] import json with open(json_file, 'rb') as f: results = json.load(f) n_indexed = flex.double() fraction_indexed = flex.double() n_spots = flex.double() n_lattices = flex.double() crystals = [] image_names = flex.std_string() for r in results: n_spots.append(r['n_spots_total']) image_names.append(str(r['image'])) if 'n_indexed' in r: n_indexed.append(r['n_indexed']) n_lattices.append(len(r['lattices'])) for d in r['lattices']: from dxtbx.serialize.crystal import from_dict crystals.append(from_dict(d['crystal'])) else: n_indexed.append(0) n_lattices.append(0) if n_indexed.size(): sel = n_spots > 0 fraction_indexed = flex.double(n_indexed.size(), 0) fraction_indexed.set_selected( sel, n_indexed.select(sel) / n_spots.select(sel)) import matplotlib matplotlib.use('Agg') from matplotlib import pyplot blue = '#3498db' red = '#e74c3c' marker = 'o' alpha = 0.5 lw = 0 plot = True table = True grid = params.grid from libtbx import group_args from dials.algorithms.spot_finding.per_image_analysis \ import plot_stats, print_table estimated_d_min = flex.double() d_min_distl_method_1 = flex.double() d_min_distl_method_2 = flex.double() n_spots_total = flex.int() n_spots_no_ice = flex.int() total_intensity = flex.double() for d in results: estimated_d_min.append(d['estimated_d_min']) d_min_distl_method_1.append(d['d_min_distl_method_1']) d_min_distl_method_2.append(d['d_min_distl_method_2']) n_spots_total.append(d['n_spots_total']) n_spots_no_ice.append(d['n_spots_no_ice']) total_intensity.append(d['total_intensity']) stats = group_args(image=image_names, n_spots_total=n_spots_total, n_spots_no_ice=n_spots_no_ice, n_spots_4A=None, n_indexed=n_indexed, fraction_indexed=fraction_indexed, total_intensity=total_intensity, estimated_d_min=estimated_d_min, d_min_distl_method_1=d_min_distl_method_1, d_min_distl_method_2=d_min_distl_method_2, noisiness_method_1=None, noisiness_method_2=None) if plot: plot_stats(stats) pyplot.clf() if table: print_table(stats) n_rows = 10 n_rows = min(n_rows, len(n_spots_total)) perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True) print 'Top %i images sorted by number of spots:' % n_rows print_table(stats, perm=perm_n_spots_total, n_rows=n_rows) if flex.max(n_indexed) > 0: perm_n_indexed = flex.sort_permutation(n_indexed, reverse=True) print 'Top %i images sorted by number of indexed reflections:' % n_rows print_table(stats, perm=perm_n_indexed, n_rows=n_rows) print "Number of indexed lattices: ", (n_indexed > 0).count(True) print "Number with valid d_min but failed indexing: ", ( (d_min_distl_method_1 > 0) & (d_min_distl_method_2 > 0) & (estimated_d_min > 0) & (n_indexed == 0)).count(True) n_bins = 20 spot_count_histogram(n_spots_total, n_bins=n_bins, filename='hist_n_spots_total.png', log=True) spot_count_histogram(n_spots_no_ice, n_bins=n_bins, filename='hist_n_spots_no_ice.png', log=True) spot_count_histogram(n_indexed.select(n_indexed > 0), n_bins=n_bins, filename='hist_n_indexed.png', log=False) if len(crystals): plot_unit_cell_histograms(crystals) if params.stereographic_projections and len(crystals): from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_filenames([image_names[0]], verbose=False) assert len(datablocks) == 1 imageset = datablocks[0].extract_imagesets()[0] s0 = imageset.get_beam().get_s0() # XXX what if no goniometer? rotation_axis = imageset.get_goniometer().get_rotation_axis() indices = ((1, 0, 0), (0, 1, 0), (0, 0, 1)) for i, index in enumerate(indices): from cctbx import crystal, miller from scitbx import matrix miller_indices = flex.miller_index([index]) symmetry = crystal.symmetry( unit_cell=crystals[0].get_unit_cell(), space_group=crystals[0].get_space_group()) miller_set = miller.set(symmetry, miller_indices) d_spacings = miller_set.d_spacings() d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1() d_spacings = d_spacings.generate_bijvoet_mates() miller_indices = d_spacings.indices() # plane normal d0 = matrix.col(s0).normalize() d1 = d0.cross(matrix.col(rotation_axis)).normalize() d2 = d1.cross(d0).normalize() reference_poles = (d0, d1, d2) from dials.command_line.stereographic_projection import stereographic_projection projections = [] for cryst in crystals: reciprocal_space_points = list( cryst.get_U() * cryst.get_B()) * miller_indices.as_vec3_double() projections.append( stereographic_projection(reciprocal_space_points, reference_poles)) #from dials.algorithms.indexing.compare_orientation_matrices import \ # difference_rotation_matrix_and_euler_angles #R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles( # crystals[0], cryst) #print max(euler_angles) from dials.command_line.stereographic_projection import plot_projections plot_projections(projections, filename='projections_%s.png' % ('hkl'[i])) pyplot.clf() def plot_grid(values, grid, file_name, cmap=pyplot.cm.Reds, vmin=None, vmax=None, invalid='white'): values = values.as_double() # At DLS, fast direction appears to be largest direction if grid[0] > grid[1]: values.reshape(flex.grid(reversed(grid))) values = values.matrix_transpose() else: values.reshape(flex.grid(grid)) Z = values.as_numpy_array() #f, (ax1, ax2) = pyplot.subplots(2) f, ax1 = pyplot.subplots(1) mesh1 = ax1.pcolormesh(values.as_numpy_array(), cmap=cmap, vmin=vmin, vmax=vmax) mesh1.cmap.set_under(color=invalid, alpha=None) mesh1.cmap.set_over(color=invalid, alpha=None) #mesh2 = ax2.contour(Z, cmap=cmap, vmin=vmin, vmax=vmax) #mesh2 = ax2.contourf(Z, cmap=cmap, vmin=vmin, vmax=vmax) ax1.set_aspect('equal') ax1.invert_yaxis() #ax2.set_aspect('equal') #ax2.invert_yaxis() pyplot.colorbar(mesh1, ax=ax1) #pyplot.colorbar(mesh2, ax=ax2) pyplot.savefig(file_name, dpi=600) pyplot.clf() def plot_positions(values, positions, file_name, cmap=pyplot.cm.Reds, vmin=None, vmax=None, invalid='white'): values = values.as_double() assert positions.size() >= values.size() positions = positions[:values.size()] if vmin is None: vmin = flex.min(values) if vmax is None: vmax = flex.max(values) x, y = positions.parts() dx = flex.abs(x[1:] - x[:-1]) dy = flex.abs(y[1:] - y[:-1]) dx = dx.select(dx > 0) dy = dy.select(dy > 0) scale = 1 / flex.min(dx) #print scale x = (x * scale).iround() y = (y * scale).iround() from libtbx.math_utils import iceil z = flex.double( flex.grid(iceil(flex.max(y)) + 1, iceil(flex.max(x)) + 1), -2) #print z.all() for x_, y_, z_ in zip(x, y, values): z[y_, x_] = z_ plot_grid(z.as_1d(), z.all(), file_name, cmap=cmap, vmin=vmin, vmax=vmax, invalid=invalid) return if grid is not None or positions is not None: if grid is not None: positions = tuple(reversed(grid)) plotter = plot_grid else: plotter = plot_positions cmap = pyplot.get_cmap(params.cmap) plotter(n_spots_total, positions, 'grid_spot_count_total.png', cmap=cmap, invalid=params.invalid) plotter(n_spots_no_ice, positions, 'grid_spot_count_no_ice.png', cmap=cmap, invalid=params.invalid) plotter(total_intensity, positions, 'grid_total_intensity.png', cmap=cmap, invalid=params.invalid) if flex.max(n_indexed) > 0: plotter(n_indexed, positions, 'grid_n_indexed.png', cmap=cmap, invalid=params.invalid) plotter(fraction_indexed, positions, 'grid_fraction_indexed.png', cmap=cmap, vmin=0, vmax=1, invalid=params.invalid) for i, d_min in enumerate( (estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)): from cctbx import uctbx d_star_sq = uctbx.d_as_d_star_sq(d_min) d_star_sq.set_selected(d_star_sq == 1, 0) vmin = flex.min(d_star_sq.select(d_star_sq > 0)) vmax = flex.max(d_star_sq) vmin = flex.min(d_min.select(d_min > 0)) vmax = flex.max(d_min) cmap = pyplot.get_cmap('%s_r' % params.cmap) d_min.set_selected(d_min <= 0, vmax) if i == 0: plotter(d_min, positions, 'grid_d_min.png', cmap=cmap, vmin=vmin, vmax=vmax, invalid=params.invalid) else: plotter(d_min, positions, 'grid_d_min_method_%i.png' % i, cmap=cmap, vmin=vmin, vmax=vmax, invalid=params.invalid) if flex.max(n_indexed) > 0: pyplot.hexbin(n_spots, n_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter(n_spots, n_indexed, marker=marker, alpha=alpha, c=blue, lw=lw) xlim = pyplot.xlim() ylim = pyplot.ylim() pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel('# spots') pyplot.ylabel('# indexed') pyplot.savefig('n_spots_vs_n_indexed.png') pyplot.clf() pyplot.hexbin(n_spots, fraction_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('Fraction indexed') pyplot.savefig('n_spots_vs_fraction_indexed.png') pyplot.clf() pyplot.hexbin(n_indexed, fraction_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_indexed, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# indexed') pyplot.ylabel('Fraction indexed') pyplot.savefig('n_indexed_vs_fraction_indexed.png') pyplot.clf() pyplot.hexbin(n_spots, n_lattices, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, n_lattices, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('# lattices') pyplot.savefig('n_spots_vs_n_lattices.png') pyplot.clf() #pyplot.scatter( # estimated_d_min, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.hexbin(estimated_d_min, d_min_distl_method_1, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(estimated_d_min), max(d_min_distl_method_1)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel('estimated_d_min') pyplot.ylabel('d_min_distl_method_1') pyplot.savefig('d_min_vs_distl_method_1.png') pyplot.clf() #pyplot.scatter( # estimated_d_min, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.hexbin(estimated_d_min, d_min_distl_method_2, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(estimated_d_min), max(d_min_distl_method_2)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel('estimated_d_min') pyplot.ylabel('d_min_distl_method_2') pyplot.savefig('d_min_vs_distl_method_2.png') pyplot.clf() #pyplot.scatter( # d_min_distl_method_1, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.hexbin(d_min_distl_method_1, d_min_distl_method_2, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(d_min_distl_method_1), max(d_min_distl_method_2)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel('d_min_distl_method_1') pyplot.ylabel('d_min_distl_method_2') pyplot.savefig('distl_method_1_vs_distl_method_2.png') pyplot.clf() pyplot.hexbin(n_spots, estimated_d_min, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, estimated_d_min, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('estimated_d_min') pyplot.savefig('n_spots_vs_d_min.png') pyplot.clf() pyplot.hexbin(n_spots, d_min_distl_method_1, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('d_min_distl_method_1') pyplot.savefig('n_spots_vs_distl_method_1.png') pyplot.clf() pyplot.hexbin(n_spots, d_min_distl_method_2, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('d_min_distl_method_2') pyplot.savefig('n_spots_vs_distl_method_2.png') pyplot.clf()