def custom_stats_imageset(imageset, reflections, resolution_analysis=False, plot=False): from dials.algorithms.spot_finding import per_image_analysis from libtbx import group_args from dials.array_family import flex n_spots_total = [] n_spots_no_ice = [] n_spots_4A = [] total_intensity = [] estimated_d_min = [] d_min_distl_method_1 = [] d_min_distl_method_2 = [] noisiness_method_1 = [] noisiness_method_2 = [] try: start, end = imageset.get_array_range() except AttributeError: start = 0 for i in range(len(imageset)): stats = per_image_analysis.stats_single_image( imageset[i:i + 1], reflections, i=i + start, #reflections.select(reflections['img_id']==i+start), i=i+start, resolution_analysis=resolution_analysis, plot=plot) n_spots_total.append(stats.n_spots_total) n_spots_no_ice.append(stats.n_spots_no_ice) n_spots_4A.append(stats.n_spots_4A) total_intensity.append(stats.total_intensity) estimated_d_min.append(stats.estimated_d_min) d_min_distl_method_1.append(stats.d_min_distl_method_1) noisiness_method_1.append(stats.noisiness_method_1) d_min_distl_method_2.append(stats.d_min_distl_method_2) noisiness_method_2.append(stats.noisiness_method_2) return group_args(n_spots_total=n_spots_total, n_spots_no_ice=n_spots_no_ice, n_spots_4A=n_spots_4A, total_intensity=total_intensity, estimated_d_min=estimated_d_min, d_min_distl_method_1=d_min_distl_method_1, noisiness_method_1=noisiness_method_1, d_min_distl_method_2=d_min_distl_method_2, noisiness_method_2=noisiness_method_2)
def run(args, verbose=False): from libtbx.utils import Sorry try: from dials.array_family import flex except ImportError: return str(Sorry("DIALS is not configured")) from iotbx.phil import parse import os from spotfinder.servers import LoggingFramework from dials.array_family import flex from dxtbx.model.experiment_list import ExperimentListFactory phil_scope = parse(""" file_name = None .type = str frame_number = None .type = int stats = True .type = bool include scope dials.algorithms.spot_finding.factory.phil_scope """, process_includes=True) #For the Apache server version, do not allow site, user, or dataset preferences #all parameters are to be passed in through the http: query line logfile = LoggingFramework() phil_objects = [] for key in args.keys(): arg = "%s=%s" % (key, args.get(key, "")) try: phil_objects.append(parse(arg)) except Exception: return str(Sorry("Unknown file or keyword: %s" % arg)) working_params = phil_scope.fetch(sources=phil_objects) params = working_params.extract() #working_params.show() if not os.path.isfile(params.file_name): return str(Sorry("%s is not a readable file" % params.file_name)) print "Image: %s\n" % params.file_name try: experiments = ExperimentListFactory.from_filenames([params.file_name]) assert len(experiments) == 1 if len(experiments[0].imageset ) > 0 and params.frame_number is not None: print "Frame number", params.frame_number experiments[0].imageset = experiments[0].imageset[ params.frame_number:params.frame_number + 1] experiments[0].scan = experiments[0].imageset.get_scan() reflections = flex.reflection_table.from_observations( experiments, params) if params.stats: from dials.algorithms.spot_finding.per_image_analysis import stats_single_image print stats_single_image(experiments[0].imageset, reflections, i=None, resolution_analysis=True, plot=False) except Exception: import traceback logger = StringIO() logger.write("Sorry, can't process %s. Please contact authors.\n" % params.file_name) traceback.print_exc(file=logger) return str(Sorry(logger.getvalue())) + logfile.getvalue() print "Found %d strong reflections" % len(reflections) return logfile.getvalue()
def work(filename, cl=None): if cl is None: cl = [] import libtbx.phil phil_scope = libtbx.phil.parse('''\ index = False .type = bool integrate = False .type = bool indexing_min_spots = 10 .type = int(value_min=1) ''') if not os.access(filename, os.R_OK): raise RuntimeError("Server does not have read access to file %s" %filename) interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( cl, custom_processor='collect_remaining') index = params.extract().index integrate = params.extract().integrate indexing_min_spots = params.extract().indexing_min_spots from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from dxtbx.datablock import DataBlockFactory from dials.array_family import flex interp = find_spots_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') logger.info('The following spotfinding parameters have been modified:') logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str()) params = phil_scope.extract() # no need to write the hot mask in the server/client params.spotfinder.write_hot_mask = False datablock = DataBlockFactory.from_filenames([filename])[0] t0 = time.time() reflections = flex.reflection_table.from_observations(datablock, params) t1 = time.time() logger.info('Spotfinding took %.2f seconds' %(t1-t0)) from dials.algorithms.spot_finding import per_image_analysis imageset = datablock.extract_imagesets()[0] scan = imageset.get_scan() if scan is not None: i = scan.get_array_range()[0] else: i = 0 stats = per_image_analysis.stats_single_image( imageset, reflections, i=i, plot=False) stats = stats.__dict__ t2 = time.time() logger.info('Resolution analysis took %.2f seconds' %(t2-t1)) if index and stats['n_spots_no_ice'] > indexing_min_spots: import logging logging.basicConfig(stream=sys.stdout, level=logging.INFO) from dials.algorithms.indexing import indexer interp = indexer.master_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') imagesets = [imageset] logger.info('The following indexing parameters have been modified:') indexer.master_phil_scope.fetch_diff(source=phil_scope).show() params = phil_scope.extract() params.indexing.scan_range=[] if (imageset.get_goniometer() is not None and imageset.get_scan() is not None and imageset.get_scan().get_oscillation()[1] == 0): imageset.set_goniometer(None) imageset.set_scan(None) try: idxr = indexer.indexer_base.from_parameters( reflections, imagesets, params=params) indexing_results = [] idxr.index() indexed_sel = idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.indexed) indexed_sel &= ~(idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.centroid_outlier)) for i_expt, expt in enumerate(idxr.refined_experiments): sel = idxr.refined_reflections['id'] == i_expt sel &= indexed_sel indexing_results.append({ 'crystal': expt.crystal.to_dict(), 'n_indexed': sel.count(True), 'fraction_indexed': sel.count(True)/sel.size()}) stats['lattices'] = indexing_results stats['n_indexed'] = indexed_sel.count(True) stats['fraction_indexed'] = indexed_sel.count(True)/len(reflections) except Exception, e: logger.error(e) stats['error'] = str(e) #stats.crystal = None #stats.n_indexed = None #stats.fraction_indexed = None finally:
def work(filename, cl=None): if cl is None: cl = [] import libtbx.phil phil_scope = libtbx.phil.parse('''\ ice_rings { filter = True .type = bool width = 0.004 .type = float(value_min=0.0) } index = False .type = bool integrate = False .type = bool indexing_min_spots = 10 .type = int(value_min=1) ''') if not os.access(filename, os.R_OK): raise RuntimeError("Server does not have read access to file %s" % filename) interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( cl, custom_processor='collect_remaining') filter_ice = params.extract().ice_rings.filter ice_rings_width = params.extract().ice_rings.width index = params.extract().index integrate = params.extract().integrate indexing_min_spots = params.extract().indexing_min_spots from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from dxtbx.datablock import DataBlockFactory from dials.array_family import flex interp = find_spots_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') logger.info('The following spotfinding parameters have been modified:') logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str()) params = phil_scope.extract() # no need to write the hot mask in the server/client params.spotfinder.write_hot_mask = False datablock = DataBlockFactory.from_filenames([filename])[0] t0 = time.time() reflections = flex.reflection_table.from_observations(datablock, params) t1 = time.time() logger.info('Spotfinding took %.2f seconds' % (t1 - t0)) from dials.algorithms.spot_finding import per_image_analysis imageset = datablock.extract_imagesets()[0] scan = imageset.get_scan() if scan is not None: i = scan.get_array_range()[0] else: i = 0 stats = per_image_analysis.stats_single_image( imageset, reflections, i=i, plot=False, filter_ice=filter_ice, ice_rings_width=ice_rings_width) stats = stats.__dict__ t2 = time.time() logger.info('Resolution analysis took %.2f seconds' % (t2 - t1)) if index and stats['n_spots_no_ice'] > indexing_min_spots: import logging logging.basicConfig(stream=sys.stdout, level=logging.INFO) from dials.algorithms.indexing import indexer interp = indexer.master_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') imagesets = [imageset] logger.info('The following indexing parameters have been modified:') indexer.master_phil_scope.fetch_diff(source=phil_scope).show() params = phil_scope.extract() params.indexing.scan_range = [] if (imageset.get_goniometer() is not None and imageset.get_scan() is not None and imageset.get_scan().get_oscillation()[1] == 0): imageset.set_goniometer(None) imageset.set_scan(None) try: idxr = indexer.indexer_base.from_parameters(reflections, imagesets, params=params) indexing_results = [] idxr.index() indexed_sel = idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.indexed) indexed_sel &= ~(idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.centroid_outlier)) for i_expt, expt in enumerate(idxr.refined_experiments): sel = idxr.refined_reflections['id'] == i_expt sel &= indexed_sel indexing_results.append({ 'crystal': expt.crystal.to_dict(), 'n_indexed': sel.count(True), 'fraction_indexed': sel.count(True) / sel.size() }) stats['lattices'] = indexing_results stats['n_indexed'] = indexed_sel.count(True) stats['fraction_indexed'] = indexed_sel.count(True) / len( reflections) except Exception as e: logger.error(e) stats['error'] = str(e) #stats.crystal = None #stats.n_indexed = None #stats.fraction_indexed = None finally: t3 = time.time() logger.info('Indexing took %.2f seconds' % (t3 - t2)) if integrate and 'lattices' in stats: from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.command_line.integrate import phil_scope as integrate_phil_scope interp = integrate_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') imagesets = [imageset] logger.error( 'The following integration parameters have been modified:') integrate_phil_scope.fetch_diff(source=phil_scope).show() params = phil_scope.extract() try: params.profile.gaussian_rs.min_spots = 0 experiments = idxr.refined_experiments reference = idxr.refined_reflections predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static) matched, reference, unmatched = predicted.match_with_reference( reference) assert (len(matched) == len(predicted)) assert (matched.count(True) <= len(reference)) if matched.count(True) == 0: raise Sorry(''' Invalid input for reference reflections. Zero reference spots were matched to predictions ''') elif matched.count(True) != len(reference): logger.info('') logger.info('*' * 80) logger.info( 'Warning: %d reference spots were not matched to predictions' % (len(reference) - matched.count(True))) logger.info('*' * 80) logger.info('') # Compute the profile model experiments = ProfileModelFactory.create( params, experiments, reference) # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() #print len(reflections) stats['integrated_intensity'] = flex.sum( reflections['intensity.sum.value']) except Exception as e: logger.error(e) stats['error'] = str(e) finally: t4 = time.time() logger.info('Integration took %.2f seconds' % (t4 - t3)) return stats
def run(args): usage = "dials.spot_counts_per_image [options] imported.expt strong.refl" parser = OptionParser( usage=usage, read_reflections=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if not any([reflections, experiments]): parser.print_help() return # FIXME may want to change this to allow many to be passed i.e. # from parallel runs if len(reflections) != 1: sys.exit("Only one reflection list may be passed") reflections = reflections[0] expts = set(reflections["id"]) if max(expts) >= len(experiments.imagesets()): sys.exit("Unknown experiments in reflection list") if params.id is not None: reflections = reflections.select(reflections["id"] == params.id) all_stats = [] for j, imageset in enumerate(experiments.imagesets()): refl = reflections.select(reflections["id"] == j) stats = per_image_analysis.stats_imageset( imageset, refl, resolution_analysis=params.resolution_analysis, plot=params.individual_plots, ) all_stats.append(stats) # transpose stats class empty(object): pass e = empty() for s in all_stats: for k in dir(s): if k.startswith("_") or k in ["merge", "next"]: continue if not hasattr(e, k): setattr(e, k, []) getattr(e, k).extend(getattr(s, k)) per_image_analysis.print_table(e) from libtbx import table_utils # FIXME this is now probably nonsense... overall_stats = per_image_analysis.stats_single_image( imageset, reflections, resolution_analysis=params.resolution_analysis ) rows = [ ("Overall statistics", ""), ("#spots", "%i" % overall_stats.n_spots_total), ("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice), ("d_min", "%.2f" % overall_stats.estimated_d_min), ( "d_min (distl method 1)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1), ), ( "d_min (distl method 2)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1), ), ] print(table_utils.format(rows, has_header=True, prefix="| ", postfix=" |")) if params.json: import json if params.split_json: for k in stats.__dict__: start, end = params.json.split(".") with open("%s_%s.%s" % (start, k, end), "wb") as fp: json.dump(stats.__dict__[k], fp) if params.joint_json: with open(params.json, "wb") as fp: json.dump(stats.__dict__, fp) if params.plot: import matplotlib matplotlib.use("Agg") per_image_analysis.plot_stats(stats, filename=params.plot)
def run(args): import libtbx.load_env usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, read_reflections=True, read_datablocks=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message) from libtbx.utils import Sorry params, options = parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) if not any([reflections, experiments, datablocks]): parser.print_help() return if len(reflections) != 1: raise Sorry('exactly 1 reflection table must be specified') if len(datablocks) != 1: if experiments: if len(experiments.imagesets()) != 1: raise Sorry('exactly 1 datablock must be specified') imageset = experiments.imagesets()[0] else: raise Sorry('exactly 1 datablock must be specified') else: imageset = datablocks[0].extract_imagesets()[0] reflections = reflections[0] if params.id is not None: reflections = reflections.select(reflections['id'] == params.id) stats = per_image_analysis.stats_imageset( imageset, reflections, resolution_analysis=params.resolution_analysis, plot=params.individual_plots) per_image_analysis.print_table(stats) from libtbx import table_utils overall_stats = per_image_analysis.stats_single_image( imageset, reflections, resolution_analysis=params.resolution_analysis) rows = [ ("Overall statistics", ""), ("#spots", "%i" % overall_stats.n_spots_total), ("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice), #("total_intensity", "%.0f" %overall_stats.total_intensity), ("d_min", "%.2f" % overall_stats.estimated_d_min), ("d_min (distl method 1)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ("d_min (distl method 2)", "%.2f (%.2f)" % (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ] print table_utils.format(rows, has_header=True, prefix="| ", postfix=" |") if params.json is not None: import json with open(params.json, 'wb') as fp: json.dump(stats.__dict__, fp) if params.plot is not None: per_image_analysis.plot_stats(stats, filename=params.plot)
def work(filename, cl=None): if cl is None: cl = [] import libtbx.phil phil_scope = libtbx.phil.parse('''\ index = False .type = bool integrate = False .type = bool indexing_min_spots = 10 .type = int(value_min=1) ''') if not os.access(filename, os.R_OK): raise RuntimeError("Server does not have read access to file %s" %filename) interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( cl, custom_processor='collect_remaining') index = params.extract().index integrate = params.extract().integrate indexing_min_spots = params.extract().indexing_min_spots from dials.command_line.find_spots import phil_scope as find_spots_phil_scope from dxtbx.datablock import DataBlockFactory from dials.array_family import flex interp = find_spots_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') logger.info('The following spotfinding parameters have been modified:') logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str()) params = phil_scope.extract() # no need to write the hot mask in the server/client params.spotfinder.write_hot_mask = False datablock = DataBlockFactory.from_filenames([filename])[0] t0 = time.time() reflections = flex.reflection_table.from_observations(datablock, params) t1 = time.time() logger.info('Spotfinding took %.2f seconds' %(t1-t0)) from dials.algorithms.spot_finding import per_image_analysis imageset = datablock.extract_imagesets()[0] scan = imageset.get_scan() if scan is not None: i = scan.get_array_range()[0] else: i = 0 stats = per_image_analysis.stats_single_image( imageset, reflections, i=i, plot=False) stats = stats.__dict__ t2 = time.time() logger.info('Resolution analysis took %.2f seconds' %(t2-t1)) if index and stats['n_spots_no_ice'] > indexing_min_spots: import logging logging.basicConfig(stream=sys.stdout, level=logging.INFO) from dials.algorithms.indexing import indexer from dxtbx.serialize.crystal import to_dict interp = indexer.master_phil_scope.command_line_argument_interpreter() phil_scope, unhandled = interp.process_and_fetch( unhandled, custom_processor='collect_remaining') imagesets = [imageset] logger.info('The following indexing parameters have been modified:') indexer.master_phil_scope.fetch_diff(source=phil_scope).show() params = phil_scope.extract() params.indexing.scan_range=[] if (imageset.get_goniometer() is not None and imageset.get_scan() is not None and imageset.get_scan().get_oscillation()[1] == 0): imageset.set_goniometer(None) imageset.set_scan(None) try: idxr = indexer.indexer_base.from_parameters( reflections, imagesets, params=params) indexing_results = [] indexed_sel = idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.indexed) indexed_sel &= ~(idxr.refined_reflections.get_flags( idxr.refined_reflections.flags.centroid_outlier)) for i_expt, expt in enumerate(idxr.refined_experiments): sel = idxr.refined_reflections['id'] == i_expt sel &= indexed_sel indexing_results.append({ 'crystal': to_dict(expt.crystal), 'n_indexed': sel.count(True), 'fraction_indexed': sel.count(True)/sel.size()}) stats['lattices'] = indexing_results stats['n_indexed'] = indexed_sel.count(True) stats['fraction_indexed'] = indexed_sel.count(True)/len(reflections) except Exception, e: logger.error(e) stats['error'] = str(e) #stats.crystal = None #stats.n_indexed = None #stats.fraction_indexed = None finally:
def run(args): import libtbx.load_env usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, read_reflections=True, read_datablocks=True, read_experiments=True, phil=phil_scope, check_format=False, epilog=help_message) from libtbx.utils import Sorry params, options = parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) if not any([reflections, experiments, datablocks]): parser.print_help() return if len(reflections) != 1: raise Sorry('exactly 1 reflection table must be specified') if len(datablocks) != 1: if experiments: if len(experiments.imagesets()) != 1: raise Sorry('exactly 1 datablock must be specified') imageset = experiments.imagesets()[0] else: raise Sorry('exactly 1 datablock must be specified') else: imageset = datablocks[0].extract_imagesets()[0] reflections = reflections[0] if params.id is not None: reflections = reflections.select(reflections['id'] == params.id) stats = per_image_analysis.stats_imageset( imageset, reflections, resolution_analysis=params.resolution_analysis, plot=params.individual_plots) per_image_analysis.print_table(stats) from libtbx import table_utils overall_stats = per_image_analysis.stats_single_image( imageset, reflections, resolution_analysis=params.resolution_analysis) rows = [ ("Overall statistics", ""), ("#spots", "%i" %overall_stats.n_spots_total), ("#spots_no_ice", "%i" %overall_stats.n_spots_no_ice), #("total_intensity", "%.0f" %overall_stats.total_intensity), ("d_min", "%.2f" %overall_stats.estimated_d_min), ("d_min (distl method 1)", "%.2f (%.2f)" %( overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ("d_min (distl method 2)", "%.2f (%.2f)" %( overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)), ] print table_utils.format(rows, has_header=True, prefix="| ", postfix=" |") if params.json is not None: import json with open(params.json, 'wb') as fp: json.dump(stats.__dict__, fp) if params.plot is not None: per_image_analysis.plot_stats(stats, filename=params.plot)