def custom_stats_imageset(imageset,
reflections,
resolution_analysis=False,
plot=False):
from dials.algorithms.spot_finding import per_image_analysis
from libtbx import group_args
from dials.array_family import flex
n_spots_total = []
n_spots_no_ice = []
n_spots_4A = []
total_intensity = []
estimated_d_min = []
d_min_distl_method_1 = []
d_min_distl_method_2 = []
noisiness_method_1 = []
noisiness_method_2 = []
try:
start, end = imageset.get_array_range()
except AttributeError:
start = 0
for i in range(len(imageset)):
stats = per_image_analysis.stats_single_image(
imageset[i:i + 1],
reflections,
i=i + start,
#reflections.select(reflections['img_id']==i+start), i=i+start,
resolution_analysis=resolution_analysis,
plot=plot)
n_spots_total.append(stats.n_spots_total)
n_spots_no_ice.append(stats.n_spots_no_ice)
n_spots_4A.append(stats.n_spots_4A)
total_intensity.append(stats.total_intensity)
estimated_d_min.append(stats.estimated_d_min)
d_min_distl_method_1.append(stats.d_min_distl_method_1)
noisiness_method_1.append(stats.noisiness_method_1)
d_min_distl_method_2.append(stats.d_min_distl_method_2)
noisiness_method_2.append(stats.noisiness_method_2)
return group_args(n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=n_spots_4A,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
noisiness_method_1=noisiness_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_2=noisiness_method_2)
def run(args, verbose=False):
from libtbx.utils import Sorry
try:
from dials.array_family import flex
except ImportError:
return str(Sorry("DIALS is not configured"))
from iotbx.phil import parse
import os
from spotfinder.servers import LoggingFramework
from dials.array_family import flex
from dxtbx.model.experiment_list import ExperimentListFactory
phil_scope = parse("""
file_name = None
.type = str
frame_number = None
.type = int
stats = True
.type = bool
include scope dials.algorithms.spot_finding.factory.phil_scope
""",
process_includes=True)
#For the Apache server version, do not allow site, user, or dataset preferences
#all parameters are to be passed in through the http: query line
logfile = LoggingFramework()
phil_objects = []
for key in args.keys():
arg = "%s=%s" % (key, args.get(key, ""))
try:
phil_objects.append(parse(arg))
except Exception:
return str(Sorry("Unknown file or keyword: %s" % arg))
working_params = phil_scope.fetch(sources=phil_objects)
params = working_params.extract()
#working_params.show()
if not os.path.isfile(params.file_name):
return str(Sorry("%s is not a readable file" % params.file_name))
print "Image: %s\n" % params.file_name
try:
experiments = ExperimentListFactory.from_filenames([params.file_name])
assert len(experiments) == 1
if len(experiments[0].imageset
) > 0 and params.frame_number is not None:
print "Frame number", params.frame_number
experiments[0].imageset = experiments[0].imageset[
params.frame_number:params.frame_number + 1]
experiments[0].scan = experiments[0].imageset.get_scan()
reflections = flex.reflection_table.from_observations(
experiments, params)
if params.stats:
from dials.algorithms.spot_finding.per_image_analysis import stats_single_image
print stats_single_image(experiments[0].imageset,
reflections,
i=None,
resolution_analysis=True,
plot=False)
except Exception:
import traceback
logger = StringIO()
logger.write("Sorry, can't process %s. Please contact authors.\n" %
params.file_name)
traceback.print_exc(file=logger)
return str(Sorry(logger.getvalue())) + logfile.getvalue()
print "Found %d strong reflections" % len(reflections)
return logfile.getvalue()
def work(filename, cl=None):
if cl is None:
cl = []
import libtbx.phil
phil_scope = libtbx.phil.parse('''\
index = False
.type = bool
integrate = False
.type = bool
indexing_min_spots = 10
.type = int(value_min=1)
''')
if not os.access(filename, os.R_OK):
raise RuntimeError("Server does not have read access to file %s" %filename)
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
cl, custom_processor='collect_remaining')
index = params.extract().index
integrate = params.extract().integrate
indexing_min_spots = params.extract().indexing_min_spots
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
interp = find_spots_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
logger.info('The following spotfinding parameters have been modified:')
logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
params = phil_scope.extract()
# no need to write the hot mask in the server/client
params.spotfinder.write_hot_mask = False
datablock = DataBlockFactory.from_filenames([filename])[0]
t0 = time.time()
reflections = flex.reflection_table.from_observations(datablock, params)
t1 = time.time()
logger.info('Spotfinding took %.2f seconds' %(t1-t0))
from dials.algorithms.spot_finding import per_image_analysis
imageset = datablock.extract_imagesets()[0]
scan = imageset.get_scan()
if scan is not None:
i = scan.get_array_range()[0]
else:
i = 0
stats = per_image_analysis.stats_single_image(
imageset, reflections, i=i, plot=False)
stats = stats.__dict__
t2 = time.time()
logger.info('Resolution analysis took %.2f seconds' %(t2-t1))
if index and stats['n_spots_no_ice'] > indexing_min_spots:
import logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
from dials.algorithms.indexing import indexer
interp = indexer.master_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
imagesets = [imageset]
logger.info('The following indexing parameters have been modified:')
indexer.master_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
params.indexing.scan_range=[]
if (imageset.get_goniometer() is not None and
imageset.get_scan() is not None and
imageset.get_scan().get_oscillation()[1] == 0):
imageset.set_goniometer(None)
imageset.set_scan(None)
try:
idxr = indexer.indexer_base.from_parameters(
reflections, imagesets, params=params)
indexing_results = []
idxr.index()
indexed_sel = idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.indexed)
indexed_sel &= ~(idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.centroid_outlier))
for i_expt, expt in enumerate(idxr.refined_experiments):
sel = idxr.refined_reflections['id'] == i_expt
sel &= indexed_sel
indexing_results.append({
'crystal': expt.crystal.to_dict(),
'n_indexed': sel.count(True),
'fraction_indexed': sel.count(True)/sel.size()})
stats['lattices'] = indexing_results
stats['n_indexed'] = indexed_sel.count(True)
stats['fraction_indexed'] = indexed_sel.count(True)/len(reflections)
except Exception, e:
logger.error(e)
stats['error'] = str(e)
#stats.crystal = None
#stats.n_indexed = None
#stats.fraction_indexed = None
finally:
def work(filename, cl=None):
if cl is None:
cl = []
import libtbx.phil
phil_scope = libtbx.phil.parse('''\
ice_rings {
filter = True
.type = bool
width = 0.004
.type = float(value_min=0.0)
}
index = False
.type = bool
integrate = False
.type = bool
indexing_min_spots = 10
.type = int(value_min=1)
''')
if not os.access(filename, os.R_OK):
raise RuntimeError("Server does not have read access to file %s" %
filename)
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
cl, custom_processor='collect_remaining')
filter_ice = params.extract().ice_rings.filter
ice_rings_width = params.extract().ice_rings.width
index = params.extract().index
integrate = params.extract().integrate
indexing_min_spots = params.extract().indexing_min_spots
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
interp = find_spots_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
logger.info('The following spotfinding parameters have been modified:')
logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
params = phil_scope.extract()
# no need to write the hot mask in the server/client
params.spotfinder.write_hot_mask = False
datablock = DataBlockFactory.from_filenames([filename])[0]
t0 = time.time()
reflections = flex.reflection_table.from_observations(datablock, params)
t1 = time.time()
logger.info('Spotfinding took %.2f seconds' % (t1 - t0))
from dials.algorithms.spot_finding import per_image_analysis
imageset = datablock.extract_imagesets()[0]
scan = imageset.get_scan()
if scan is not None:
i = scan.get_array_range()[0]
else:
i = 0
stats = per_image_analysis.stats_single_image(
imageset,
reflections,
i=i,
plot=False,
filter_ice=filter_ice,
ice_rings_width=ice_rings_width)
stats = stats.__dict__
t2 = time.time()
logger.info('Resolution analysis took %.2f seconds' % (t2 - t1))
if index and stats['n_spots_no_ice'] > indexing_min_spots:
import logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
from dials.algorithms.indexing import indexer
interp = indexer.master_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
imagesets = [imageset]
logger.info('The following indexing parameters have been modified:')
indexer.master_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
params.indexing.scan_range = []
if (imageset.get_goniometer() is not None
and imageset.get_scan() is not None
and imageset.get_scan().get_oscillation()[1] == 0):
imageset.set_goniometer(None)
imageset.set_scan(None)
try:
idxr = indexer.indexer_base.from_parameters(reflections,
imagesets,
params=params)
indexing_results = []
idxr.index()
indexed_sel = idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.indexed)
indexed_sel &= ~(idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.centroid_outlier))
for i_expt, expt in enumerate(idxr.refined_experiments):
sel = idxr.refined_reflections['id'] == i_expt
sel &= indexed_sel
indexing_results.append({
'crystal':
expt.crystal.to_dict(),
'n_indexed':
sel.count(True),
'fraction_indexed':
sel.count(True) / sel.size()
})
stats['lattices'] = indexing_results
stats['n_indexed'] = indexed_sel.count(True)
stats['fraction_indexed'] = indexed_sel.count(True) / len(
reflections)
except Exception as e:
logger.error(e)
stats['error'] = str(e)
#stats.crystal = None
#stats.n_indexed = None
#stats.fraction_indexed = None
finally:
t3 = time.time()
logger.info('Indexing took %.2f seconds' % (t3 - t2))
if integrate and 'lattices' in stats:
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.command_line.integrate import phil_scope as integrate_phil_scope
interp = integrate_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
imagesets = [imageset]
logger.error(
'The following integration parameters have been modified:')
integrate_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
try:
params.profile.gaussian_rs.min_spots = 0
experiments = idxr.refined_experiments
reference = idxr.refined_reflections
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert (len(matched) == len(predicted))
assert (matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif matched.count(True) != len(reference):
logger.info('')
logger.info('*' * 80)
logger.info(
'Warning: %d reference spots were not matched to predictions'
% (len(reference) - matched.count(True)))
logger.info('*' * 80)
logger.info('')
# Compute the profile model
experiments = ProfileModelFactory.create(
params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments,
predicted)
# Integrate the reflections
reflections = integrator.integrate()
#print len(reflections)
stats['integrated_intensity'] = flex.sum(
reflections['intensity.sum.value'])
except Exception as e:
logger.error(e)
stats['error'] = str(e)
finally:
t4 = time.time()
logger.info('Integration took %.2f seconds' % (t4 - t3))
return stats
def run(args):
usage = "dials.spot_counts_per_image [options] imported.expt strong.refl"
parser = OptionParser(
usage=usage,
read_reflections=True,
read_experiments=True,
phil=phil_scope,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=False)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if not any([reflections, experiments]):
parser.print_help()
return
# FIXME may want to change this to allow many to be passed i.e.
# from parallel runs
if len(reflections) != 1:
sys.exit("Only one reflection list may be passed")
reflections = reflections[0]
expts = set(reflections["id"])
if max(expts) >= len(experiments.imagesets()):
sys.exit("Unknown experiments in reflection list")
if params.id is not None:
reflections = reflections.select(reflections["id"] == params.id)
all_stats = []
for j, imageset in enumerate(experiments.imagesets()):
refl = reflections.select(reflections["id"] == j)
stats = per_image_analysis.stats_imageset(
imageset,
refl,
resolution_analysis=params.resolution_analysis,
plot=params.individual_plots,
)
all_stats.append(stats)
# transpose stats
class empty(object):
pass
e = empty()
for s in all_stats:
for k in dir(s):
if k.startswith("_") or k in ["merge", "next"]:
continue
if not hasattr(e, k):
setattr(e, k, [])
getattr(e, k).extend(getattr(s, k))
per_image_analysis.print_table(e)
from libtbx import table_utils
# FIXME this is now probably nonsense...
overall_stats = per_image_analysis.stats_single_image(
imageset, reflections, resolution_analysis=params.resolution_analysis
)
rows = [
("Overall statistics", ""),
("#spots", "%i" % overall_stats.n_spots_total),
("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice),
("d_min", "%.2f" % overall_stats.estimated_d_min),
(
"d_min (distl method 1)",
"%.2f (%.2f)"
% (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1),
),
(
"d_min (distl method 2)",
"%.2f (%.2f)"
% (overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1),
),
]
print(table_utils.format(rows, has_header=True, prefix="| ", postfix=" |"))
if params.json:
import json
if params.split_json:
for k in stats.__dict__:
start, end = params.json.split(".")
with open("%s_%s.%s" % (start, k, end), "wb") as fp:
json.dump(stats.__dict__[k], fp)
if params.joint_json:
with open(params.json, "wb") as fp:
json.dump(stats.__dict__, fp)
if params.plot:
import matplotlib
matplotlib.use("Agg")
per_image_analysis.plot_stats(stats, filename=params.plot)
def run(args):
import libtbx.load_env
usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
read_reflections=True,
read_datablocks=True,
read_experiments=True,
phil=phil_scope,
check_format=False,
epilog=help_message)
from libtbx.utils import Sorry
params, options = parser.parse_args(show_diff_phil=False)
reflections = flatten_reflections(params.input.reflections)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
if not any([reflections, experiments, datablocks]):
parser.print_help()
return
if len(reflections) != 1:
raise Sorry('exactly 1 reflection table must be specified')
if len(datablocks) != 1:
if experiments:
if len(experiments.imagesets()) != 1:
raise Sorry('exactly 1 datablock must be specified')
imageset = experiments.imagesets()[0]
else:
raise Sorry('exactly 1 datablock must be specified')
else:
imageset = datablocks[0].extract_imagesets()[0]
reflections = reflections[0]
if params.id is not None:
reflections = reflections.select(reflections['id'] == params.id)
stats = per_image_analysis.stats_imageset(
imageset,
reflections,
resolution_analysis=params.resolution_analysis,
plot=params.individual_plots)
per_image_analysis.print_table(stats)
from libtbx import table_utils
overall_stats = per_image_analysis.stats_single_image(
imageset, reflections, resolution_analysis=params.resolution_analysis)
rows = [
("Overall statistics", ""),
("#spots", "%i" % overall_stats.n_spots_total),
("#spots_no_ice", "%i" % overall_stats.n_spots_no_ice),
#("total_intensity", "%.0f" %overall_stats.total_intensity),
("d_min", "%.2f" % overall_stats.estimated_d_min),
("d_min (distl method 1)",
"%.2f (%.2f)" % (overall_stats.d_min_distl_method_1,
overall_stats.noisiness_method_1)),
("d_min (distl method 2)",
"%.2f (%.2f)" % (overall_stats.d_min_distl_method_1,
overall_stats.noisiness_method_1)),
]
print table_utils.format(rows, has_header=True, prefix="| ", postfix=" |")
if params.json is not None:
import json
with open(params.json, 'wb') as fp:
json.dump(stats.__dict__, fp)
if params.plot is not None:
per_image_analysis.plot_stats(stats, filename=params.plot)
def work(filename, cl=None):
if cl is None:
cl = []
import libtbx.phil
phil_scope = libtbx.phil.parse('''\
index = False
.type = bool
integrate = False
.type = bool
indexing_min_spots = 10
.type = int(value_min=1)
''')
if not os.access(filename, os.R_OK):
raise RuntimeError("Server does not have read access to file %s" %filename)
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
cl, custom_processor='collect_remaining')
index = params.extract().index
integrate = params.extract().integrate
indexing_min_spots = params.extract().indexing_min_spots
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
interp = find_spots_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
logger.info('The following spotfinding parameters have been modified:')
logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
params = phil_scope.extract()
# no need to write the hot mask in the server/client
params.spotfinder.write_hot_mask = False
datablock = DataBlockFactory.from_filenames([filename])[0]
t0 = time.time()
reflections = flex.reflection_table.from_observations(datablock, params)
t1 = time.time()
logger.info('Spotfinding took %.2f seconds' %(t1-t0))
from dials.algorithms.spot_finding import per_image_analysis
imageset = datablock.extract_imagesets()[0]
scan = imageset.get_scan()
if scan is not None:
i = scan.get_array_range()[0]
else:
i = 0
stats = per_image_analysis.stats_single_image(
imageset, reflections, i=i, plot=False)
stats = stats.__dict__
t2 = time.time()
logger.info('Resolution analysis took %.2f seconds' %(t2-t1))
if index and stats['n_spots_no_ice'] > indexing_min_spots:
import logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
from dials.algorithms.indexing import indexer
from dxtbx.serialize.crystal import to_dict
interp = indexer.master_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
imagesets = [imageset]
logger.info('The following indexing parameters have been modified:')
indexer.master_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
params.indexing.scan_range=[]
if (imageset.get_goniometer() is not None and
imageset.get_scan() is not None and
imageset.get_scan().get_oscillation()[1] == 0):
imageset.set_goniometer(None)
imageset.set_scan(None)
try:
idxr = indexer.indexer_base.from_parameters(
reflections, imagesets, params=params)
indexing_results = []
indexed_sel = idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.indexed)
indexed_sel &= ~(idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.centroid_outlier))
for i_expt, expt in enumerate(idxr.refined_experiments):
sel = idxr.refined_reflections['id'] == i_expt
sel &= indexed_sel
indexing_results.append({
'crystal': to_dict(expt.crystal),
'n_indexed': sel.count(True),
'fraction_indexed': sel.count(True)/sel.size()})
stats['lattices'] = indexing_results
stats['n_indexed'] = indexed_sel.count(True)
stats['fraction_indexed'] = indexed_sel.count(True)/len(reflections)
except Exception, e:
logger.error(e)
stats['error'] = str(e)
#stats.crystal = None
#stats.n_indexed = None
#stats.fraction_indexed = None
finally:
def run(args):
import libtbx.load_env
usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name
parser = OptionParser(
usage=usage,
read_reflections=True,
read_datablocks=True,
read_experiments=True,
phil=phil_scope,
check_format=False,
epilog=help_message)
from libtbx.utils import Sorry
params, options = parser.parse_args(show_diff_phil=False)
reflections = flatten_reflections(params.input.reflections)
datablocks = flatten_datablocks(params.input.datablock)
experiments = flatten_experiments(params.input.experiments)
if not any([reflections, experiments, datablocks]):
parser.print_help()
return
if len(reflections) != 1:
raise Sorry('exactly 1 reflection table must be specified')
if len(datablocks) != 1:
if experiments:
if len(experiments.imagesets()) != 1:
raise Sorry('exactly 1 datablock must be specified')
imageset = experiments.imagesets()[0]
else:
raise Sorry('exactly 1 datablock must be specified')
else:
imageset = datablocks[0].extract_imagesets()[0]
reflections = reflections[0]
if params.id is not None:
reflections = reflections.select(reflections['id'] == params.id)
stats = per_image_analysis.stats_imageset(
imageset, reflections, resolution_analysis=params.resolution_analysis,
plot=params.individual_plots)
per_image_analysis.print_table(stats)
from libtbx import table_utils
overall_stats = per_image_analysis.stats_single_image(
imageset, reflections, resolution_analysis=params.resolution_analysis)
rows = [
("Overall statistics", ""),
("#spots", "%i" %overall_stats.n_spots_total),
("#spots_no_ice", "%i" %overall_stats.n_spots_no_ice),
#("total_intensity", "%.0f" %overall_stats.total_intensity),
("d_min", "%.2f" %overall_stats.estimated_d_min),
("d_min (distl method 1)", "%.2f (%.2f)" %(
overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)),
("d_min (distl method 2)", "%.2f (%.2f)" %(
overall_stats.d_min_distl_method_1, overall_stats.noisiness_method_1)),
]
print table_utils.format(rows, has_header=True, prefix="| ", postfix=" |")
if params.json is not None:
import json
with open(params.json, 'wb') as fp:
json.dump(stats.__dict__, fp)
if params.plot is not None:
per_image_analysis.plot_stats(stats, filename=params.plot)
