def model(tmpdir):
from dials.algorithms.background.gmodel import StaticBackgroundModel
from dials.array_family import flex
ysize = 2527
xsize = 2463
data = flex.double(flex.grid(ysize, xsize), 1)
model = StaticBackgroundModel()
model.add(data)
model_file = tmpdir.join("model.pickle")
with model_file.open("wb") as fh:
pickle.dump(model, fh, pickle.HIGHEST_PROTOCOL)
return model_file
def run(self, args=None):
"""Execute the script."""
from dials.algorithms.background.modeller import BackgroundModeller
from dials.array_family import flex
from dials.util.command_line import heading
from dials.util.options import flatten_experiments
# Parse the command line
params, options = self.parser.parse_args(args, show_diff_phil=False)
# Configure the logging
dials.util.log.config(verbosity=options.verbose,
logfile=params.output.log)
if params.integration.mp.nproc != 1 or params.integration.mp.njobs != 1:
# https://github.com/dials/dials/issues/1083
logger.warning("Multiprocessing is currently disabled. "
"Setting nproc = njobs = 1")
params.integration.mp.nproc = 1
params.integration.mp.njobs = 1
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
# Ensure we have a data block
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
# Only handle a single imageset at once
imagesets = {expr.imageset for expr in experiments}
if len(imagesets) != 1:
sys.exit("Can only process a single imageset at a time")
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
)
# Create the modeller
modeller = BackgroundModeller(experiments, predicted, params)
model = modeller.compute()
# Save the background model
logger.info("Saving background model to %s" % params.output.model)
from dials.algorithms.background.gmodel import StaticBackgroundModel
static_model = StaticBackgroundModel()
for m in model:
static_model.add(m.model)
with open(params.output.model, "wb") as outfile:
pickle.dump(static_model,
outfile,
protocol=pickle.HIGHEST_PROTOCOL)
# Output some diagnostic images
image_generator = ImageGenerator(model)
image_generator.save_mean(params.output.mean_image_prefix)
image_generator.save_variance(params.output.variance_image_prefix)
image_generator.save_dispersion(params.output.dispersion_image_prefix)
image_generator.save_mask(params.output.mask_image_prefix)
image_generator.save_min(params.output.min_image_prefix)
image_generator.save_max(params.output.max_image_prefix)
image_generator.save_model(params.output.model_image_prefix)
def run(self):
'''Execute the script.'''
from dials.util.command_line import heading
from dials.array_family import flex
from dials.util.options import flatten_experiments
from time import time
from dials.util import log
from libtbx.utils import Sorry
from dials.algorithms.background.modeller import BackgroundModeller
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
# Configure the logging
log.config(
params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Ensure we have a data block
experiments = flatten_experiments(params.input.experiments)
if len(experiments) == 0:
self.parser.print_help()
return
# Only handle a single imageset at once
imagesets = set(expr.imageset for expr in experiments)
if len(imagesets) != 1:
raise Sorry("Can only process a single imageset at a time")
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Create the modeller
modeller = BackgroundModeller(experiments, predicted, params)
model = modeller.compute()
# Save the background model
logger.info("Saving background model to %s" % params.output.model)
from dials.algorithms.background.gmodel import StaticBackgroundModel
static_model = StaticBackgroundModel()
for i in range(len(model)):
static_model.add(model[i].model)
with open(params.output.model, "w") as outfile:
import cPickle as pickle
pickle.dump(static_model, outfile, protocol=pickle.HIGHEST_PROTOCOL)
# Output some diagnostic images
image_generator = ImageGenerator(model)
image_generator.save_mean(params.output.mean_image_prefix)
image_generator.save_variance(params.output.variance_image_prefix)
image_generator.save_dispersion(params.output.dispersion_image_prefix)
image_generator.save_mask(params.output.mask_image_prefix)
image_generator.save_min(params.output.min_image_prefix)
image_generator.save_max(params.output.max_image_prefix)
image_generator.save_model(params.output.model_image_prefix)
#image_generator.save_polar_model(params.output.polar_model_image_prefix)
# Print the time
logger.info("Time Taken: %f" % (time() - start_time))
def run(self):
from os.path import join, exists
from libtbx import easy_run
import os
from uuid import uuid4
os.mkdir('simple')
os.mkdir('robust')
os.mkdir('gmodel_simple')
os.mkdir('gmodel_robust')
assert exists(join(self.path, 'experiments.json'))
from dials.array_family import flex
from dials.algorithms.background.gmodel import StaticBackgroundModel
ysize = 2527
xsize = 2463
data = flex.double(flex.grid(ysize, xsize), 1)
model = StaticBackgroundModel()
model.add(data)
import cPickle as pickle
pickle.dump(model, open("model.pickle", "w"))
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
join(self.path, 'experiments.json'),
'profile.fitting=False',
'background.algorithm=simple',
'background.simple.outlier.algorithm=null',
'output.reflections=./simple/reflections.pickle'
]).raise_if_errors()
assert exists("simple/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
join(self.path, 'experiments.json'),
'profile.fitting=False',
'background.algorithm=glm',
'output.reflections=./robust/reflections.pickle'
]).raise_if_errors()
assert exists("robust/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
join(self.path, 'experiments.json'),
'profile.fitting=False',
'background.algorithm=gmodel',
'background.gmodel.robust.algorithm=False',
'background.gmodel.model=model.pickle',
'output.reflections=./gmodel_simple/reflections.pickle'
]).raise_if_errors()
assert exists("gmodel_simple/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
join(self.path, 'experiments.json'),
'profile.fitting=False',
'background.algorithm=gmodel',
'background.gmodel.robust.algorithm=True',
'background.gmodel.model=model.pickle',
'output.reflections=./gmodel_robust/reflections.pickle'
]).raise_if_errors()
assert exists("gmodel_robust/reflections.pickle")
from dials.array_family import flex
reflections1 = flex.reflection_table.from_pickle("simple/reflections.pickle")
reflections2 = flex.reflection_table.from_pickle("robust/reflections.pickle")
reflections3 = flex.reflection_table.from_pickle("gmodel_simple/reflections.pickle")
reflections4 = flex.reflection_table.from_pickle("gmodel_robust/reflections.pickle")
assert len(reflections1) == len(reflections3)
assert len(reflections2) == len(reflections4)
flag = flex.reflection_table.flags.integrated_sum
integrated1 = reflections1.select(reflections1.get_flags(flag, all=True))
integrated2 = reflections2.select(reflections2.get_flags(flag, all=True))
integrated3 = reflections3.select(reflections3.get_flags(flag, all=True))
integrated4 = reflections4.select(reflections4.get_flags(flag, all=True))
assert len(integrated1) > 0
assert len(integrated2) > 0
assert len(integrated1) == len(integrated3)
assert len(integrated2) == len(integrated4)
mean_bg1 = integrated1['background.mean']
mean_bg2 = integrated2['background.mean']
mean_bg3 = integrated3['background.mean']
mean_bg4 = integrated4['background.mean']
scale3 = integrated3['background.scale']
scale4 = integrated4['background.scale']
diff1 = flex.abs(mean_bg1 - mean_bg3)
diff2 = flex.abs(mean_bg2 - mean_bg4)
assert (scale3 > 0).count(False) == 0
assert (scale4 > 0).count(False) == 0
assert (diff1 < 1e-5).count(False) == 0
print 'OK'
def run(self):
from os.path import join, exists
from libtbx import easy_run
import os
from uuid import uuid4
os.mkdir("simple")
os.mkdir("robust")
os.mkdir("gmodel_simple")
os.mkdir("gmodel_robust")
assert exists(join(self.path, "experiments.json"))
from dials.array_family import flex
from dials.algorithms.background.gmodel import StaticBackgroundModel
ysize = 2527
xsize = 2463
data = flex.double(flex.grid(ysize, xsize), 1)
model = StaticBackgroundModel()
model.add(data)
import cPickle as pickle
pickle.dump(model, open("model.pickle", "w"))
# Call dials.integrate
easy_run.fully_buffered(
[
"dials.integrate",
join(self.path, "experiments.json"),
"profile.fitting=False",
"background.algorithm=simple",
"background.simple.outlier.algorithm=null",
"output.reflections=./simple/reflections.pickle",
]
).raise_if_errors()
assert exists("simple/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered(
[
"dials.integrate",
join(self.path, "experiments.json"),
"profile.fitting=False",
"background.algorithm=glm",
"output.reflections=./robust/reflections.pickle",
]
).raise_if_errors()
assert exists("robust/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered(
[
"dials.integrate",
join(self.path, "experiments.json"),
"profile.fitting=False",
"background.algorithm=gmodel",
"background.gmodel.robust.algorithm=False",
"background.gmodel.model=model.pickle",
"output.reflections=./gmodel_simple/reflections.pickle",
]
).raise_if_errors()
assert exists("gmodel_simple/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered(
[
"dials.integrate",
join(self.path, "experiments.json"),
"profile.fitting=False",
"background.algorithm=gmodel",
"background.gmodel.robust.algorithm=True",
"background.gmodel.model=model.pickle",
"output.reflections=./gmodel_robust/reflections.pickle",
]
).raise_if_errors()
assert exists("gmodel_robust/reflections.pickle")
from dials.array_family import flex
reflections1 = flex.reflection_table.from_pickle("simple/reflections.pickle")
reflections2 = flex.reflection_table.from_pickle("robust/reflections.pickle")
reflections3 = flex.reflection_table.from_pickle("gmodel_simple/reflections.pickle")
reflections4 = flex.reflection_table.from_pickle("gmodel_robust/reflections.pickle")
assert len(reflections1) == len(reflections3)
assert len(reflections2) == len(reflections4)
flag = flex.reflection_table.flags.integrated_sum
integrated1 = reflections1.select(reflections1.get_flags(flag, all=True))
integrated2 = reflections2.select(reflections2.get_flags(flag, all=True))
integrated3 = reflections3.select(reflections3.get_flags(flag, all=True))
integrated4 = reflections4.select(reflections4.get_flags(flag, all=True))
assert len(integrated1) > 0
assert len(integrated2) > 0
assert len(integrated1) == len(integrated3)
assert len(integrated2) == len(integrated4)
mean_bg1 = integrated1["background.mean"]
mean_bg2 = integrated2["background.mean"]
mean_bg3 = integrated3["background.mean"]
mean_bg4 = integrated4["background.mean"]
scale3 = integrated3["background.scale"]
scale4 = integrated4["background.scale"]
diff1 = flex.abs(mean_bg1 - mean_bg3)
diff2 = flex.abs(mean_bg2 - mean_bg4)
assert (scale3 > 0).count(False) == 0
assert (scale4 > 0).count(False) == 0
assert (diff1 < 1e-5).count(False) == 0
print "OK"
def test_run(dials_regression, tmpdir):
import os
path = os.path.join(dials_regression, "centroid_test_data")
tmpdir.chdir()
from libtbx import easy_run
os.mkdir('simple')
os.mkdir('robust')
os.mkdir('gmodel_simple')
os.mkdir('gmodel_robust')
assert os.path.exists(os.path.join(path, 'experiments.json'))
from dials.array_family import flex
from dials.algorithms.background.gmodel import StaticBackgroundModel
ysize = 2527
xsize = 2463
data = flex.double(flex.grid(ysize, xsize), 1)
model = StaticBackgroundModel()
model.add(data)
import six.moves.cPickle as pickle
with open("model.pickle", "wb") as fh:
pickle.dump(model, fh, pickle.HIGHEST_PROTOCOL)
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
os.path.join(path, 'experiments.json'), 'profile.fitting=False',
'background.algorithm=simple',
'background.simple.outlier.algorithm=null',
'output.reflections=./simple/reflections.pickle'
]).raise_if_errors()
assert os.path.exists("simple/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
os.path.join(path, 'experiments.json'), 'profile.fitting=False',
'background.algorithm=glm',
'output.reflections=./robust/reflections.pickle'
]).raise_if_errors()
assert os.path.exists("robust/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
os.path.join(path, 'experiments.json'), 'profile.fitting=False',
'background.algorithm=gmodel',
'background.gmodel.robust.algorithm=False',
'background.gmodel.model=model.pickle',
'output.reflections=./gmodel_simple/reflections.pickle'
]).raise_if_errors()
assert os.path.exists("gmodel_simple/reflections.pickle")
# Call dials.integrate
easy_run.fully_buffered([
'dials.integrate',
os.path.join(path, 'experiments.json'), 'profile.fitting=False',
'background.algorithm=gmodel',
'background.gmodel.robust.algorithm=True',
'background.gmodel.model=model.pickle',
'output.reflections=./gmodel_robust/reflections.pickle'
]).raise_if_errors()
assert os.path.exists("gmodel_robust/reflections.pickle")
reflections1 = flex.reflection_table.from_pickle(
"simple/reflections.pickle")
reflections2 = flex.reflection_table.from_pickle(
"robust/reflections.pickle")
reflections3 = flex.reflection_table.from_pickle(
"gmodel_simple/reflections.pickle")
reflections4 = flex.reflection_table.from_pickle(
"gmodel_robust/reflections.pickle")
assert len(reflections1) == len(reflections3)
assert len(reflections2) == len(reflections4)
flag = flex.reflection_table.flags.integrated_sum
integrated1 = reflections1.select(reflections1.get_flags(flag, all=True))
integrated2 = reflections2.select(reflections2.get_flags(flag, all=True))
integrated3 = reflections3.select(reflections3.get_flags(flag, all=True))
integrated4 = reflections4.select(reflections4.get_flags(flag, all=True))
assert len(integrated1) > 0
assert len(integrated2) > 0
assert len(integrated1) == len(integrated3)
assert len(integrated2) == len(integrated4)
mean_bg1 = integrated1['background.mean']
mean_bg2 = integrated2['background.mean']
mean_bg3 = integrated3['background.mean']
mean_bg4 = integrated4['background.mean']
scale3 = integrated3['background.scale']
scale4 = integrated4['background.scale']
diff1 = flex.abs(mean_bg1 - mean_bg3)
diff2 = flex.abs(mean_bg2 - mean_bg4)
assert (scale3 > 0).count(False) == 0
assert (scale4 > 0).count(False) == 0
assert (diff1 < 1e-5).count(False) == 0