def tst_dump_formats(self):
from uuid import uuid4
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
# Read all the experiment lists in
elist1 = ExperimentListFactory.from_json_file(filename1)
# Create the experiment list dumper
dump = ExperimentListDumper(elist1)
# Dump as JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as split JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename, split=True)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as pickle and reload
filename = 'temp%s.pickle' % uuid4().hex
dump.as_pickle(filename)
elist2 = ExperimentListFactory.from_pickle_file(filename)
self.check(elist1, elist2)
def tst_dump_formats(self):
from uuid import uuid4
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
# Read all the experiment lists in
elist1 = ExperimentListFactory.from_json_file(filename1)
# Create the experiment list dumper
dump = ExperimentListDumper(elist1)
# Dump as JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as split JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename, split=True)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as pickle and reload
filename = 'temp%s.pickle' % uuid4().hex
dump.as_pickle(filename)
elist2 = ExperimentListFactory.from_pickle_file(filename)
self.check(elist1, elist2)
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
data_dir = os.path.join(dials_regression, "refinement_test_data",
"multi_stills")
experiments_path = os.path.join(data_dir, "combined_experiments.json")
reflections_path = os.path.join(data_dir, "combined_reflections.pickle")
cmd = "dials.refine " + experiments_path + " " + reflections_path
print cmd
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="tst_refine_multi_stills1")
os.chdir(tmp_dir)
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
reg_exp = ExperimentListFactory.from_json_file(os.path.join(
data_dir, "regression_experiments.json"),
check_format=False)
ref_exp = ExperimentListFactory.from_json_file(
"refined_experiments.json", check_format=False)
finally:
os.chdir(cwd)
print "OK"
# compare results
tol = 1e-5
for b1, b2 in zip(reg_exp.beams(), ref_exp.beams()):
assert b1.is_similar_to(b2,
wavelength_tolerance=tol,
direction_tolerance=tol,
polarization_normal_tolerance=tol,
polarization_fraction_tolerance=tol)
s0_1 = matrix.col(b1.get_unit_s0())
s0_2 = matrix.col(b2.get_unit_s0())
assert s0_1.accute_angle(s0_2, deg=True) < 0.0057 # ~0.1 mrad
for c1, c2 in zip(reg_exp.crystals(), ref_exp.crystals()):
assert c1.is_similar_to(c2)
for d1, d2 in zip(reg_exp.detectors(), ref_exp.detectors()):
assert d1.is_similar_to(d2,
fast_axis_tolerance=1e-4,
slow_axis_tolerance=1e-4,
origin_tolerance=1e-2)
print "OK"
return
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
data_dir = os.path.join(dials_regression, "refinement_test_data",
"multi_stills")
experiments_path = os.path.join(data_dir, "combined_experiments.json")
reflections_path = os.path.join(data_dir, "combined_reflections.pickle")
cmd = "dials.refine " + experiments_path + " " + reflections_path
print cmd
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="tst_refine_multi_stills1")
os.chdir(tmp_dir)
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
reg_exp = ExperimentListFactory.from_json_file(
os.path.join(data_dir, "regression_experiments.json"),
check_format=False)
ref_exp = ExperimentListFactory.from_json_file("refined_experiments.json",
check_format=False)
finally:
os.chdir(cwd)
# clean up tmp dir
shutil.rmtree(tmp_dir)
print "OK"
# compare results
tol = 1e-5
for b1, b2 in zip(reg_exp.beams(), ref_exp.beams()):
assert b1.is_similar_to(b2, wavelength_tolerance=tol,
direction_tolerance=tol,
polarization_normal_tolerance=tol,
polarization_fraction_tolerance=tol)
s0_1 = matrix.col(b1.get_unit_s0())
s0_2 = matrix.col(b2.get_unit_s0())
assert s0_1.accute_angle(s0_2, deg=True) < 0.0057 # ~0.1 mrad
for c1, c2 in zip(reg_exp.crystals(), ref_exp.crystals()):
assert c1.is_similar_to(c2)
for d1, d2 in zip(reg_exp.detectors(), ref_exp.detectors()):
assert d1.is_similar_to(d2,
fast_axis_tolerance=1e-4, slow_axis_tolerance=1e-4, origin_tolerance=1e-2)
print "OK"
return
class TestSummation(object):
def __init__(self):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from dials.algorithms.profile_model.gaussian_rs import Model
import libtbx.load_env
from dials.array_family import flex
from os.path import join
from math import pi
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
path = join(dials_regression, "centroid_test_data", "experiments.json")
exlist = ExperimentListFactory.from_json_file(path)
exlist[0].profile = Model(None,
n_sigma=3,
sigma_b=0.024 * pi / 180.0,
sigma_m=0.044 * pi / 180.0)
rlist = flex.reflection_table.from_predictions(exlist[0])
rlist['id'] = flex.int(len(rlist), 0)
self.rlist = rlist
self.exlist = exlist
def run(self):
from os.path import join
from libtbx import easy_run
from dials.algorithms.profile_model.factory import phil_scope
from libtbx.phil import parse
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
# Call dials.create_profile_model
easy_run.fully_buffered([
'dials.create_profile_model',
join(self.path, 'experiments.json'),
join(self.path, 'indexed.pickle'),
]).raise_if_errors()
experiments = ExperimentListFactory.from_json_file(
"experiments_with_profile_model.json", check_format=False)
sigma_b = experiments[0].profile.sigma_b(deg=True)
sigma_m = experiments[0].profile.sigma_m(deg=True)
eps = 1e-3
try:
assert (abs(sigma_b - 0.02195) < eps)
assert (abs(sigma_m - 0.06833) < eps)
except Exception:
print sigma_b
print sigma_m
raise
print 'OK'
def tst_dump_empty_sweep(self):
from dxtbx.imageset import ImageSweep, NullReader, SweepFileList
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
from uuid import uuid4
imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 3))))
imageset.set_beam(Beam((1, 0, 0)))
imageset.set_detector(Detector())
imageset.set_goniometer(Goniometer())
imageset.set_scan(Scan((1, 3), (0.0, 1.0)))
crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=1)
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments2 = ExperimentListFactory.from_json_file(filename,
check_format=False)
self.check(experiments, experiments2)
print 'OK'
def print_someting(experiments_argv):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_argv,
check_format=False)
print "len(experiments)", len(experiments)
print experiments[0]
for exp in experiments:
print "\n\n exp =", exp
print "dir(exp) =", dir(exp), "\n\n"
#print "dir(exp.crystal) =", dir(exp.crystal)
print "exp.crystal.get_space_group =", exp.crystal.get_space_group()
print "exp.crystal.get_unit_cell =", exp.crystal.get_unit_cell()
#detc = exp.detector
#scan = exp.scan
#prof = exp.profile
gonio = exp.goniometer
#print "\n\n dir(detc) =", dir(detc)
#print "\n\n dir(scan) =", dir(scan)
#print "\n\n dir(prof) =", dir(prof)
print "\n\n dir(goni) =", dir(gonio)
print "Pass"
class Test(object):
def __init__(self):
from math import pi
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
import os
filename = os.path.join(dials_regression, 'centroid_test_data',
'fake_long_experiments.json')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from dxtbx.model.experiment.experiment_list import ExperimentList
exlist = ExperimentListFactory.from_json_file(filename)
assert (len(exlist) == 1)
self.experiment = exlist[0]
# Set the delta_divergence/mosaicity
self.n_sigma = 5
self.sigma_b = 0.060 * pi / 180
self.sigma_m = 0.154 * pi / 180
from dials.algorithms.profile_model.gaussian_rs import Model
self.profile_model = Model(None, self.n_sigma, self.sigma_b,
self.sigma_m)
self.experiment.profile = self.profile_model
self.experiments = ExperimentList()
self.experiments.append(self.experiment)
def tst_dump_empty_sweep(self):
from dxtbx.imageset import ImageSweep, NullReader, SweepFileList
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
from uuid import uuid4
imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 3))))
imageset.set_beam(Beam((1, 0, 0)))
imageset.set_detector(Detector())
imageset.set_goniometer(Goniometer())
imageset.set_scan(Scan((1, 3), (0.0, 1.0)))
crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=1)
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments2 = ExperimentListFactory.from_json_file(filename,
check_format=False)
self.check(experiments, experiments2)
print 'OK'
def run(self):
from os.path import join
from libtbx import easy_run
from dials.algorithms.profile_model.factory import phil_scope
from libtbx.phil import parse
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
# Call dials.create_profile_model
easy_run.fully_buffered([
'dials.create_profile_model',
join(self.path, 'experiments.json'),
join(self.path, 'indexed.pickle'),
]).raise_if_errors()
experiments = ExperimentListFactory.from_json_file(
"experiments_with_profile_model.json",
check_format=False)
sigma_b = experiments[0].profile.sigma_b(deg=True)
sigma_m = experiments[0].profile.sigma_m(deg=True)
eps = 1e-3
try:
assert(abs(sigma_b - 0.02195) < eps)
assert(abs(sigma_m - 0.06833) < eps)
except Exception:
print sigma_b
print sigma_m
raise
print 'OK'
class Test(object):
def __init__(self):
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'SKIP: dials_regression not configured'
exit(0)
import os
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
path = os.path.join(dials_regression, 'centroid_test_data',
'experiments.json')
self.experiments = ExperimentListFactory.from_json_file(path)
assert (len(self.experiments) == 1)
self.experiments[0].imageset.set_beam(self.experiments[0].beam)
self.experiments[0].imageset.set_detector(self.experiments[0].detector)
self.experiments[0].imageset.set_goniometer(
self.experiments[0].goniometer)
self.experiments[0].imageset.set_scan(self.experiments[0].scan)
reflection_filename = os.path.join(dials_regression,
'prediction_test_data',
'expected_reflections.pickle')
from dials.array_family import flex
self.reflections = flex.reflection_table.from_pickle(
reflection_filename)
def test2():
"""Compare results of multiprocess vs single process refinement to ensure
they are the same"""
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
data_dir = os.path.join(dials_regression, "refinement_test_data",
"multi_stills")
experiments_path = os.path.join(data_dir, "combined_experiments.json")
reflections_path = os.path.join(data_dir, "combined_reflections.pickle")
cmd = "dials.refine " + experiments_path + " " + reflections_path + \
" outlier.algorithm=null engine=LBFGScurvs output.reflections=None "
cmd1 = cmd + "output.experiments=refined_experiments_nproc1.json nproc=1"
print cmd1
cmd2= cmd + "output.experiments=refined_experiments_nproc4.json nproc=4"
print cmd2
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="tst_refine_multi_stills2")
os.chdir(tmp_dir)
try:
result1 = easy_run.fully_buffered(command=cmd1).raise_if_errors()
result2 = easy_run.fully_buffered(command=cmd2).raise_if_errors()
# load results
nproc1 = ExperimentListFactory.from_json_file(
"refined_experiments_nproc1.json", check_format=False)
nproc4 = ExperimentListFactory.from_json_file(
"refined_experiments_nproc4.json", check_format=False)
finally:
os.chdir(cwd)
# clean up tmp dir
shutil.rmtree(tmp_dir)
print "OK"
# compare results
for b1, b2 in zip(nproc1.beams(), nproc4.beams()):
assert b1.is_similar_to(b2)
for c1, c2 in zip(nproc1.crystals(), nproc4.crystals()):
assert c1.is_similar_to(c2)
for d1, d2 in zip(nproc1.detectors(), nproc4.detectors()):
assert d1.is_similar_to(d2,
fast_axis_tolerance=5e-5, slow_axis_tolerance=5e-5, origin_tolerance=5e-5)
print "OK"
return
def test2():
"""Compare results of multiprocess vs single process refinement to ensure
they are the same"""
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
data_dir = os.path.join(dials_regression, "refinement_test_data",
"multi_stills")
experiments_path = os.path.join(data_dir, "combined_experiments.json")
reflections_path = os.path.join(data_dir, "combined_reflections.pickle")
cmd = "dials.refine " + experiments_path + " " + reflections_path + \
" outlier.algorithm=null engine=LBFGScurvs output.reflections=None "
cmd1 = cmd + "output.experiments=refined_experiments_nproc1.json nproc=1"
print cmd1
cmd2 = cmd + "output.experiments=refined_experiments_nproc4.json nproc=4"
print cmd2
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="tst_refine_multi_stills2")
os.chdir(tmp_dir)
try:
result1 = easy_run.fully_buffered(command=cmd1).raise_if_errors()
result2 = easy_run.fully_buffered(command=cmd2).raise_if_errors()
# load results
nproc1 = ExperimentListFactory.from_json_file(
"refined_experiments_nproc1.json", check_format=False)
nproc4 = ExperimentListFactory.from_json_file(
"refined_experiments_nproc4.json", check_format=False)
finally:
os.chdir(cwd)
print "OK"
# compare results
for b1, b2 in zip(nproc1.beams(), nproc4.beams()):
assert b1.is_similar_to(b2)
for c1, c2 in zip(nproc1.crystals(), nproc4.crystals()):
assert c1.is_similar_to(c2)
for d1, d2 in zip(nproc1.detectors(), nproc4.detectors()):
assert d1.is_similar_to(d2,
fast_axis_tolerance=5e-5,
slow_axis_tolerance=5e-5,
origin_tolerance=5e-5)
print "OK"
return
def tst_from_json(self):
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
filename2 = join(self.path, 'experiment_test_data', 'experiment_2.json')
filename3 = join(self.path, 'experiment_test_data', 'experiment_3.json')
filename4 = join(self.path, 'experiment_test_data', 'experiment_4.json')
# Read all the experiment lists in
el1 = ExperimentListFactory.from_json_file(filename1)
el2 = ExperimentListFactory.from_json_file(filename2)
el3 = ExperimentListFactory.from_json_file(filename3)
el4 = ExperimentListFactory.from_json_file(filename4)
# All the experiment lists should be the same length
assert(len(el1) == 1)
assert(len(el1) == len(el2))
assert(len(el1) == len(el3))
assert(len(el1) == len(el4))
# Check all the models are the same
for e in zip(el1, el2, el3, el4):
e1 = e[0]
assert(e1.imageset is not None)
assert(e1.beam is not None)
assert(e1.detector is not None)
assert(e1.goniometer is not None)
assert(e1.scan is not None)
assert(e1.crystal is not None)
for ee in e[1:]:
assert(e1.imageset == ee.imageset)
assert(e1.beam == ee.beam)
assert(e1.detector == ee.detector)
assert(e1.goniometer == ee.goniometer)
assert(e1.scan == ee.scan)
assert(e1.crystal == ee.crystal)
# test passed
print 'OK'
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
# use the i04_weak_data for this test
data_dir = os.path.join(dials_regression, "refinement_test_data", "i04_weak_data")
experiments_path = os.path.join(data_dir, "experiments.json")
pickle_path = os.path.join(data_dir, "indexed_strong.pickle")
for pth in (experiments_path, pickle_path):
assert os.path.exists(pth)
# set close_to_spindle_cutoff to old default
cmd = "dials.refine close_to_spindle_cutoff=0.05 reflections_per_degree=100 " + \
experiments_path + " " + pickle_path
print cmd
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="test_dials_refine")
os.chdir(tmp_dir)
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
reg_exp = ExperimentListFactory.from_json_file(
os.path.join(data_dir, "regression_experiments.json"),
check_format=False)[0]
ref_exp = ExperimentListFactory.from_json_file("refined_experiments.json",
check_format=False)[0]
finally:
os.chdir(cwd)
# clean up tmp dir
shutil.rmtree(tmp_dir)
# test refined models against expected
assert reg_exp.crystal == ref_exp.crystal
assert reg_exp.detector == ref_exp.detector
assert reg_exp.beam == ref_exp.beam
print "OK"
return
def tst_from_json(self):
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
filename2 = join(self.path, 'experiment_test_data', 'experiment_2.json')
filename3 = join(self.path, 'experiment_test_data', 'experiment_3.json')
filename4 = join(self.path, 'experiment_test_data', 'experiment_4.json')
# Read all the experiment lists in
el1 = ExperimentListFactory.from_json_file(filename1)
#el2 = ExperimentListFactory.from_json_file(filename2)
el3 = ExperimentListFactory.from_json_file(filename3)
el4 = ExperimentListFactory.from_json_file(filename4)
# All the experiment lists should be the same length
assert(len(el1) == 1)
#assert(len(el1) == len(el2))
assert(len(el1) == len(el3))
assert(len(el1) == len(el4))
# Check all the models are the same
for e in zip(el1, el3, el4):
e1 = e[0]
assert(e1.imageset is not None)
assert(e1.beam is not None)
assert(e1.detector is not None)
assert(e1.goniometer is not None)
assert(e1.scan is not None)
assert(e1.crystal is not None)
for ee in e[1:]:
assert(e1.imageset == ee.imageset)
assert(e1.beam == ee.beam)
assert(e1.detector == ee.detector)
assert(e1.goniometer == ee.goniometer)
assert(e1.scan == ee.scan)
assert(e1.crystal == ee.crystal)
# test passed
print 'OK'
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
# use the i04_weak_data for this test
data_dir = os.path.join(dials_regression, "refinement_test_data",
"i04_weak_data")
experiments_path = os.path.join(data_dir, "experiments.json")
pickle_path = os.path.join(data_dir, "indexed_strong.pickle")
for pth in (experiments_path, pickle_path):
assert os.path.exists(pth)
# set some old defaults
cmd = "dials.refine close_to_spindle_cutoff=0.05 reflections_per_degree=100 " + \
"outlier.separate_blocks=False " + experiments_path + " " + pickle_path
print cmd
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="test_dials_refine")
os.chdir(tmp_dir)
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
reg_exp = ExperimentListFactory.from_json_file(os.path.join(
data_dir, "regression_experiments.json"),
check_format=False)[0]
ref_exp = ExperimentListFactory.from_json_file(
"refined_experiments.json", check_format=False)[0]
finally:
os.chdir(cwd)
# test refined models against expected
assert reg_exp.crystal == ref_exp.crystal
assert reg_exp.detector == ref_exp.detector
assert reg_exp.beam == ref_exp.beam
print "OK"
return
def _refine(self):
"""Do refinement and load the results"""
# turn off outlier rejection so that test takes about 4s rather than 10s
# set close_to_spindle_cutoff to old default
cmd = ("dials.refine combined_experiments.json combined_reflections.pickle"
" outlier.algorithm=null close_to_spindle_cutoff=0.05")
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
self._refined_experiments = ExperimentListFactory.from_json_file(
"refined_experiments.json", check_format=False)
return
def _refine(self):
"""Do refinement and load the results"""
# turn off outlier rejection so that test takes about 4s rather than 10s
# set close_to_spindle_cutoff to old default
cmd = (
"dials.refine combined_experiments.json combined_reflections.pickle"
" outlier.algorithm=null close_to_spindle_cutoff=0.05")
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
self._refined_experiments = ExperimentListFactory.from_json_file(
"refined_experiments.json", check_format=False)
return
def from_string(self, s):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from os.path import exists
from libtbx.utils import Sorry
if s is None:
return None
if s not in self.cache:
if not exists(s):
raise Sorry('File %s does not exist' % s)
self.cache[s] = FilenameDataWrapper(s,
ExperimentListFactory.from_json_file(s,
check_format=self._check_format))
return self.cache[s]
def select_importer(self, args):
from os.path import split
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import libtbx.load_env
path, filename = split(args[0])
if filename == 'SPOT.XDS':
return SpotXDSImporter(args[0])
elif filename == 'INTEGRATE.HKL':
assert(len(args) == 2)
experiments = ExperimentListFactory.from_json_file(args[1])
assert(len(experiments) == 1)
return IntegrateHKLImporter(args[0], experiments[0])
else:
raise RuntimeError('expected (SPOT.XDS|INTEGRATE.HKL), got %s' % filename)
class Test(object):
def __init__(self):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import libtbx.load_env
from os.path import join
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
path = join(dials_regression, "centroid_test_data", "experiments.json")
self.experiments = ExperimentListFactory.from_json_file(path)
def from_string(self, s):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from os.path import exists
from libtbx.utils import Sorry
if s is None:
return None
if s not in self.cache:
if not exists(s):
raise Sorry('File %s does not exist' % s)
self.cache[s] = FilenameDataWrapper(
s,
ExperimentListFactory.from_json_file(
s, check_format=self._check_format))
return self.cache[s]
def regression(self):
"""Check results are as expected"""
regression_experiments = ExperimentListFactory.from_json_file(
os.path.join(self._data_dir, "regression_experiments.json"),
check_format=False)
for e1, e2 in zip(self._refined_experiments, regression_experiments):
assert e1.crystal.is_similar_to(e2.crystal)
# FIXME need is_similar_to for detector that checks geometry
#assert e1.detector == e2.detector
s0_1 = matrix.col(e1.beam.get_unit_s0())
s0_2 = matrix.col(e1.beam.get_unit_s0())
assert s0_1.accute_angle(s0_2, deg=True) < 0.0057 # ~0.1 mrad
print "OK"
return
def regression(self):
"""Check results are as expected"""
regression_experiments = ExperimentListFactory.from_json_file(
os.path.join(self._data_dir, "regression_experiments.json"),
check_format=False)
for e1, e2 in zip(self._refined_experiments, regression_experiments):
assert e1.crystal.is_similar_to(e2.crystal)
# FIXME need is_similar_to for detector that checks geometry
#assert e1.detector == e2.detector
s0_1 = matrix.col(e1.beam.get_unit_s0())
s0_2 = matrix.col(e1.beam.get_unit_s0())
assert s0_1.accute_angle(s0_2, deg=True) < 0.0057 # ~0.1 mrad
print "OK"
return
def _create_profile_model(self):
info("\nCreating profile model...")
command = [ "dials.create_profile_model", "experiments.json", "indexed.pickle" ]
result = run_process(command, print_stdout=False, debug=procrunner_debug)
debug("result = %s" % self._prettyprint_dictionary(result))
if result['exitcode'] == 0:
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
db = ExperimentListFactory.from_json_file('experiments_with_profile_model.json')[0]
self._num_images = db.imageset.get_scan().get_num_images()
self._oscillation = db.imageset.get_scan().get_oscillation()[1]
self._sigma_m = db.profile.sigma_m()
info("%d images, %s deg. oscillation, sigma_m=%.3f" % (self._num_images, str(self._oscillation), self._sigma_m))
info("Successfully completed (%.1f sec)" % result['runtime'])
return True
else:
warn("Failed with exit code %d" % result['exitcode'])
return False
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
# use multiple scan small molecule data for this test
data_dir = os.path.join(dials_regression, "xia2-28")
prefix = ["20", "25", "30", "35"]
exp_path = [e + "_integrated_experiments.json" for e in prefix]
exp_path = [os.path.join(data_dir, e) for e in exp_path]
pkl_path = [e + "_integrated.pickle" for e in prefix]
pkl_path = [os.path.join(data_dir, e) for e in pkl_path]
for pth in exp_path + pkl_path:
assert os.path.exists(pth), "%s missing" % pth
cmd = "dials.two_theta_refine " + " ".join(exp_path) + " " + " ".join(
pkl_path) + " cif=refined_cell.cif"
print cmd
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="test_dials_two_theta_refine")
os.chdir(tmp_dir)
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
ref_exp = ExperimentListFactory.from_json_file("refined_cell.json",
check_format=False)
finally:
os.chdir(cwd)
xls = ref_exp.crystals()
assert len(xls) == 1 # crystal models should have been combined
xl = xls[0]
# test refined crystal model against expected values
assert approx_equal(
xl.get_unit_cell().parameters(),
(5.428022880, 8.144145476, 12.039666971, 90.0, 90.0, 90.0))
assert approx_equal(xl.get_cell_parameter_sd(),
(9.58081e-5, 0.000149909, 0.000215765, 0, 0, 0))
assert approx_equal(xl.get_cell_volume_sd(), 0.0116254298)
print "OK"
return
def load_reference_geometry(self):
if self.params.input.reference_geometry is None: return
try:
ref_datablocks = DataBlockFactory.from_json_file(self.params.input.reference_geometry, check_format=False)
except Exception:
ref_datablocks = None
if ref_datablocks is None:
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
try:
ref_experiments = ExperimentListFactory.from_json_file(self.params.input.reference_geometry, check_format=False)
except Exception:
raise Sorry("Couldn't load geometry file %s"%self.params.input.reference_geometry)
assert len(ref_experiments.detectors()) == 1
self.reference_detector = ref_experiments.detectors()[0]
else:
assert len(ref_datablocks) == 1 and len(ref_datablocks[0].unique_detectors()) == 1
self.reference_detector = ref_datablocks[0].unique_detectors()[0]
def print_someting(experiments_argv):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_argv,
check_format=False)
print "len(experiments)", len(experiments)
print experiments[0]
for exp in experiments:
print "\n exp =", exp
#print "dir(exp) =", dir(exp), "\n"
#print "dir(exp.crystal) =", dir(exp.crystal)
print "exp.crystal.get_space_group =", exp.crystal.get_space_group()
print "exp.crystal.get_unit_cell =", exp.crystal.get_unit_cell()
#detc = exp.detector
#scan = exp.scan
#prof = exp.profile
gonio = exp.goniometer
#print "\n\n dir(detc) =", dir(detc)
#print "\n\n dir(scan) =", dir(scan)
#print "\n\n dir(prof) =", dir(prof)
print "\n dir(gonio) =", dir(gonio), "\n"
'''
print "gonio.get_fixed_rotation() =", gonio.get_fixed_rotation()
print "gonio.get_rotation_axis() =", gonio.get_rotation_axis()
print "gonio.get_rotation_axis_datum() =", gonio.get_rotation_axis_datum()
print "gonio.get_setting_rotation() =", gonio.get_setting_rotation()
'''
#get_U().elems
print "\nexp.crystal.get_U() =\n", exp.crystal.get_U().elems
print "\nexp.crystal.get_A() =\n", exp.crystal.get_A().elems
print "\nexp.crystal.get_B() =\n", exp.crystal.get_B().elems
print "Pass 01"
'''
def update_crystal(experiments_path):
dat = CrystalData()
try:
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_path,
check_format=False)
print "len(experiments)", len(experiments)
exp = experiments[0]
unit_cell = exp.crystal.get_unit_cell()
dat.a, dat.b, dat.c, dat.alpha, dat.beta, dat.gamma = unit_cell.parameters(
)
except:
print "Unable to find cell data"
return dat
class Test(object):
def __init__(self):
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
import dials
import os
filename = os.path.join(dials_regression, 'centroid_test_data',
'experiments.json')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
self.exlist = ExperimentListFactory.from_json_file(filename)
assert (len(self.exlist) == 1)
from dials.array_family import flex
self.rlist = flex.reflection_table.from_predictions_multi(self.exlist)
class Test(object):
def __init__(self):
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'SKIP: dials_regression not configured'
exit(0)
import os
path = os.path.join(
dials_regression,
'prediction_test_data',
'experiments_scan_varying_crystal.json')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
self.experiments = ExperimentListFactory.from_json_file(path)
assert(len(self.experiments) == 1)
assert(self.experiments[0].crystal.num_scan_points ==
self.experiments[0].scan.get_num_images() + 1)
def test1():
dials_regression = libtbx.env.find_in_repositories(relative_path="dials_regression", test=os.path.isdir)
# use multiple scan small molecule data for this test
data_dir = os.path.join(dials_regression, "xia2-28")
prefix = ["20", "25", "30", "35"]
exp_path = [e + "_integrated_experiments.json" for e in prefix]
exp_path = [os.path.join(data_dir, e) for e in exp_path]
pkl_path = [e + "_integrated.pickle" for e in prefix]
pkl_path = [os.path.join(data_dir, e) for e in pkl_path]
for pth in exp_path + pkl_path:
assert os.path.exists(pth), "%s missing" % pth
cmd = "dials.two_theta_refine " + " ".join(exp_path) + " " + " ".join(pkl_path) + " cif=refined_cell.cif"
print cmd
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="test_dials_two_theta_refine")
os.chdir(tmp_dir)
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
ref_exp = ExperimentListFactory.from_json_file("refined_cell.json", check_format=False)
finally:
os.chdir(cwd)
xls = ref_exp.crystals()
assert len(xls) == 1 # crystal models should have been combined
xl = xls[0]
# test refined crystal model against expected values
assert approx_equal(xl.get_unit_cell().parameters(), (5.428022880, 8.144145476, 12.039666971, 90.0, 90.0, 90.0))
assert approx_equal(xl.get_cell_parameter_sd(), (9.58081e-5, 0.000149909, 0.000215765, 0, 0, 0))
assert approx_equal(xl.get_cell_volume_sd(), 0.0116254298)
print "OK"
return
def load_reference_geometry(self):
if self.params.input.reference_geometry is None: return
try:
ref_datablocks = DataBlockFactory.from_json_file(
self.params.input.reference_geometry, check_format=False)
except Exception:
ref_datablocks = None
if ref_datablocks is None:
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
try:
ref_experiments = ExperimentListFactory.from_json_file(
self.params.input.reference_geometry, check_format=False)
except Exception:
raise Sorry("Couldn't load geometry file %s" %
self.params.input.reference_geometry)
assert len(ref_experiments.detectors()) == 1
self.reference_detector = ref_experiments.detectors()[0]
else:
assert len(ref_datablocks) == 1 and len(
ref_datablocks[0].unique_detectors()) == 1
self.reference_detector = ref_datablocks[0].unique_detectors()[0]
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
# use the i04_weak_data for this test
data_dir = os.path.join(dials_regression, "refinement_test_data", "i04_weak_data")
experiments_path = os.path.join(data_dir, "experiments.json")
pickle_path = os.path.join(data_dir, "indexed_strong.pickle")
for pth in (experiments_path, pickle_path):
assert os.path.exists(pth)
cmd = "dials.slice_sweep " + experiments_path + " " + pickle_path + \
' "scan_range=1 20"'
print cmd
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="test_dials_slice_sweep")
os.chdir(tmp_dir)
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
sliced_exp = ExperimentListFactory.from_json_file("experiments_1_20.json",
check_format=False)[0]
with open("indexed_strong_1_20.pickle", "r") as f:
sliced_refs = pickle.load(f)
finally:
os.chdir(cwd)
# simple test of results
assert sliced_exp.scan.get_image_range() == (1, 20)
assert len(sliced_refs) == 3670
print "OK"
return
def test1():
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression", test=os.path.isdir)
# use the i04_weak_data for this test
data_dir = os.path.join(dials_regression, "refinement_test_data",
"i04_weak_data")
experiments_path = os.path.join(data_dir, "experiments.json")
pickle_path = os.path.join(data_dir, "indexed_strong.pickle")
for pth in (experiments_path, pickle_path):
assert os.path.exists(pth)
cmd = "dials.slice_sweep " + experiments_path + " " + pickle_path + \
' "scan_range=1 20"'
print cmd
# work in a temporary directory
cwd = os.path.abspath(os.curdir)
tmp_dir = open_tmp_directory(suffix="test_dials_slice_sweep")
os.chdir(tmp_dir)
try:
result = easy_run.fully_buffered(command=cmd).raise_if_errors()
# load results
sliced_exp = ExperimentListFactory.from_json_file(
"experiments_1_20.json", check_format=False)[0]
with open("indexed_strong_1_20.pickle", "r") as f:
sliced_refs = pickle.load(f)
finally:
os.chdir(cwd)
# simple test of results
assert sliced_exp.scan.get_image_range() == (1, 20)
assert len(sliced_refs) == 3670
print "OK"
return
def tst_from_pickle(self):
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
# Read all the experiment lists in
el1 = ExperimentListFactory.from_json_file(filename1)
# Pickle then load again
el2 = self.pickle_then_unpickle(el1)
# All the experiment lists should be the same length
assert(len(el1) == 1)
assert(len(el1) == len(el2))
# Check all the models are the same
for e1, e2 in zip(el1, el2):
assert(e1.imageset is not None)
assert(e1.beam is not None)
assert(e1.detector is not None)
assert(e1.goniometer is not None)
assert(e1.scan is not None)
assert(e1.crystal is not None)
assert(e1.imageset == e2.imageset)
assert(e1.beam == e2.beam)
assert(e1.detector == e2.detector)
assert(e1.goniometer == e2.goniometer)
assert(e1.scan == e2.scan)
assert(e1.crystal == e2.crystal)
# test passed
print 'OK'
def tst_from_pickle(self):
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
# Read all the experiment lists in
el1 = ExperimentListFactory.from_json_file(filename1)
# Pickle then load again
el2 = self.pickle_then_unpickle(el1)
# All the experiment lists should be the same length
assert(len(el1) == 1)
assert(len(el1) == len(el2))
# Check all the models are the same
for e1, e2 in zip(el1, el2):
assert(e1.imageset is not None)
assert(e1.beam is not None)
assert(e1.detector is not None)
assert(e1.goniometer is not None)
assert(e1.scan is not None)
assert(e1.crystal is not None)
assert(e1.imageset == e2.imageset)
assert(e1.beam == e2.beam)
assert(e1.detector == e2.detector)
assert(e1.goniometer == e2.goniometer)
assert(e1.scan == e2.scan)
assert(e1.crystal == e2.crystal)
# test passed
print 'OK'
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
path = join(dials_regression, "centroid_test_data")
import sys
assert(len(sys.argv) == 1)
sys.argv.append(join(path, "experiments.json"))
sys.argv.append(join(path, "profile.phil"))
parser = OptionParser(phil=phil_scope)
params, options, args = parser.parse_args()
assert(len(args) == 1)
exlist = ExperimentListFactory.from_json_file(args[0])
assert(len(exlist) == 1)
profile_model = ProfileModelList.load(params)
rlist = flex.reflection_table.from_predictions_multi(exlist)
rlist.compute_bbox(exlist, profile_model)
rlist['shoebox'] = flex.shoebox(rlist['panel'], rlist['bbox'])
rlist['shoebox'].allocate()
rlist.extract_shoeboxes(exlist[0].imageset)
show_reflection(rlist[len(rlist)//2])
#show_reflection(rlist[len(rlist)//2], orient = "porTrait")
#show_reflection(rlist[len(rlist)//2], orient = "lanDscape")
self.n_sigma * self.sigma_m,
)
# Mask the foreground
mask_foreground(refl["shoebox"], refl["s1"], refl["xyzcal.px"].parts()[2], refl["panel"])
Command.end("Masked foreground for %d reflections" % len(refl))
# Return the reflections
return refl
if __name__ == "__main__":
from math import pi
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(
"/home/upc86896/Projects/cctbx/sources/dials_regression/centroid_test_data/experiments.json", check_format=False
)
sigma_b = 0.058 * pi / 180
sigma_m = 0.157 * pi / 180
n_sigma = 3
N = 100
I = 1000
B = 10
simulate = Simulator(experiments[0], sigma_b, sigma_m, n_sigma)
simulate.with_random_intensity(N, I, B)
# simulate(experiments[0], sigma_b, sigma_m, n_sigma, N, I, B)
from uuid import uuid4
import libtbx.load_env
import os
from os.path import join
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
filename = join(dials_regression, "centroid_test_data",
"experiments_with_bad_lookup.json")
experiments = ExperimentListFactory.from_json_file(
filename,
check_format=False)
imageset = experiments[0].imageset
assert imageset.external_lookup.mask.data is None
assert imageset.external_lookup.gain.data is None
assert imageset.external_lookup.pedestal.data is None
assert imageset.external_lookup.mask.filename is not None
assert imageset.external_lookup.gain.filename is not None
assert imageset.external_lookup.pedestal.filename is not None
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments = ExperimentListFactory.from_json_file(
def get_dials_matrix(experiments_json):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_json)
return experiments[0].crystal.get_A()
def get_dials_coordinate_frame(experiments_json):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_json)
return experiments[0].beam.get_direction(), \
experiments[0].goniometer.get_rotation_axis()
def get_dials_coordinate_frame(experiments_json):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_json)
return experiments[0].beam.get_direction(), experiments[0].goniometer.get_rotation_axis()
class Test(object):
def __init__(self):
from os.path import join, isfile
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'SKIP: dials_regression not configured'
exit(0)
# The base path
path = join(dials_regression, 'integration_test_data', 'simulated')
# Set the experiment filename
expr_filename = join(path, 'experiments.json')
# The reference spots filesname
reference_filename = join(path, 'simulated_n10000_i10000_b0.pickle')
# The paths to the reflection files
self.refl_filenames = [
#join(path, 'simulated_n10000_i0_b10.pickle'),
#join(path, 'simulated_n10000_i0_b100.pickle'),
#join(path, 'simulated_n10000_i0_b1000.pickle'),
#join(path, 'simulated_r_n10000_i0_b1000.pickle'),
#join(path, 'simulated_n10000_i10_b0.pickle'),
#join(path, 'simulated_n10000_i100_b0.pickle'),
#join(path, 'simulated_n10000_i1000_b0.pickle'),
#join(path, 'simulated_r_n10000_i10000_b0.pickle'),
join(path, 'simulated_n10000_i10_b10.pickle'),
join(path, 'simulated_n10000_i100_b10.pickle'),
join(path, 'simulated_n10000_i1000_b10.pickle'),
join(path, 'simulated_n10000_i10000_b10.pickle'),
join(path, 'simulated_r_n10000_i10000_b10.pickle'),
join(path, 'simulated_n10000_i10_b100.pickle'),
join(path, 'simulated_n10000_i100_b100.pickle'),
join(path, 'simulated_n10000_i1000_b100.pickle'),
join(path, 'simulated_n10000_i10000_b100.pickle'),
join(path, 'simulated_r_n10000_i10000_b100.pickle'),
]
# Check the files exist
for filename in self.refl_filenames:
if not isfile(filename):
print 'SKIP: simulated test data does not exist'
print 'Generate by running the following commands:'
print ' cd dials_regression/integration_test_data/simulated'
print ' ./simulate'
exit(0)
# Load the experiments
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(expr_filename,
check_format=False)
assert (len(experiments) == 1)
self.experiments = experiments
from dials.algorithms.profile_model.gaussian_rs import Model as ProfileModel
from dials.algorithms.profile_model.factory import phil_scope
from libtbx.phil import parse
params = phil_scope.fetch(parse('')).extract()
self.experiments[0].profile = ProfileModel(params.profile,
sigma_b=0.024,
sigma_m=0.044,
n_sigma=3,
deg=True)
# Load the reference spots
from dials.array_family import flex
self.reference = flex.reflection_table.from_pickle(reference_filename)
self.reference.compute_partiality(self.experiments)
mask = flex.bool(len(self.reference), True)
value = self.reference.flags.reference_spot
self.reference.set_flags(mask, value)
self.reference.set_flags(mask, self.reference.flags.integrated_sum)
class TestExperimentListDumper(object):
def __init__(self, path):
self.path = path
def run(self):
self.tst_dump_formats()
self.tst_dump_empty_sweep()
self.tst_dump_with_lookup()
self.tst_dump_with_bad_lookup()
def tst_dump_formats(self):
from uuid import uuid4
from os.path import join
import os
os.environ['DIALS_REGRESSION'] = self.path
# Get all the filenames
filename1 = join(self.path, 'experiment_test_data', 'experiment_1.json')
# Read all the experiment lists in
elist1 = ExperimentListFactory.from_json_file(filename1)
# Create the experiment list dumper
dump = ExperimentListDumper(elist1)
# Dump as JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as split JSON file and reload
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename, split=True)
elist2 = ExperimentListFactory.from_json_file(filename)
self.check(elist1, elist2)
# Dump as pickle and reload
filename = 'temp%s.pickle' % uuid4().hex
dump.as_pickle(filename)
elist2 = ExperimentListFactory.from_pickle_file(filename)
self.check(elist1, elist2)
def tst_dump_empty_sweep(self):
from dxtbx.imageset import ImageSweep, NullReader, SweepFileList
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
from uuid import uuid4
imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 3))))
imageset.set_beam(Beam((1, 0, 0)))
imageset.set_detector(Detector())
imageset.set_goniometer(Goniometer())
imageset.set_scan(Scan((1, 3), (0.0, 1.0)))
crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=1)
experiments = ExperimentListFactory.from_imageset_and_crystal(
imageset, crystal)
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments2 = ExperimentListFactory.from_json_file(filename,
check_format=False)
self.check(experiments, experiments2)
print 'OK'
def tst_dump_with_lookup(self):
from dxtbx.imageset import ImageSweep, NullReader, SweepFileList
from dxtbx.model import Beam, Detector, Goniometer, Scan
from dxtbx.model.crystal import crystal_model
from uuid import uuid4
import libtbx.load_env
import os
from os.path import join
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
filename = join(dials_regression, "centroid_test_data",
"experiments_with_lookup.json")
experiments = ExperimentListFactory.from_json_file(
filename,
check_format=True)
imageset = experiments[0].imageset
assert imageset.external_lookup.mask.data is not None
assert imageset.external_lookup.gain.data is not None
assert imageset.external_lookup.pedestal.data is not None
assert imageset.external_lookup.mask.filename is not None
assert imageset.external_lookup.gain.filename is not None
assert imageset.external_lookup.pedestal.filename is not None
assert imageset.external_lookup.mask.data.all_eq(True)
assert imageset.external_lookup.gain.data.all_eq(1)
assert imageset.external_lookup.pedestal.data.all_eq(0)
dump = ExperimentListDumper(experiments)
filename = 'temp%s.json' % uuid4().hex
dump.as_json(filename)
experiments = ExperimentListFactory.from_json_file(
filename,
check_format=True)
imageset = experiments[0].imageset
assert imageset.external_lookup.mask.data is not None
assert imageset.external_lookup.gain.data is not None
assert imageset.external_lookup.pedestal.data is not None
assert imageset.external_lookup.mask.filename is not None
assert imageset.external_lookup.gain.filename is not None
assert imageset.external_lookup.pedestal.filename is not None
assert imageset.external_lookup.mask.data.all_eq(True)
assert imageset.external_lookup.gain.data.all_eq(1)
assert imageset.external_lookup.pedestal.data.all_eq(0)
def run():
from dials.util.nexus import dump, load
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from dials.array_family import flex
from os.path import join
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
path = join(dials_regression, "nexus_test_data")
# Read the experiments
experiments1 = ExperimentListFactory.from_json_file(
join(path, "refined_experiments.json"))
# Read the reflections
reflections1 = flex.reflection_table.from_pickle(
join(path, "integrated.pickle"))
# Delete some columns for the test
del reflections1['s1']
del reflections1['zeta']
del reflections1['background.mse']
# Dump the reflections
dump(experiments1, reflections1, "hklout.nxs")
# Load them again
experiments2, reflections2 = load("hklout.nxs")
def load_sweeps_with_common_indexing():
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
import dials # required for gaussian_rs warning
from xia2.Wrappers.Dials.Reindex import Reindex
Citations.cite('dials')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import cPickle as pickle
crystals = xinfo.get_crystals()
assert len(crystals) == 1
crystal = next(crystals.itervalues())
working_directory = Environment.generate_directory([crystal.get_name(), 'analysis'])
os.chdir(working_directory)
scaler = crystal._get_scaler()
epoch_to_batches = {}
epoch_to_integrated_intensities = {}
epoch_to_sweep_name = {}
# Aimless only
epochs = scaler._sweep_handler.get_epochs()
reference_cell = None
reference_lattice = None
reference_vectors = None
reference_wavelength = None
# Reindex each sweep to same setting
all_miller_indices = flex.miller_index()
all_two_thetas = flex.double()
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
Chatter.smallbanner(si.get_sweep_name(), True)
Debug.smallbanner(si.get_sweep_name(), True)
intgr = si.get_integrater()
experiments_filename = intgr.get_integrated_experiments()
reflections_filename = intgr.get_integrated_reflections()
refiner = intgr.get_integrater_refiner()
Debug.write('experiment: %s' % experiments_filename)
Debug.write('reflection: %s' % reflections_filename)
# Use setting of first sweep as reference
if reference_vectors is None:
reference_vectors = experiments_filename
# Assume that all sweeps have the same lattice system
if reference_lattice is None:
reference_lattice = refiner.get_refiner_lattice()
else:
assert reference_lattice == refiner.get_refiner_lattice()
Debug.write("lattice: %s" % refiner.get_refiner_lattice())
# Read .json file for sweep
db = ExperimentListFactory.from_json_file(experiments_filename)
# Assume that each file only contains a single experiment
assert (len(db) == 1)
db = db[0]
# Get beam vector
s0 = db.beam.get_unit_s0()
# Use the unit cell of the first sweep as reference
if reference_cell is None:
reference_cell = db.crystal.get_unit_cell()
Debug.write("Reference cell: %s" % str(reference_cell))
dials_reindex = Reindex()
dials_reindex.set_working_directory(working_directory)
dials_reindex.set_cb_op("auto")
dials_reindex.set_reference_filename(reference_vectors)
dials_reindex.set_experiments_filename(experiments_filename)
dials_reindex.set_indexed_filename(reflections_filename)
auto_logfiler(dials_reindex)
dials_reindex.run()
# Assume that all data are collected at same wavelength
if reference_wavelength is None:
reference_wavelength = intgr.get_wavelength()
else:
assert abs(reference_wavelength - intgr.get_wavelength()) < 0.01
Debug.write("wavelength: %f A" % intgr.get_wavelength())
Debug.write("distance: %f mm" % intgr.get_distance())
# Get integrated reflection data
import dials
with open(dials_reindex.get_reindexed_reflections_filename(), 'rb') as fh:
reflections = pickle.load(fh)
selection = reflections.get_flags(reflections.flags.used_in_refinement)
Chatter.write("Found %d reflections used in refinement (out of %d entries)" % (selection.count(True), len(reflections['miller_index'])))
reflections = reflections.select(selection)
# Filter bad reflections
selection = reflections['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
reflections.del_selected(selection)
print 'Removing %d reflections with negative variance' % \
selection.count(True)
if 'intensity.prf.variance' in reflections:
selection = reflections['intensity.prf.variance'] <= 0
if selection.count(True) > 0:
reflections.del_selected(selection)
print 'Removing %d profile reflections with negative variance' % \
selection.count(True)
# Find the observed 2theta angles
miller_indices = flex.miller_index()
two_thetas_obs = flex.double()
for pixel, panel, hkl in zip(reflections['xyzobs.px.value'], reflections['panel'], reflections['miller_index']):
assert hkl != (0, 0, 0)
two_thetas_obs.append(db.detector[panel].get_two_theta_at_pixel(s0, pixel[0:2]))
miller_indices.append(hkl)
# Convert observed 2theta angles to degrees
two_thetas_obs = two_thetas_obs * 180 / 3.14159265359
Chatter.write("Remaining %d reflections are in 2theta range %.3f - %.3f deg" % (len(miller_indices), min(two_thetas_obs), max(two_thetas_obs)))
all_miller_indices.extend(miller_indices)
all_two_thetas.extend(two_thetas_obs)
return all_miller_indices, all_two_thetas, reference_cell, reference_lattice, reference_wavelength
def reconstruct_rogues(params):
assert os.path.exists('xia2.json')
from xia2.Schema.XProject import XProject
xinfo = XProject.from_json(filename='xia2.json')
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
import cPickle as pickle
import dials # because WARNING:root:No profile class gaussian_rs registered
crystals = xinfo.get_crystals()
assert len(crystals) == 1
for xname in crystals:
crystal = crystals[xname]
scaler = crystal._get_scaler()
epochs = scaler._sweep_handler.get_epochs()
rogues = os.path.join(scaler.get_working_directory(),
xname, 'scale', 'ROGUES')
rogue_reflections = munch_rogues(rogues)
batched_reflections = { }
for epoch in epochs:
si = scaler._sweep_handler.get_sweep_information(epoch)
intgr = si.get_integrater()
experiments = ExperimentListFactory.from_json_file(
intgr.get_integrated_experiments())
reflections = pickle.load(open(intgr.get_integrated_reflections()))
batched_reflections[si.get_batch_range()] = (experiments, reflections,
si.get_sweep_name())
# - look up reflection in reflection list, get bounding box
# - pull pixels given from image set, flatten these, write out
from dials.array_family import flex
from annlib_ext import AnnAdaptor as ann_adaptor
reflections_run = { }
for run in batched_reflections:
reflections_run[run] = []
for rogue in rogue_reflections:
b = rogue[0]
for run in batched_reflections:
if b >= run[0] and b <= run[1]:
reflections_run[run].append(rogue)
break
for run_no, run in enumerate(reflections_run):
experiment = batched_reflections[run][0]
reflections = batched_reflections[run][1]
name = batched_reflections[run][2]
rogues = reflections_run[run]
reference = flex.double()
scan = experiment.scans()[0]
images = experiment.imagesets()[0]
for xyz in reflections['xyzcal.px']:
reference.append(xyz[0])
reference.append(xyz[1])
reference.append(xyz[2])
search = flex.double()
for rogue in rogues:
search.append(rogue[1])
search.append(rogue[2])
search.append(scan.get_array_index_from_angle(rogue[3]))
ann = ann_adaptor(data=reference, dim=3, k=1)
ann.query(search)
keep = flex.bool(len(reflections), False)
for j, rogue in enumerate(rogues):
keep[ann.nn[j]] = True
reflections = reflections.select(keep==True)
if params.extract:
reflections["shoebox"] = flex.shoebox(
reflections["panel"],
reflections["bbox"],
allocate=True)
reflections.extract_shoeboxes(images, verbose=False)
if len(reflections_run) > 1:
output = params.output.reflections.replace(
'.pickle', '-%s.pickle' % name)
print 'Extracted %d rogue reflections for %s to %s' % \
(len(reflections), name, output)
reflections.as_pickle(output)
else:
output = params.output.reflections
print 'Extracted %d rogue reflections to %s' % \
(len(reflections), output)
reflections.as_pickle(output)
def _write_pickle(self, batch):
pass
def _write_predictions(self, predictions):
pass
@contextmanager
def open_shoebox_writer(filename):
writer = ShoeboxWriter(filename)
yield writer
if __name__ == '__main__':
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from dials.array_family import flex
experiments = ExperimentListFactory.from_json_file(
'/home/upc86896/Data/Data/i04-BAG-training/dials_processed/experiments.json')
predictions = flex.reflection_table.from_predictions(experiments[0])
predictions.compute_bbox(experiments[0], nsigma=3, sigma_d=0.024,
sigma_m=0.044)
zeta = predictions.compute_zeta(experiments[0])
mask = flex.abs(zeta) < 0.05
predictions.del_selected(mask)
with open_shoebox_writer("extracted.tar") as writer:
writer.write(predictions, experiments[0].imageset)
def run(self):
from xia2.Handlers.Streams import Chatter, Debug
if self._reindexing_operator:
Debug.write('Reindexing sweeps for dials.two_theta_refine')
from xia2.lib.bits import auto_logfiler
from xia2.Wrappers.Dials.Reindex import Reindex
self._reindexed_experiments, self._reindexed_reflections = [], []
for e, p in zip(self._experiments, self._pickles):
reindexer = Reindex()
reindexer.set_cb_op(self._reindexing_operator)
reindexer.set_experiments_filename(e)
reindexer.set_indexed_filename(p)
reindexer.set_working_directory(self.get_working_directory())
auto_logfiler(reindexer)
reindexer.run()
self._reindexed_experiments.append(reindexer.get_reindexed_experiments_filename())
self._reindexed_reflections.append(reindexer.get_reindexed_reflections_filename())
Debug.write('Running dials.two_theta_refine')
self._output_cif = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.cif' % self.get_xpid())
self._output_mmcif = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.mmcif' % self.get_xpid())
self._output_correlation_plot = os.path.join(
self.get_working_directory(),
'%s_dials.two_theta_refine.png' % self.get_xpid())
self._output_experiments = os.path.join(
self.get_working_directory(),
'%s_refined_cell.json' % self.get_xpid())
self.clear_command_line()
if self._reindexing_operator:
for experiment in self._reindexed_experiments:
self.add_command_line(experiment)
for pickle in self._reindexed_reflections:
self.add_command_line(pickle)
else:
for experiment in self._experiments:
self.add_command_line(experiment)
for pickle in self._pickles:
self.add_command_line(pickle)
self.add_command_line('output.cif=%s' % self._output_cif)
self.add_command_line('output.mmcif=%s' % self._output_mmcif)
if self._output_correlation_plot is not None:
self.add_command_line(
'output.correlation_plot.filename=%s' % self._output_correlation_plot)
if self._output_experiments is not None:
self.add_command_line(
'output.experiments=%s' % self._output_experiments)
if self._phil_file is not None:
self.add_command_line('%s' %self._phil_file)
self.start()
self.close_wait()
if not os.path.isfile(self._output_cif):
Chatter.write(
"TwoTheta refinement failed, see log file for more details:\n %s" % self.get_log_file())
raise RuntimeError, 'unit cell not refined'
self.check_for_errors()
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(self.get_output_experiments())
self._crystal = experiments.crystals()[0]
def run(self):
'''Execute the script.'''
import os, math
from cctbx.crystal import symmetry
from scitbx.array_family import flex
from libtbx import table_utils, easy_pickle
from xfel.command_line.cspad_cbf_metrology import find_files
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
table_header = ["","","","I","IsigI","N >","RMSD","Cutoff"]
table_header2 = ["Bin","Resolution Range","Completeness","","","cutoff","(um)",""]
# Parse the command line
params, options, all_paths = self.parser.parse_args(show_diff_phil=False, return_unhandled=True)
exp_paths = []
refl_paths = []
for path in all_paths:
exps, refs = find_files(path, "integrated")
exp_paths.extend(exps)
refl_paths.extend(refs)
assert len(exp_paths) == len(refl_paths)
best_data = {}
best_limits = flex.double()
for exp_path, refl_path in zip(exp_paths, refl_paths):
experiments = ExperimentListFactory.from_json_file(exp_path)
reflections = easy_pickle.load(refl_path)
exp_name = os.path.basename(exp_path)
if exp_name.startswith("idx-") and exp_name.endswith("_refined_experiments.json"):
tag = exp_name.lstrip("idx-").rstrip("_refined_experiments.json")
else:
tag = "%s, %s"%(exp_path, refl_path)
for exp_id, experiment in enumerate(experiments):
print "*"*80
print "Data table for", tag
table_data = []
table_data.append(table_header)
table_data.append(table_header2)
crystal = experiment.crystal
refls = reflections.select(reflections['id'] == exp_id)
sym = symmetry(unit_cell = crystal.get_unit_cell(), space_group = crystal.get_space_group())
d = crystal.get_unit_cell().d(refls['miller_index'])
mset = sym.miller_set(indices = refls['miller_index'].select(d>=params.d_min), anomalous_flag=False)
binner = mset.setup_binner(n_bins=params.n_bins)
acceptable_resolution_bins = []
for i in binner.range_used():
d_max, d_min = binner.bin_d_range(i)
sel = (d <= d_max) & (d > d_min)
sel &= refls['intensity.sum.value'] > 0
bin_refls = refls.select(sel)
n_refls = len(bin_refls)
avg_i = flex.mean(bin_refls['intensity.sum.value']) if n_refls > 0 else 0
avg_i_sigi = flex.mean(bin_refls['intensity.sum.value'] /
flex.sqrt(bin_refls['intensity.sum.variance'])) if n_refls > 0 else 0
acceptable_resolution_bins.append(avg_i_sigi >= params.sig_filter_sigma)
bright_refls = bin_refls.select((bin_refls['intensity.sum.value']/flex.sqrt(bin_refls['intensity.sum.variance'])) >= params.sig_filter_sigma)
n_bright = len(bright_refls)
rmsd_obs = 1000*math.sqrt((bright_refls['xyzcal.mm']-bright_refls['xyzobs.mm.value']).sum_sq()/n_bright) if n_bright > 0 else 0
table_row = []
table_row.append("%3d"%i)
table_row.append("%-13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False,
show_d_range=True, show_counts=False))
table_row.append("%13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False,
show_d_range=False, show_counts=True))
table_row.append("%.1f"%(avg_i))
table_row.append("%.1f"%(avg_i_sigi))
table_row.append("%3d"%n_bright)
table_row.append("%.1f"%(rmsd_obs))
table_data.append(table_row)
acceptable_resolution_bins = [acceptable_resolution_bins[i] for i in xrange(len(acceptable_resolution_bins))
if False not in acceptable_resolution_bins[:i+1]]
for b, row in zip(acceptable_resolution_bins, table_data[2:]):
if b:
row.append("X")
print table_utils.format(table_data,has_header=2,justify='center',delim=" ")
if any(acceptable_resolution_bins):
best_index = acceptable_resolution_bins.count(True)-1
best_row = table_data[best_index+2]
d_min = binner.bin_d_range(binner.range_used()[best_index])[1]
if len(best_limits) < params.best_count:
best_limits.append(d_min)
best_data[tag] = d_min, best_row
elif (d_min < best_limits).count(True) > 0:
worst_d_min = flex.max(best_limits)
for tag, data in best_data.iteritems():
if worst_d_min == data[0]:
best_data[tag] = d_min, best_row
best_limits[flex.first_index(best_limits, worst_d_min)] = d_min
break
print tag, "best row:", " ".join(best_row)
else:
print "Data didn't pass cutoff"
if len(best_limits) > 0:
print "*"*80
print "Top", len(best_limits)
for tag, data in best_data.iteritems():
print tag, " ".join(data[1])
def test4():
'''Test group restraint with multiple crystals, and a stills refiner'''
if not libtbx.env.has_module("dials_regression"):
print "Skipping test2 in " + __file__ + " as dials_regression not present"
return
# The phil scope
from dials.algorithms.refinement.refiner import phil_scope
user_phil = parse('''
refinement
{
parameterisation
{
crystal
{
unit_cell
{
restraints
{
tie_to_group
{
sigmas=1,0,2,0,0,0
apply_to_all=true
}
}
}
}
}
}
''')
working_phil = phil_scope.fetch(source=user_phil)
working_params = working_phil.extract()
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
# use the multi stills test data
data_dir = os.path.join(dials_regression, "refinement_test_data", "multi_stills")
experiments_path = os.path.join(data_dir, "combined_experiments.json")
pickle_path = os.path.join(data_dir, "combined_reflections.pickle")
experiments = ExperimentListFactory.from_json_file(experiments_path,
check_format=False)
reflections = flex.reflection_table.from_pickle(pickle_path)
refiner = RefinerFactory.from_parameters_data_experiments(working_params,
reflections, experiments)
# hack to extract the objects needed from the Refiner
rp = refiner._target._restraints_parameterisation
pred_param = refiner._pred_param
# get analytical values and gradients
vals, grads, weights = rp.get_residuals_gradients_and_weights()
# get finite difference gradients
p_vals = pred_param.get_param_vals()
deltas = [1.e-7] * len(p_vals)
fd_grad=[]
for i in range(len(deltas)):
val = p_vals[i]
p_vals[i] -= deltas[i] / 2.
pred_param.set_param_vals(p_vals)
rev_state, foo, bar = rp.get_residuals_gradients_and_weights()
rev_state = flex.double(rev_state)
p_vals[i] += deltas[i]
pred_param.set_param_vals(p_vals)
fwd_state, foo, bar = rp.get_residuals_gradients_and_weights()
fwd_state = flex.double(fwd_state)
p_vals[i] = val
fd = (fwd_state - rev_state) / deltas[i]
fd_grad.append(fd)
# for comparison, fd_grad is a list of flex.doubles, each of which corresponds
# to the gradients of the residuals wrt to a single parameter.
pnames = pred_param.get_param_names()
for i, (pname, fd) in enumerate(zip(pnames, fd_grad)):
# extract dense column from the sparse matrix
an = grads.col(i).as_dense_vector()
#print pname
#print list(an.round(6))
#print list(fd.round(6))
#print
assert approx_equal(an, fd, eps=1e-5)
print "OK"
return
def get_dials_matrix(experiments_json):
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_json_file(experiments_json)
return experiments[0].crystal.get_A()
# run_test_single(imageset, 10000000),
# run_test_single(imageset, 50000000),
# run_test_single(imageset, 100000000),
# run_test_single(imageset, 250000000)
# ]
# outfile = open("profile.txt", "w")
# for r in results:
# print >>outfile, r[0], r[1]
if __name__ == '__main__':
import sys
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
exlist = ExperimentListFactory.from_json_file(sys.argv[1])
if len(sys.argv) > 2:
imageset = exlist[0].imageset[0:int(sys.argv[2])]
else:
imageset = exlist[0].imageset
print len(imageset)
# run_test(imageset)
from time import time
st = time()
n = int(len(imageset) / 4)
for i in range(0, n):
image = imageset[i]
from os.path import join
# path = '/home/upc86896/Projects/cctbx/sources/dials_regression/centroid_test_data'
path = "/home/upc86896/Data/Data/i04-BAG-training/dials_processed/"
experiment_list_filename = join(path, "experiments.json")
if len(sys.argv) > 1:
nproc = int(sys.argv[1])
else:
nproc = 1
print("Reading Experiments")
from math import pi
experiments = ExperimentListFactory.from_json_file(
experiment_list_filename)
profile_model = ProfileModelList()
profile_model.append(
ProfileModel(n_sigma=3,
sigma_b=0.024 * pi / 180,
sigma_m=0.044 * pi / 180))
print("Predicting Reflections")
rlist = flex.reflection_table.from_predictions(experiments[0])
rlist["id"] = flex.int(len(rlist), 0)
rlist.compute_bbox(experiments, profile_model)
rlist.compute_zeta_multi(experiments)
rlist.compute_d(experiments)
print("")
def run():
from dxtbx.model.experiment.experiment_list import ExperimentListFactory
from os.path import join
import libtbx.load_env
try:
dials_regression = libtbx.env.dist_path('dials_regression')
except KeyError, e:
print 'FAIL: dials_regression not configured'
exit(0)
path = join(dials_regression, "nexus_test_data", "shared_models")
filename_list = [
'single',
'multiple_unrelated',
'multi_crystal',
'two_colour',
'multiple_sweeps',
'stills'
]
for filename in filename_list:
filename_in = join(path, "%s.json" % filename)
filename_out = "%s.nxs" % filename
experiments = ExperimentListFactory.from_json_file(filename_in)
run_single(experiments, filename_out)
if __name__ == '__main__':
from dials.test import cd_auto
with cd_auto(__file__):
test_polarization_conversion()
run()
if __name__ == '__main__': from dials.array_family import flex from dxtbx.model.experiment.experiment_list import ExperimentListFactory import os.path path = "/home/upc86896/Projects/cctbx/sources/dials_regression/centroid_test_data" rlist_filename = os.path.join(path, "integrated.pickle") exlist_filename = os.path.join(path, "experiments.json") rlist = flex.reflection_table.from_pickle(rlist_filename) exlist = ExperimentListFactory.from_json_file(exlist_filename) panel = rlist['panel'] bbox = rlist['bbox'] rlist['shoebox'] = flex.shoebox(panel, bbox) rlist['shoebox'].allocate() rlist.fill_shoeboxes(exlist[0].imageset)
