def candidate_orientation_matrices(basis_vectors, max_combinations=None):
# select unique combinations of input vectors to test
# the order of combinations is such that combinations comprising vectors
# nearer the beginning of the input list will appear before combinations
# comprising vectors towards the end of the list
n = len(basis_vectors)
# hardcoded limit on number of vectors, fixes issue #72
# https://github.com/dials/dials/issues/72
n = min(n, 100)
basis_vectors = basis_vectors[:n]
combinations = flex.vec3_int(flex.nested_loop((n, n, n)))
combinations = combinations.select(
flex.sort_permutation(combinations.as_vec3_double().norms()))
# select only those combinations where j > i and k > j
i, j, k = combinations.as_vec3_double().parts()
sel = flex.bool(len(combinations), True)
sel &= j > i
sel &= k > j
combinations = combinations.select(sel)
if max_combinations is not None and max_combinations < len(combinations):
combinations = combinations[:max_combinations]
half_pi = 0.5 * math.pi
min_angle = 20 / 180 * math.pi # 20 degrees, arbitrary cutoff
for i, j, k in combinations:
a = basis_vectors[i]
b = basis_vectors[j]
angle = a.angle(b)
if angle < min_angle or (math.pi - angle) < min_angle:
continue
a_cross_b = a.cross(b)
gamma = a.angle(b)
if gamma < half_pi:
# all angles obtuse if possible please
b = -b
a_cross_b = -a_cross_b
c = basis_vectors[k]
if abs(half_pi - a_cross_b.angle(c)) < min_angle:
continue
alpha = b.angle(c, deg=True)
if alpha < half_pi:
c = -c
if a_cross_b.dot(c) < 0:
# we want right-handed basis set, therefore invert all vectors
a = -a
b = -b
c = -c
model = Crystal(a, b, c, space_group_symbol="P 1")
uc = model.get_unit_cell()
cb_op_to_niggli = uc.change_of_basis_op_to_niggli_cell()
model = model.change_basis(cb_op_to_niggli)
uc = model.get_unit_cell()
params = uc.parameters()
if uc.volume() > (params[0] * params[1] * params[2] / 100):
# unit cell volume cutoff from labelit 2004 paper
yield model
def apply_symmetry(self, crystal_model):
if not (self.target_symmetry_primitive
and self.target_symmetry_primitive.space_group()):
return crystal, sgtbx.change_of_basis_op()
target_space_group = self.target_symmetry_primitive.space_group()
A = crystal_model.get_A()
max_delta = self._max_delta
items = iotbx_converter(crystal_model.get_unit_cell(),
max_delta=max_delta)
target_sg_ref = target_space_group.info().reference_setting().group()
best_angular_difference = 1e8
best_subgroup = None
for item in items:
if bravais_lattice(group=target_sg_ref) != bravais_lattice(
group=item["ref_subsym"].space_group()):
continue
if item["max_angular_difference"] < best_angular_difference:
best_angular_difference = item["max_angular_difference"]
best_subgroup = item
if best_subgroup is None:
return None, None
cb_op_inp_best = best_subgroup["cb_op_inp_best"]
orient = crystal_orientation(A, True)
orient_best = orient.change_basis(
scitbx.matrix.sqr(
cb_op_inp_best.c().as_double_array()[0:9]).transpose())
constrain_orient = orient_best.constrain(best_subgroup["system"])
best_subsym = best_subgroup["best_subsym"]
cb_op_best_ref = best_subsym.change_of_basis_op_to_reference_setting()
target_sg_best = target_sg_ref.change_basis(cb_op_best_ref.inverse())
ref_subsym = best_subsym.change_basis(cb_op_best_ref)
cb_op_ref_primitive = ref_subsym.change_of_basis_op_to_primitive_setting(
)
cb_op_best_primitive = cb_op_ref_primitive * cb_op_best_ref
cb_op_inp_primitive = cb_op_ref_primitive * cb_op_best_ref * cb_op_inp_best
direct_matrix = constrain_orient.direct_matrix()
a = scitbx.matrix.col(direct_matrix[:3])
b = scitbx.matrix.col(direct_matrix[3:6])
c = scitbx.matrix.col(direct_matrix[6:9])
model = Crystal(a, b, c, space_group=target_sg_best)
assert target_sg_best.is_compatible_unit_cell(model.get_unit_cell())
model = model.change_basis(cb_op_best_primitive)
return model, cb_op_inp_primitive
def filter_doubled_cell(solutions):
accepted_solutions = []
for i1, s1 in enumerate(solutions):
doubled_cell = False
for (m1, m2, m3) in (
(2, 1, 1),
(1, 2, 1),
(1, 1, 2),
(2, 2, 1),
(2, 1, 2),
(1, 2, 2),
(2, 2, 2),
):
if doubled_cell:
break
a, b, c = (matrix.col(v)
for v in s1.crystal.get_real_space_vectors())
new_cryst = Crystal(
real_space_a=1 / m1 * a,
real_space_b=1 / m2 * b,
real_space_c=1 / m3 * c,
space_group=s1.crystal.get_space_group(),
)
new_unit_cell = new_cryst.get_unit_cell()
for s2 in solutions:
if s2 is s1:
continue
if new_unit_cell.is_similar_to(s2.crystal.get_unit_cell(),
relative_length_tolerance=0.05):
R, axis, angle, cb = difference_rotation_matrix_axis_angle(
new_cryst, s2.crystal)
if (angle < 1) and (s1.n_indexed < (1.1 * s2.n_indexed)):
doubled_cell = True
break
if not doubled_cell:
accepted_solutions.append(s1)
return accepted_solutions
def test_crystal_model():
real_space_a = matrix.col((10, 0, 0))
real_space_b = matrix.col((0, 11, 0))
real_space_c = matrix.col((0, 0, 12))
model = Crystal(
real_space_a=(10, 0, 0),
real_space_b=(0, 11, 0),
real_space_c=(0, 0, 12),
space_group_symbol="P 1",
)
# This doesn't work as python class uctbx.unit_cell(uctbx_ext.unit_cell)
# so C++ and python classes are different types
# assert isinstance(model.get_unit_cell(), uctbx.unit_cell)
assert model.get_unit_cell().parameters() == (10.0, 11.0, 12.0, 90.0, 90.0,
90.0)
assert approx_equal(model.get_A(),
(1 / 10, 0, 0, 0, 1 / 11, 0, 0, 0, 1 / 12))
assert approx_equal(
matrix.sqr(model.get_A()).inverse(), (10, 0, 0, 0, 11, 0, 0, 0, 12))
assert approx_equal(model.get_B(), model.get_A())
assert approx_equal(model.get_U(), (1, 0, 0, 0, 1, 0, 0, 0, 1))
assert approx_equal(model.get_real_space_vectors(),
(real_space_a, real_space_b, real_space_c))
assert (model.get_crystal_symmetry().unit_cell().parameters() ==
model.get_unit_cell().parameters())
assert model.get_crystal_symmetry().space_group() == model.get_space_group(
)
model2 = Crystal(
real_space_a=(10, 0, 0),
real_space_b=(0, 11, 0),
real_space_c=(0, 0, 12),
space_group_symbol="P 1",
)
assert model == model2
model2a = Crystal(model.get_A(), model.get_space_group())
assert model == model2a
model2b = Crystal(
matrix.sqr(model.get_A()).inverse().elems,
model.get_space_group().type().lookup_symbol(),
reciprocal=False,
)
assert model == model2b
# rotate 45 degrees about x-axis
R1 = matrix.sqr((
1,
0,
0,
0,
math.cos(math.pi / 4),
-math.sin(math.pi / 4),
0,
math.sin(math.pi / 4),
math.cos(math.pi / 4),
))
# rotate 30 degrees about y-axis
R2 = matrix.sqr((
math.cos(math.pi / 6),
0,
math.sin(math.pi / 6),
0,
1,
0,
-math.sin(math.pi / 6),
0,
math.cos(math.pi / 6),
))
# rotate 60 degrees about z-axis
R3 = matrix.sqr((
math.cos(math.pi / 3),
-math.sin(math.pi / 3),
0,
math.sin(math.pi / 3),
math.cos(math.pi / 3),
0,
0,
0,
1,
))
R = R1 * R2 * R3
model.set_U(R)
# B is unchanged
assert approx_equal(model.get_B(),
(1 / 10, 0, 0, 0, 1 / 11, 0, 0, 0, 1 / 12))
assert approx_equal(model.get_U(), R)
assert approx_equal(model.get_A(),
matrix.sqr(model.get_U()) * matrix.sqr(model.get_B()))
a_, b_, c_ = model.get_real_space_vectors()
assert approx_equal(a_, R * real_space_a)
assert approx_equal(b_, R * real_space_b)
assert approx_equal(c_, R * real_space_c)
assert (str(model).replace("-0.0000", " 0.0000") == """\
Crystal:
Unit cell: (10.000, 11.000, 12.000, 90.000, 90.000, 90.000)
Space group: P 1
U matrix: {{ 0.4330, -0.7500, 0.5000},
{ 0.7891, 0.0474, -0.6124},
{ 0.4356, 0.6597, 0.6124}}
B matrix: {{ 0.1000, 0.0000, 0.0000},
{ 0.0000, 0.0909, 0.0000},
{ 0.0000, 0.0000, 0.0833}}
A = UB: {{ 0.0433, -0.0682, 0.0417},
{ 0.0789, 0.0043, -0.0510},
{ 0.0436, 0.0600, 0.0510}}
""")
model.set_B((1 / 12, 0, 0, 0, 1 / 12, 0, 0, 0, 1 / 12))
assert approx_equal(model.get_unit_cell().parameters(),
(12, 12, 12, 90, 90, 90))
U = matrix.sqr((0.3455, -0.2589, -0.9020, 0.8914, 0.3909, 0.2293, 0.2933,
-0.8833, 0.3658))
B = matrix.sqr((1 / 13, 0, 0, 0, 1 / 13, 0, 0, 0, 1 / 13))
model.set_A(U * B)
assert approx_equal(model.get_A(), U * B)
assert approx_equal(model.get_U(), U, 1e-4)
assert approx_equal(model.get_B(), B, 1e-5)
model3 = Crystal(
real_space_a=(10, 0, 0),
real_space_b=(0, 11, 0),
real_space_c=(0, 0, 12),
space_group=sgtbx.space_group_info("P 222").group(),
)
assert model3.get_space_group().type().hall_symbol() == " P 2 2"
assert model != model3
#
sgi_ref = sgtbx.space_group_info(number=230)
model_ref = Crystal(
real_space_a=(44, 0, 0),
real_space_b=(0, 44, 0),
real_space_c=(0, 0, 44),
space_group=sgi_ref.group(),
)
assert approx_equal(model_ref.get_U(), (1, 0, 0, 0, 1, 0, 0, 0, 1))
assert approx_equal(model_ref.get_B(),
(1 / 44, 0, 0, 0, 1 / 44, 0, 0, 0, 1 / 44))
assert approx_equal(model_ref.get_A(), model_ref.get_B())
assert approx_equal(model_ref.get_unit_cell().parameters(),
(44, 44, 44, 90, 90, 90))
a_ref, b_ref, c_ref = map(matrix.col, model_ref.get_real_space_vectors())
cb_op_to_primitive = sgi_ref.change_of_basis_op_to_primitive_setting()
model_primitive = model_ref.change_basis(cb_op_to_primitive)
cb_op_to_reference = (model_primitive.get_space_group().info().
change_of_basis_op_to_reference_setting())
a_prim, b_prim, c_prim = map(matrix.col,
model_primitive.get_real_space_vectors())
assert (cb_op_to_primitive.as_abc() ==
"-1/2*a+1/2*b+1/2*c,1/2*a-1/2*b+1/2*c,1/2*a+1/2*b-1/2*c")
assert approx_equal(a_prim, -1 / 2 * a_ref + 1 / 2 * b_ref + 1 / 2 * c_ref)
assert approx_equal(b_prim, 1 / 2 * a_ref - 1 / 2 * b_ref + 1 / 2 * c_ref)
assert approx_equal(c_prim, 1 / 2 * a_ref + 1 / 2 * b_ref - 1 / 2 * c_ref)
assert cb_op_to_reference.as_abc() == "b+c,a+c,a+b"
assert approx_equal(a_ref, b_prim + c_prim)
assert approx_equal(b_ref, a_prim + c_prim)
assert approx_equal(c_ref, a_prim + b_prim)
assert approx_equal(
model_primitive.get_U(),
[
-0.5773502691896258,
0.40824829046386285,
0.7071067811865476,
0.5773502691896257,
-0.4082482904638631,
0.7071067811865476,
0.5773502691896257,
0.8164965809277259,
0.0,
],
)
assert approx_equal(
model_primitive.get_B(),
[
0.0262431940540739,
0.0,
0.0,
0.00927837023781507,
0.02783511071344521,
0.0,
0.01607060866333063,
0.01607060866333063,
0.03214121732666125,
],
)
assert approx_equal(
model_primitive.get_A(),
(0, 1 / 44, 1 / 44, 1 / 44, 0, 1 / 44, 1 / 44, 1 / 44, 0),
)
assert approx_equal(
model_primitive.get_unit_cell().parameters(),
[
38.1051177665153,
38.1051177665153,
38.1051177665153,
109.47122063449069,
109.47122063449069,
109.47122063449069,
],
)
assert model_ref != model_primitive
model_ref_recycled = model_primitive.change_basis(cb_op_to_reference)
assert approx_equal(model_ref.get_U(), model_ref_recycled.get_U())
assert approx_equal(model_ref.get_B(), model_ref_recycled.get_B())
assert approx_equal(model_ref.get_A(), model_ref_recycled.get_A())
assert approx_equal(
model_ref.get_unit_cell().parameters(),
model_ref_recycled.get_unit_cell().parameters(),
)
assert model_ref == model_ref_recycled
uc = uctbx.unit_cell(
(58.2567, 58.1264, 39.7093, 46.9077, 46.8612, 62.1055))
sg = sgtbx.space_group_info(symbol="P1").group()
cs = crystal.symmetry(unit_cell=uc, space_group=sg)
cb_op_to_minimum = cs.change_of_basis_op_to_minimum_cell()
# the reciprocal matrix
B = matrix.sqr(uc.fractionalization_matrix()).transpose()
U = random_rotation()
direct_matrix = (U * B).inverse()
model = Crystal(direct_matrix[:3],
direct_matrix[3:6],
direct_matrix[6:9],
space_group=sg)
assert uc.is_similar_to(model.get_unit_cell())
uc_minimum = uc.change_basis(cb_op_to_minimum)
model_minimum = model.change_basis(cb_op_to_minimum)
assert uc_minimum.is_similar_to(model_minimum.get_unit_cell())
assert model_minimum != model
model_minimum.update(model)
assert model_minimum == model # lgtm
A_static = matrix.sqr(model.get_A())
A_as_scan_points = [A_static]
num_scan_points = 11
for i in range(num_scan_points - 1):
A_as_scan_points.append(
A_as_scan_points[-1] *
matrix.sqr(euler_angles.xyz_matrix(0.1, 0.2, 0.3)))
model.set_A_at_scan_points(A_as_scan_points)
model_minimum = model.change_basis(cb_op_to_minimum)
assert model.num_scan_points == model_minimum.num_scan_points == num_scan_points
M = matrix.sqr(cb_op_to_minimum.c_inv().r().transpose().as_double())
M_inv = M.inverse()
for i in range(num_scan_points):
A_orig = matrix.sqr(model.get_A_at_scan_point(i))
A_min = matrix.sqr(model_minimum.get_A_at_scan_point(i))
assert approx_equal(A_min, A_orig * M_inv)
assert model.get_unit_cell().parameters() == pytest.approx(
(58.2567, 58.1264, 39.7093, 46.9077, 46.8612, 62.1055))
uc = uctbx.unit_cell((10, 11, 12, 91, 92, 93))
model.set_unit_cell(uc)
assert model.get_unit_cell().parameters() == pytest.approx(uc.parameters())
def test_check_old_vs_new():
from dxtbx.tests.model.crystal_model_old import crystal_model_old
model_1 = Crystal(
real_space_a=(10, 0, 0),
real_space_b=(0, 11, 0),
real_space_c=(0, 0, 12),
space_group_symbol="P 1",
)
model_2 = crystal_model_old(
real_space_a=(10, 0, 0),
real_space_b=(0, 11, 0),
real_space_c=(0, 0, 12),
space_group_symbol="P 1",
)
cov_B = matrix.sqr([1] * (9 * 9))
model_1.set_B_covariance(cov_B)
model_2.set_B_covariance(cov_B)
A_list = [model_1.get_A() for i in range(20)]
model_1.set_A_at_scan_points(A_list)
model_2.set_A_at_scan_points(A_list)
A1 = model_1.get_A()
A2 = model_2.get_A()
U1 = model_1.get_U()
U2 = model_2.get_U()
B1 = model_1.get_B()
B2 = model_2.get_B()
UC1 = model_1.get_unit_cell()
UC2 = model_2.get_unit_cell()
RSV1 = model_1.get_real_space_vectors()
RSV2 = model_2.get_real_space_vectors()
SG1 = model_1.get_space_group()
SG2 = model_2.get_space_group()
assert model_1.num_scan_points == model_2.num_scan_points
A_list_1 = [
model_1.get_A_at_scan_point(i)
for i in range(model_1.get_num_scan_points())
]
A_list_2 = [
model_2.get_A_at_scan_point(i)
for i in range(model_1.get_num_scan_points())
]
B_list_1 = [
model_1.get_B_at_scan_point(i)
for i in range(model_1.get_num_scan_points())
]
B_list_2 = [
model_2.get_B_at_scan_point(i)
for i in range(model_1.get_num_scan_points())
]
U_list_1 = [
model_1.get_U_at_scan_point(i)
for i in range(model_1.get_num_scan_points())
]
U_list_2 = [
model_2.get_U_at_scan_point(i)
for i in range(model_1.get_num_scan_points())
]
assert approx_equal(A1, A2)
assert approx_equal(B1, B2)
assert approx_equal(U1, U2)
assert approx_equal(UC1.parameters(), UC2.parameters())
assert approx_equal(RSV1[0], RSV2[0])
assert approx_equal(RSV1[1], RSV2[1])
assert approx_equal(RSV1[2], RSV2[2])
assert str(SG1.info()) == str(SG2.info())
for i in range(model_1.get_num_scan_points()):
assert approx_equal(A_list_1[i], A_list_2[i])
assert approx_equal(B_list_1[i], B_list_2[i])
assert approx_equal(U_list_1[i], U_list_2[i])
cell_sd_1 = model_1.get_cell_parameter_sd()
cell_sd_2 = model_2.get_cell_parameter_sd()
cell_volume_sd_1 = model_1.get_cell_volume_sd()
cell_volume_sd_2 = model_2.get_cell_volume_sd()
covB1 = model_1.get_B_covariance()
covB2 = model_1.get_B_covariance()
assert approx_equal(covB1, covB2)
assert approx_equal(cell_volume_sd_1, cell_volume_sd_2)
assert approx_equal(cell_sd_1, cell_sd_2)
def test_refinement(dials_regression):
"""Test a refinement run"""
# Get a beam and detector from a experiments. This one has a CS-PAD, but that
# is irrelevant
data_dir = os.path.join(dials_regression, "refinement_test_data",
"hierarchy_test")
experiments_path = os.path.join(data_dir, "datablock.json")
assert os.path.exists(experiments_path)
# load models
from dxtbx.model.experiment_list import ExperimentListFactory
experiments = ExperimentListFactory.from_serialized_format(
experiments_path, check_format=False)
im_set = experiments.imagesets()[0]
detector = deepcopy(im_set.get_detector())
beam = im_set.get_beam()
# Invent a crystal, goniometer and scan for this test
from dxtbx.model import Crystal
crystal = Crystal((40.0, 0.0, 0.0), (0.0, 40.0, 0.0), (0.0, 0.0, 40.0),
space_group_symbol="P1")
orig_xl = deepcopy(crystal)
from dxtbx.model import GoniometerFactory
goniometer = GoniometerFactory.known_axis((1.0, 0.0, 0.0))
# Build a mock scan for a 180 degree sequence
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=list(range(1800)),
deg=True,
)
sequence_range = scan.get_oscillation_range(deg=False)
im_width = scan.get_oscillation(deg=False)[1]
assert sequence_range == (0.0, pi)
assert approx_equal(im_width, 0.1 * pi / 180.0)
# Build an experiment list
experiments = ExperimentList()
experiments.append(
Experiment(
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None,
))
# simulate some reflections
refs, _ = generate_reflections(experiments)
# change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of
# alpha and beta angles)
from dials.algorithms.refinement.parameterisation.crystal_parameters import (
CrystalUnitCellParameterisation, )
xluc_param = CrystalUnitCellParameterisation(crystal)
cell_params = crystal.get_unit_cell().parameters()
cell_params = [
a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])
]
from cctbx.uctbx import unit_cell
from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge
from scitbx import matrix
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(crystal.get_space_group())
S.set_orientation(orientation=newB)
X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()])
xluc_param.set_param_vals(X)
# reparameterise the crystal at the perturbed geometry
xluc_param = CrystalUnitCellParameterisation(crystal)
# Dummy parameterisations for other models
beam_param = None
xlo_param = None
det_param = None
# parameterisation of the prediction equation
from dials.algorithms.refinement.parameterisation.parameter_report import (
ParameterReporter, )
pred_param = TwoThetaPredictionParameterisation(experiments, det_param,
beam_param, xlo_param,
[xluc_param])
param_reporter = ParameterReporter(det_param, beam_param, xlo_param,
[xluc_param])
# reflection manager
refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20)
# reflection predictor
ref_predictor = TwoThetaExperimentsPredictor(experiments)
# target function
target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param)
# minimisation engine
from dials.algorithms.refinement.engine import (
LevenbergMarquardtIterations as Refinery, )
refinery = Refinery(
target=target,
prediction_parameterisation=pred_param,
log=None,
max_iterations=20,
)
# Refiner
from dials.algorithms.refinement.refiner import Refiner
refiner = Refiner(
experiments=experiments,
pred_param=pred_param,
param_reporter=param_reporter,
refman=refman,
target=target,
refinery=refinery,
)
refiner.run()
# compare crystal with original crystal
refined_xl = refiner.get_experiments()[0].crystal
# print refined_xl
assert refined_xl.is_similar_to(orig_xl,
uc_rel_length_tolerance=0.001,
uc_abs_angle_tolerance=0.01)
def write_par_file(file_name, experiment):
from dxtbx.model import Crystal
from iotbx.mtz.extract_from_symmetry_lib import ccp4_symbol
from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame
from scitbx import matrix
imageset = experiment.imageset
detector = imageset.get_detector()
goniometer = imageset.get_goniometer()
beam = imageset.get_beam()
scan = imageset.get_scan()
R_to_mosflm = align_reference_frame(beam.get_s0(), (1.0, 0.0, 0.0),
goniometer.get_rotation_axis(),
(0.0, 0.0, 1.0))
cryst = experiment.crystal
cryst = cryst.change_basis(cryst.get_space_group().info().
change_of_basis_op_to_reference_setting())
A = matrix.sqr(cryst.get_A())
A_inv = A.inverse()
real_space_a = R_to_mosflm * A_inv.elems[:3]
real_space_b = R_to_mosflm * A_inv.elems[3:6]
real_space_c = R_to_mosflm * A_inv.elems[6:9]
cryst_mosflm = Crystal(real_space_a,
real_space_b,
real_space_c,
space_group=cryst.get_space_group())
U_mosflm = matrix.sqr(cryst_mosflm.get_U())
B_mosflm = matrix.sqr(cryst_mosflm.get_B())
UB_mosflm = U_mosflm * B_mosflm
uc_params = cryst_mosflm.get_unit_cell().parameters()
assert U_mosflm.is_r3_rotation_matrix(), U_mosflm
beam_centre = tuple(reversed(detector[0].get_beam_centre(beam.get_s0())))
distance = detector[0].get_directed_distance()
polarization = R_to_mosflm * matrix.col(beam.get_polarization_normal())
rotation = matrix.col(goniometer.get_rotation_axis())
if rotation.angle(matrix.col(
detector[0].get_fast_axis())) < rotation.angle(
matrix.col(detector[0].get_slow_axis())):
direction = "FAST"
else:
direction = "SLOW"
rotation = R_to_mosflm * rotation
# Calculate average spot diameter for SEPARATION parameter
# http://xds.mpimf-heidelberg.mpg.de/html_doc/xds_parameters.html
# BEAM_DIVERGENCE=
# This value is approximately arctan(spot diameter/DETECTOR_DISTANCE)
profile = experiment.profile
spot_diameter = math.tan(profile.delta_b() * math.pi / 180) * distance
spot_diameter_px = spot_diameter * detector[0].get_pixel_size()[0]
# determine parameters for RASTER keyword
# http://www.mrc-lmb.cam.ac.uk/harry/cgi-bin/keyword2.cgi?RASTER
# NXS, NYS (odd integers) define the overall dimensions of the rectangular array of pixels for each spot
# NXS and NYS are set to twice the spot size plus 5 pixels
nxs = 2 * int(math.ceil(spot_diameter_px)) + 5
nys = nxs
# NRX, NRY are the number of columns or rows of points in the background rim
# NRX and NRY are set to half the spot size plus 2 pixels
nrx = int(math.ceil(0.5 * spot_diameter_px)) + 2
nry = nrx
# NC the corner background cut-off which corresponds to a half-square of side NC points
# NC is set to the mean of the spot size in X and Y plus 4
nc = int(math.ceil(spot_diameter_px)) + 4
def space_group_symbol(space_group):
symbol = ccp4_symbol(space_group.info(),
lib_name="syminfo.lib",
require_at_least_one_lib=False)
if symbol != "P 1":
symbol = symbol.replace(" 1", "")
symbol = symbol.replace(" ", "")
return symbol
logger.info("Saving BEST parameter file to %s", file_name)
with open(file_name, "w") as f:
print("# parameter file for BEST", file=f)
print("TITLE From DIALS", file=f)
print("DETECTOR PILA", file=f)
print("SITE Not set", file=f)
print(
"DIAMETER %6.2f" % (max(detector[0].get_image_size()) *
detector[0].get_pixel_size()[0]),
file=f,
)
print(f"PIXEL {round(detector[0].get_pixel_size()[0], 10)}",
file=f)
print("ROTAXIS %4.2f %4.2f %4.2f" % rotation.elems,
direction,
file=f)
print("POLAXIS %4.2f %4.2f %4.2f" % polarization.elems, file=f)
print("GAIN 1.00", file=f) # correct for Pilatus images
# http://strucbio.biologie.uni-konstanz.de/xdswiki/index.php/FAQ#You_said_that_the_XDS_deals_with_high_mosaicity._How_high_mosaicity_is_still_manageable.3F
# http://journals.iucr.org/d/issues/2012/01/00/wd5161/index.html
# Transform from XDS definition of sigma_m to FWHM (MOSFLM mosaicity definition)
print(f"CMOSAIC {experiment.profile.sigma_m() * 2.355:.2f}",
file=f)
print(f"PHISTART {scan.get_oscillation_range()[0]:.2f}",
file=f)
print(f"PHIWIDTH {scan.get_oscillation()[1]:.2f}", file=f)
print(f"DISTANCE {distance:7.2f}", file=f)
print(f"WAVELENGTH {beam.get_wavelength():.5f}", file=f)
print(f"POLARISATION {beam.get_polarization_fraction():7.5f}",
file=f)
print(f"SYMMETRY {space_group_symbol(cryst.get_space_group())}",
file=f)
print("UB %9.6f %9.6f %9.6f" % UB_mosflm[:3], file=f)
print(" %9.6f %9.6f %9.6f" % UB_mosflm[3:6], file=f)
print(" %9.6f %9.6f %9.6f" % UB_mosflm[6:], file=f)
print("CELL %8.2f %8.2f %8.2f %6.2f %6.2f %6.2f" % uc_params,
file=f)
print("RASTER %i %i %i %i %i" % (nxs, nys, nc, nrx, nry),
file=f)
print(f"SEPARATION {spot_diameter:.3f} {spot_diameter:.3f}",
file=f)
print("BEAM %8.3f %8.3f" % beam_centre, file=f)
print("# end of parameter file for BEST", file=f)
def load(self):
# some parameters
# NOTE: for reference, inside each h5 file there is
# [u'Amatrices', u'Hi', u'bboxes', u'h5_path']
# get the total number of shots using worker 0
if rank == 0:
print("I am root. I am calculating total number of shots")
h5s = [h5py_File(f, "r") for f in self.fnames]
Nshots_per_file = [h["h5_path"].shape[0] for h in h5s]
Nshots_tot = sum(Nshots_per_file)
print("I am root. Total number of shots is %d" % Nshots_tot)
print("I am root. I will divide shots amongst workers.")
shot_tuples = []
for i_f, fname in enumerate(self.fnames):
fidx_shotidx = [(i_f, i_shot)
for i_shot in range(Nshots_per_file[i_f])]
shot_tuples += fidx_shotidx
from numpy import array_split
print("I am root. Number of uniques = %d" % len(set(shot_tuples)))
shots_for_rank = array_split(shot_tuples, size)
# close the open h5s..
for h in h5s:
h.close()
else:
Nshots_tot = None
shots_for_rank = None
h5s = None
#Nshots_tot = comm.bcast( Nshots_tot, root=0)
if has_mpi:
shots_for_rank = comm.bcast(shots_for_rank, root=0)
#h5s = comm.bcast( h5s, root=0) # pull in the open hdf5 files
my_shots = shots_for_rank[rank]
if self.Nload is not None:
my_shots = my_shots[:self.Nload]
# open the unique filenames for this rank
# TODO: check max allowed pointers to open hdf5 file
my_unique_fids = set([fidx for fidx, _ in my_shots])
my_open_files = {
fidx: h5py_File(self.fnames[fidx], "r")
for fidx in my_unique_fids
}
Ntot = 0
self.all_bbox_pixels = []
for img_num, (fname_idx, shot_idx) in enumerate(my_shots):
if img_num == args.Nmax:
#print("Already processed maximum number images!")
continue
h = my_open_files[fname_idx]
# load the dxtbx image data directly:
npz_path = h["h5_path"][shot_idx]
# NOTE take me out!
if args.testmode:
import os
npz_path = os.path.basename(npz_path)
img_handle = numpy_load(npz_path)
img = img_handle["img"]
if len(img.shape) == 2: # if single panel>>
img = np.array([img])
#D = det_from_dict(img_handle["det"][()])
B = beam_from_dict(img_handle["beam"][()])
# get the indexed crystal Amatrix
Amat = h["Amatrices"][shot_idx]
amat_elems = list(sqr(Amat).inverse().elems)
# real space basis vectors:
a_real = amat_elems[:3]
b_real = amat_elems[3:6]
c_real = amat_elems[6:]
# dxtbx indexed crystal model
C = Crystal(a_real, b_real, c_real, "P43212")
# change basis here ? Or maybe just average a/b
a, b, c, _, _, _ = C.get_unit_cell().parameters()
a_init = .5 * (a + b)
c_init = c
# shoe boxes where we expect spots
bbox_dset = h["bboxes"]["shot%d" % shot_idx]
n_bboxes_total = bbox_dset.shape[0]
# is the shoe box within the resolution ring and does it have significant SNR (see filter_bboxes.py)
is_a_keeper = h["bboxes"]["keepers%d" % shot_idx][()]
# tilt plane to the background pixels in the shoe boxes
tilt_abc_dset = h["tilt_abc"]["shot%d" % shot_idx]
try:
panel_ids_dset = h["panel_ids"]["shot%d" % shot_idx]
has_panels = True
except KeyError:
has_panels = False
# apply the filters:
bboxes = [
bbox_dset[i_bb] for i_bb in range(n_bboxes_total)
if is_a_keeper[i_bb]
]
tilt_abc = [
tilt_abc_dset[i_bb] for i_bb in range(n_bboxes_total)
if is_a_keeper[i_bb]
]
if has_panels:
panel_ids = [
panel_ids_dset[i_bb] for i_bb in range(n_bboxes_total)
if is_a_keeper[i_bb]
]
else:
panel_ids = [0] * len(tilt_abc)
# how many pixels do we have
tot_pix = [(j2 - j1) * (i2 - i1) for i1, i2, j1, j2 in bboxes]
Ntot += sum(tot_pix)
# actually load the pixels...
#data_boxes = [ img[j1:j2, i1:i2] for i1,i2,j1,j2 in bboxes]
# Here we will try a per-shot refinement of the unit cell and Umatrix, as well as ncells abc
# and spot scale etc..
# load some ground truth data from the simulation dumps (e.g. spectrum)
h5_fname = h["h5_path"][shot_idx].replace(".npz", "")
# NOTE remove me
if args.testmode:
h5_fname = os.path.basename(h5_fname)
data = h5py_File(h5_fname, "r")
tru = sqr(data["crystalA"][()]).inverse().elems
a_tru = tru[:3]
b_tru = tru[3:6]
c_tru = tru[6:]
C_tru = Crystal(a_tru, b_tru, c_tru, "P43212")
try:
angular_offset_init = compare_with_ground_truth(
a_tru, b_tru, c_tru, [C], symbol="P43212")[0]
except Exception as err:
print(
"Rank %d: Boo cant use the comparison w GT function: %s" %
(rank, err))
fluxes = data["spectrum"][()]
es = data["exposure_s"][()]
fluxes *= es # multiply by the exposure time
spectrum = zip(wavelens, fluxes)
# dont simulate when there are no photons!
spectrum = [(wave, flux) for wave, flux in spectrum
if flux > self.flux_min]
# make a unit cell manager that the refiner will use to track the B-matrix
aa, _, cc, _, _, _ = C_tru.get_unit_cell().parameters()
ucell_man = TetragonalManager(a=a_init, c=c_init)
if args.startwithtruth:
ucell_man = TetragonalManager(a=aa, c=cc)
# create the sim_data instance that the refiner will use to run diffBragg
# create a nanoBragg crystal
nbcryst = nanoBragg_crystal()
nbcryst.dxtbx_crystal = C
if args.startwithtruth:
nbcryst.dxtbx_crystal = C_tru
nbcryst.thick_mm = 0.1
nbcryst.Ncells_abc = 30, 30, 30
nbcryst.miller_array = Fhkl_guess.as_amplitude_array()
nbcryst.n_mos_domains = 1
nbcryst.mos_spread_deg = 0.0
# create a nanoBragg beam
nbbeam = nanoBragg_beam()
nbbeam.size_mm = 0.001
nbbeam.unit_s0 = B.get_unit_s0()
nbbeam.spectrum = spectrum
# sim data instance
SIM = SimData()
SIM.detector = CSPAD
#SIM.detector = D
SIM.crystal = nbcryst
SIM.beam = nbbeam
SIM.panel_id = 0 # default
spot_scale = 12
if args.sad:
spot_scale = 1
SIM.instantiate_diffBragg(default_F=0, oversample=0)
SIM.D.spot_scale = spot_scale
img_in_photons = img / self.gain
print("Rank %d, Starting refinement!" % rank)
try:
RUC = RefineAllMultiPanel(
spot_rois=bboxes,
abc_init=tilt_abc,
img=img_in_photons, # NOTE this is now a multi panel image
SimData_instance=SIM,
plot_images=args.plot,
plot_residuals=args.residual,
ucell_manager=ucell_man)
RUC.panel_ids = panel_ids
RUC.multi_panel = True
RUC.split_evaluation = args.split
RUC.trad_conv = True
RUC.refine_detdist = False
RUC.refine_background_planes = False
RUC.refine_Umatrix = True
RUC.refine_Bmatrix = True
RUC.refine_ncells = True
RUC.use_curvatures = False # args.curvatures
RUC.calc_curvatures = True #args.curvatures
RUC.refine_crystal_scale = True
RUC.refine_gain_fac = False
RUC.plot_stride = args.stride
RUC.poisson_only = False
RUC.trad_conv_eps = 5e-3 # NOTE this is for single panel model
RUC.max_calls = 300
RUC.verbose = False
RUC.use_rot_priors = True
RUC.use_ucell_priors = True
if args.verbose:
if rank == 0: # only show refinement stats for rank 0
RUC.verbose = True
RUC.run()
if RUC.hit_break_to_use_curvatures:
RUC.use_curvatures = True
RUC.run(setup=False)
except AssertionError as err:
print(
"Rank %d, filename %s Hit assertion error during refinement: %s"
% (rank, data.filename, err))
continue
angle, ax = RUC.get_correction_misset(as_axis_angle_deg=True)
if args.startwithtruth:
C = Crystal(a_tru, b_tru, c_tru, "P43212")
C.rotate_around_origin(ax, angle)
C.set_B(RUC.get_refined_Bmatrix())
a_ref, _, c_ref, _, _, _ = C.get_unit_cell().parameters()
# compute missorientation with ground truth model
try:
angular_offset = compare_with_ground_truth(a_tru,
b_tru,
c_tru, [C],
symbol="P43212")[0]
print(
"Rank %d, filename=%s, ang=%f, init_ang=%f, a=%f, init_a=%f, c=%f, init_c=%f"
% (rank, data.filename, angular_offset,
angular_offset_init, a_ref, a_init, c_ref, c_init))
except Exception as err:
print("Rank %d, filename=%s, error %s" %
(rank, data.filename, err))
# free the memory from diffBragg instance
RUC.S.D.free_all()
del img # not sure if needed here..
del img_in_photons
if args.testmode:
exit()
# peak at the memory usage of this rank
mem = getrusage(RUSAGE_SELF).ru_maxrss # peak mem usage in KB
mem = mem / 1e6 # convert to GB
if rank == 0:
print "RANK 0: %.2g total pixels in %d/%d bboxes (file %d / %d); MemUsg=%2.2g GB" \
% (Ntot, len(bboxes), n_bboxes_total, img_num+1, len(my_shots), mem)
# TODO: accumulate all pixels
#self.all_bbox_pixels += data_boxes
for h in my_open_files.values():
h.close()
print("Rank %d; all subimages loaded!" % rank)
def test2():
"""Test on simulated data"""
# Get models for reflection prediction
import dials.test.algorithms.refinement.setup_geometry as setup_geometry
from libtbx.phil import parse
overrides = """geometry.parameters.crystal.a.length.value = 77
geometry.parameters.crystal.b.length.value = 77
geometry.parameters.crystal.c.length.value = 37"""
master_phil = parse(
"""
include scope dials.test.algorithms.refinement.geometry_phil
""",
process_includes=True,
)
from dxtbx.model import Crystal
models = setup_geometry.Extract(master_phil)
crystal = Crystal(
real_space_a=(2.62783398111729, -63.387215823567125,
-45.751375737456975),
real_space_b=(15.246640559660356, -44.48254330406616,
62.50501032727026),
real_space_c=(-76.67246874451074, -11.01804131886244,
10.861322446352226),
space_group_symbol="I 2 3",
)
detector = models.detector
goniometer = models.goniometer
beam = models.beam
# Build a mock scan for a 180 degree sweep
from dxtbx.model import ScanFactory
sf = ScanFactory()
scan = sf.make_scan(
image_range=(1, 1800),
exposure_times=0.1,
oscillation=(0, 0.1),
epochs=range(1800),
deg=True,
)
# Build an experiment list
from dxtbx.model.experiment_list import ExperimentList, Experiment
experiments = ExperimentList()
experiments.append(
Experiment(
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
crystal=crystal,
imageset=None,
))
# Generate all indices in a 1.5 Angstrom sphere
from dials.algorithms.spot_prediction import IndexGenerator
from cctbx.sgtbx import space_group, space_group_symbols
resolution = 1.5
index_generator = IndexGenerator(
crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution,
)
indices = index_generator.to_array()
# Predict rays within the sweep range
from dials.algorithms.refinement.prediction import ScansRayPredictor
sweep_range = scan.get_oscillation_range(deg=False)
ray_predictor = ScansRayPredictor(experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
from dials.algorithms.spot_prediction import ray_intersection
intersects = ray_intersection(detector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Make a reflection predictor and re-predict for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
from dials.algorithms.refinement.prediction import ExperimentsPredictor
ref_predictor = ExperimentsPredictor(experiments)
obs_refs["id"] = flex.int(len(obs_refs), 0)
obs_refs = ref_predictor(obs_refs)
# Copy 'observed' centroids from the predicted ones, applying sinusoidal
# offsets
obs_x, obs_y, obs_z = obs_refs["xyzcal.mm"].parts()
# obs_z is in range (0, pi). Calculate offsets for phi at twice that
# frequency
im_width = scan.get_oscillation(deg=False)[1]
z_off = flex.sin(2 * obs_z) * im_width
obs_z += z_off
# Calculate offsets for x
pixel_size = detector[0].get_pixel_size()
x_off = flex.sin(20 * obs_z) * pixel_size[0]
# Calculate offsets for y with a phase-shifted sine wave
from math import pi
y_off = flex.sin(4 * obs_z + pi / 6) * pixel_size[1]
# Incorporate the offsets into the 'observed' centroids
obs_z += z_off
obs_x += x_off
obs_y += y_off
obs_refs["xyzobs.mm.value"] = flex.vec3_double(obs_x, obs_y, obs_z)
# Now do centroid analysis of the residuals
results = CentroidAnalyser(obs_refs, debug=True)()
# FIXME this test shows that the suggested interval width heuristic is not
# yet robust. This simulation function seems a useful direction to proceed
# in though
raise RuntimeError("test2 failed")
print("OK")
return
def apply_symmetry(self, crystal_model):
"""Apply symmetry constraints to a crystal model.
Returns the crystal model (with symmetry constraints applied) in the same
setting as provided as input. The cb_op returned by the method is
that necessary to transform that model to the user-provided
target symmetry.
Args:
crystal_model (dxtbx.model.Crystal): The input crystal model to which to
apply symmetry constraints.
Returns: (dxtbx.model.Crystal, cctbx.sgtbx.change_of_basis_op):
The crystal model with symmetry constraints applied, and the change_of_basis_op
that transforms the returned model to the user-specified target symmetry.
"""
if not (
self.target_symmetry_primitive
and self.target_symmetry_primitive.space_group()
):
return crystal, sgtbx.change_of_basis_op()
target_space_group = self.target_symmetry_primitive.space_group()
A = crystal_model.get_A()
max_delta = self._max_delta
items = iotbx_converter(crystal_model.get_unit_cell(), max_delta=max_delta)
target_sg_ref = target_space_group.info().reference_setting().group()
best_angular_difference = 1e8
best_subgroup = None
for item in items:
if bravais_lattice(group=target_sg_ref) != item["bravais"]:
continue
if item["max_angular_difference"] < best_angular_difference:
best_angular_difference = item["max_angular_difference"]
best_subgroup = item
if best_subgroup is None:
return None, None
cb_op_inp_best = best_subgroup["cb_op_inp_best"]
best_subsym = best_subgroup["best_subsym"]
ref_subsym = best_subgroup["ref_subsym"]
cb_op_ref_best = ref_subsym.change_of_basis_op_to_best_cell()
cb_op_best_ref = cb_op_ref_best.inverse()
cb_op_inp_ref = cb_op_best_ref * cb_op_inp_best
cb_op_ref_inp = cb_op_inp_ref.inverse()
orient = crystal_orientation(A, True)
orient_ref = orient.change_basis(
scitbx.matrix.sqr((cb_op_inp_ref).c().as_double_array()[0:9]).transpose()
)
constrain_orient = orient_ref.constrain(best_subgroup["system"])
direct_matrix = constrain_orient.direct_matrix()
a = scitbx.matrix.col(direct_matrix[:3])
b = scitbx.matrix.col(direct_matrix[3:6])
c = scitbx.matrix.col(direct_matrix[6:9])
model = Crystal(a, b, c, space_group=target_sg_ref)
assert target_sg_ref.is_compatible_unit_cell(model.get_unit_cell())
model = model.change_basis(cb_op_ref_inp)
if self.cb_op_inp_best is not None:
# Then the unit cell has been provided: this is the cb_op to map to the
# user-provided input unit cell
return model, self.cb_op_inp_best.inverse() * cb_op_inp_best
if not self.cb_op_ref_inp.is_identity_op():
if self.target_symmetry_inp.space_group() == best_subsym.space_group():
# Handle where e.g. the user has requested I2 instead of the reference C2
return model, cb_op_inp_best
# The user has specified a setting that is not the reference setting
return model, self.cb_op_ref_inp * cb_op_inp_ref
# Default to reference setting
# This change of basis op will ensure that we get the best beta angle without
# changing the centring (e.g. from C2 to I2)
cb_op_ref_best = ref_subsym.change_of_basis_op_to_best_cell(
best_monoclinic_beta=False
)
return model, cb_op_ref_best * cb_op_inp_ref
def test():
import random
import textwrap
from cctbx.uctbx import unit_cell
from libtbx.test_utils import approx_equal
def random_direction_close_to(vector):
return vector.rotate_around_origin(
matrix.col((random.random(), random.random(),
random.random())).normalize(),
random.gauss(0, 1.0),
deg=True,
)
# make a random P1 crystal and parameterise it
a = random.uniform(10, 50) * random_direction_close_to(
matrix.col((1, 0, 0)))
b = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 1, 0)))
c = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 0, 1)))
xl = Crystal(a, b, c, space_group_symbol="P 1")
xl_op = CrystalOrientationParameterisation(xl)
xl_ucp = CrystalUnitCellParameterisation(xl)
null_mat = matrix.sqr((0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
# compare analytical and finite difference derivatives
an_ds_dp = xl_op.get_ds_dp()
fd_ds_dp = get_fd_gradients(xl_op, [1.0e-6 * pi / 180] * 3)
for e, f in zip(an_ds_dp, fd_ds_dp):
assert approx_equal((e - f), null_mat, eps=1.0e-6)
an_ds_dp = xl_ucp.get_ds_dp()
fd_ds_dp = get_fd_gradients(xl_ucp, [1.0e-7] * xl_ucp.num_free())
for e, f in zip(an_ds_dp, fd_ds_dp):
assert approx_equal((e - f), null_mat, eps=1.0e-6)
# random initial orientations with a random parameter shift at each
attempts = 100
for i in range(attempts):
# make a random P1 crystal and parameterise it
a = random.uniform(10, 50) * random_direction_close_to(
matrix.col((1, 0, 0)))
b = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 1, 0)))
c = random.uniform(10, 50) * random_direction_close_to(
matrix.col((0, 0, 1)))
xl = Crystal(a, b, c, space_group_symbol="P 1")
xl_op = CrystalOrientationParameterisation(xl)
xl_uc = CrystalUnitCellParameterisation(xl)
# apply a random parameter shift to the orientation
p_vals = xl_op.get_param_vals()
p_vals = random_param_shift(
p_vals, [1000 * pi / 9, 1000 * pi / 9, 1000 * pi / 9])
xl_op.set_param_vals(p_vals)
# compare analytical and finite difference derivatives
xl_op_an_ds_dp = xl_op.get_ds_dp()
xl_op_fd_ds_dp = get_fd_gradients(xl_op, [1.0e-5 * pi / 180] * 3)
# apply a random parameter shift to the unit cell. We have to
# do this in a way that is respectful to metrical constraints,
# so don't modify the parameters directly; modify the cell
# constants and extract the new parameters
cell_params = xl.get_unit_cell().parameters()
cell_params = random_param_shift(cell_params, [1.0] * 6)
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = S.forward_independent_parameters()
xl_uc.set_param_vals(X)
xl_uc_an_ds_dp = xl_ucp.get_ds_dp()
# now doing finite differences about each parameter in turn
xl_uc_fd_ds_dp = get_fd_gradients(xl_ucp, [1.0e-7] * xl_ucp.num_free())
for j in range(3):
assert approx_equal((xl_op_fd_ds_dp[j] - xl_op_an_ds_dp[j]),
null_mat,
eps=1.0e-6), textwrap.dedent("""\
Failure in try {i}
failure for parameter number {j}
of the orientation parameterisation
with fd_ds_dp =
{fd}
and an_ds_dp =
{an}
so that difference fd_ds_dp - an_ds_dp =
{diff}
""").format(
i=i,
j=j,
fd=xl_op_fd_ds_dp[j],
an=xl_op_an_ds_dp[j],
diff=xl_op_fd_ds_dp[j] - xl_op_an_ds_dp[j],
)
for j in range(xl_ucp.num_free()):
assert approx_equal((xl_uc_fd_ds_dp[j] - xl_uc_an_ds_dp[j]),
null_mat,
eps=1.0e-6), textwrap.dedent("""\
Failure in try {i}
failure for parameter number {j}
of the unit cell parameterisation
with fd_ds_dp =
{fd}
and an_ds_dp =
{an}
so that difference fd_ds_dp - an_ds_dp =
{diff}
""").format(
i=i,
j=j,
fd=xl_uc_fd_ds_dp[j],
an=xl_uc_an_ds_dp[j],
diff=xl_uc_fd_ds_dp[j] - xl_uc_an_ds_dp[j],
)
# apply a random parameter shift to the orientation
p_vals = xl_op.get_param_vals()
p_vals = random_param_shift(
p_vals, [1000 * pi / 9, 1000 * pi / 9, 1000 * pi / 9])
xl_op.set_param_vals(p_vals)
# compare analytical and finite difference derivatives
xl_op_an_ds_dp = xl_op.get_ds_dp()
xl_op_fd_ds_dp = get_fd_gradients(xl_op, [1.e-5 * pi / 180] * 3)
# apply a random parameter shift to the unit cell. We have to
# do this in a way that is respectful to metrical constraints,
# so don't modify the parameters directly; modify the cell
# constants and extract the new parameters
cell_params = xl.get_unit_cell().parameters()
cell_params = random_param_shift(cell_params, [1.] * 6)
new_uc = unit_cell(cell_params)
newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose()
S = symmetrize_reduce_enlarge(xl.get_space_group())
S.set_orientation(orientation=newB)
X = S.forward_independent_parameters()
xl_uc.set_param_vals(X)
xl_uc_an_ds_dp = xl_ucp.get_ds_dp()
# now doing finite differences about each parameter in turn
xl_uc_fd_ds_dp = get_fd_gradients(xl_ucp, [1.e-7] * xl_ucp.num_free())
for j in range(3):
try:
def load(self):
# some parameters
# NOTE: for reference, inside each h5 file there is
# [u'Amatrices', u'Hi', u'bboxes', u'h5_path']
# get the total number of shots using worker 0
if rank == 0:
self.time_load_start = time.time()
print("I am root. I am calculating total number of shots")
h5s = [h5py_File(f, "r") for f in self.fnames]
Nshots_per_file = [h["h5_path"].shape[0] for h in h5s]
Nshots_tot = sum(Nshots_per_file)
print("I am root. Total number of shots is %d" % Nshots_tot)
print("I am root. I will divide shots amongst workers.")
shot_tuples = []
roi_per = []
for i_f, fname in enumerate(self.fnames):
fidx_shotidx = [(i_f, i_shot) for i_shot in range(Nshots_per_file[i_f])]
shot_tuples += fidx_shotidx
# store the number of usable roi per shot in order to divide shots amongst ranks equally
roi_per += [sum(h5s[i_f]["bboxes"]["%s%d" % (args.keeperstag, i_shot)][()])
for i_shot in range(Nshots_per_file[i_f])]
from numpy import array_split
from numpy.random import permutation
print ("I am root. Number of uniques = %d" % len(set(shot_tuples)))
# divide the array into chunks of roughly equal sum (total number of ROI)
if args.partition and args.restartfile is None and args.xinitfile is None:
diff = np.inf
roi_per = np.array(roi_per)
tstart = time.time()
best_order = range(len(roi_per))
print("Partitioning for better load balancing across ranks.. ")
while 1:
order = permutation(len(roi_per))
res = [sum(a) for a in np.array_split(roi_per[order], size)]
new_diff = max(res) - min(res)
t_elapsed = time.time() - tstart
t_remain = args.partitiontime - t_elapsed
if new_diff < diff:
diff = new_diff
best_order = order.copy()
print("Best diff=%d, Parition time remaining: %.3f seconds" % (diff, t_remain))
if t_elapsed > args.partitiontime:
break
shot_tuples = [shot_tuples[i] for i in best_order]
elif args.partition and args.restartfile is not None:
print ("Warning: skipping partitioning time to use shot mapping as laid out in restart file dir")
else:
print ("Proceeding without partitioning")
# optional to divide into a sub group
shot_tuples = array_split(shot_tuples, args.ngroups)[args.groupId]
shots_for_rank = array_split(shot_tuples, size)
import os # FIXME, I thought I was imported already!
if args.outdir is not None: # save for a fast restart (shot order is important!)
np.save(os.path.join(args.outdir, "shots_for_rank"), shots_for_rank)
if args.restartfile is not None:
# the directory containing the restart file should have a shots for rank file
dirname = os.path.dirname(args.restartfile)
print ("Loading shot mapping from dir %s" % dirname)
shots_for_rank = np.load(os.path.join(dirname, "shots_for_rank.npy"))
# propagate the shots for rank file...
if args.outdir is not None:
np.save(os.path.join(args.outdir, "shots_for_rank"), shots_for_rank)
if args.xinitfile is not None:
# the directory containing the restart file should have a shots for rank file
dirname = os.path.dirname(args.xinitfile)
print ("Loading shot mapping from dir %s" % dirname)
shots_for_rank = np.load(os.path.join(dirname, "shots_for_rank.npy"))
# propagate the shots for rank file...
if args.outdir is not None:
np.save(os.path.join(args.outdir, "shots_for_rank"), shots_for_rank)
# close the open h5s..
for h in h5s:
h.close()
else:
Nshots_tot = None
shots_for_rank = None
h5s = None
# Nshots_tot = comm.bcast( Nshots_tot, root=0)
if has_mpi:
shots_for_rank = comm.bcast(shots_for_rank, root=0)
# h5s = comm.bcast( h5s, root=0) # pull in the open hdf5 files
my_shots = shots_for_rank[rank]
if self.Nload is not None:
start = 0
if args.loadstart is not None:
start = args.loadstart
my_shots = my_shots[start: start + self.Nload]
print("Rank %d: I will load %d shots, first shot: %s, last shot: %s"
% (comm.rank, len(my_shots), my_shots[0], my_shots[-1]))
# open the unique filenames for this rank
# TODO: check max allowed pointers to open hdf5 file
import h5py
my_unique_fids = set([fidx for fidx, _ in my_shots])
self.my_open_files = {fidx: h5py_File(self.fnames[fidx], "r") for fidx in my_unique_fids}
# for fidx in my_unique_fids:
# fpath = self.fnames[fidx]
# if args.imgdirname is not None:
# fpath = fpath.split("/kaladin/")[1]
# fpath = os.path.join(args.imgdirname, fpath)
# self.my_open_files[fidx] = h5py.File(fpath, "r")
Ntot = 0
for img_num, (fname_idx, shot_idx) in enumerate(my_shots):
h = self.my_open_files[fname_idx]
# load the dxtbx image data directly:
npz_path = h["h5_path"][shot_idx]
if args.imgdirname is not None:
import os
npz_path = npz_path.split("/kaladin/")[1]
npz_path = os.path.join(args.imgdirname, npz_path)
if args.noiseless:
noiseless_path = npz_path.replace(".npz", ".noiseless.npz")
img_handle = numpy_load(noiseless_path)
else:
img_handle = numpy_load(npz_path)
img = img_handle["img"]
if len(img.shape) == 2: # if single panel
img = array([img])
B = beam_from_dict(img_handle["beam"][()])
log_init_crystal_scale = 0 # default
if args.usepreoptscale:
log_init_crystal_scale = h["crystal_scale_%s" % args.preopttag][shot_idx]
# get the indexed crystal Amatrix
Amat = h["Amatrices"][shot_idx]
if args.usepreoptAmat:
Amat = h["Amatrices_%s" % args.preopttag][shot_idx]
amat_elems = list(sqr(Amat).inverse().elems)
# real space basis vectors:
a_real = amat_elems[:3]
b_real = amat_elems[3:6]
c_real = amat_elems[6:]
# dxtbx indexed crystal model
C = Crystal(a_real, b_real, c_real, "P43212")
# change basis here ? Or maybe just average a/b
a, b, c, _, _, _ = C.get_unit_cell().parameters()
a_init = .5 * (a + b)
c_init = c
# shoe boxes where we expect spots
bbox_dset = h["bboxes"]["shot%d" % shot_idx]
n_bboxes_total = bbox_dset.shape[0]
# is the shoe box within the resolution ring and does it have significant SNR (see filter_bboxes.py)
is_a_keeper = h["bboxes"]["%s%d" % (args.keeperstag, shot_idx)][()]
# tilt plane to the background pixels in the shoe boxes
tilt_abc_dset = h["tilt_abc"]["shot%d" % shot_idx]
# miller indices (not yet reduced by symm equivs)
Hi_dset = h["Hi"]["shot%d" % shot_idx]
try:
panel_ids_dset = h["panel_ids"]["shot%d" % shot_idx]
has_panels = True
except KeyError:
has_panels = False
# apply the filters:
bboxes = [bbox_dset[i_bb] for i_bb in range(n_bboxes_total) if is_a_keeper[i_bb]]
tilt_abc = [tilt_abc_dset[i_bb] for i_bb in range(n_bboxes_total) if is_a_keeper[i_bb]]
Hi = [tuple(Hi_dset[i_bb]) for i_bb in range(n_bboxes_total) if is_a_keeper[i_bb]]
proc_file_idx = [i_bb for i_bb in range(n_bboxes_total) if is_a_keeper[i_bb]]
if has_panels:
panel_ids = [panel_ids_dset[i_bb] for i_bb in range(n_bboxes_total) if is_a_keeper[i_bb]]
else:
panel_ids = [0] * len(tilt_abc)
# how many pixels do we have
tot_pix = [(j2 - j1) * (i2 - i1) for i1, i2, j1, j2 in bboxes]
Ntot += sum(tot_pix)
# load some ground truth data from the simulation dumps (e.g. spectrum)
# h5_fname = h["h5_path"][shot_idx].replace(".npz", "")
h5_fname = npz_path.replace(".npz", "")
if args.character is not None:
h5_fname = h5_fname.replace("rock", args.character)
if args.testmode2:
h5_fname = npz_path.split(".npz")[0]
data = h5py_File(h5_fname, "r")
xtal_scale_truth = data["spot_scale"][()]
tru = sqr(data["crystalA"][()]).inverse().elems
a_tru = tru[:3]
b_tru = tru[3:6]
c_tru = tru[6:]
C_tru = Crystal(a_tru, b_tru, c_tru, "P43212")
fluxes = data["spectrum"][()]
es = data["exposure_s"][()]
# comm.Barrier()
# exit()
fluxes *= es # multiply by the exposure time
# TODO: wavelens should come from the imageset file itself
if "wavelengths" in data.keys():
wavelens = data["wavelengths"][()]
else:
raise KeyError("Wavelengths missing from hdf5 data")
#from cxid9114.parameters import WAVELEN_HIGH
#wavelens = [WAVELEN_HIGH]
spectrum = zip(wavelens, fluxes)
# dont simulate when there are no photons!
spectrum = [(wave, flux) for wave, flux in spectrum if flux > self.flux_min]
if args.forcemono:
spectrum = [(B.get_wavelength(), sum(fluxes))]
# make a unit cell manager that the refiner will use to track the B-matrix
aa, _, cc, _, _, _ = C_tru.get_unit_cell().parameters()
ucell_man = TetragonalManager(a=a_init, c=c_init)
# create the sim_data instance that the refiner will use to run diffBragg
# create a nanoBragg crystal
self.Fhkl_obs = open_flex(args.Fobs).as_amplitude_array()
self.Fhkl_ref = None
if args.Fref is not None:
self.Fhkl_ref = open_flex(
args.Fref).as_amplitude_array() # this reference miller array is used to track CC and R-factor
if img_num == 0: # only initialize the simulator after loading the first image
self.initialize_simulator(C, B, spectrum, self.Fhkl_obs)
# map the miller array to ASU
Hi_asu = map_hkl_list(Hi, self.anomalous_flag, self.symbol)
# copy the image as photons (NOTE: Dont forget to ditch its references!)
img_in_photons = (img / args.gainval).astype('float32')
# Here, takeout from the image only whats necessary to perform refinement
# first filter the spot rois so they dont occur exactly at the boundary of the image (inclusive range in nB)
assert len(img_in_photons.shape) == 3 # sanity
nslow, nfast = img_in_photons[0].shape
bboxes = array(bboxes)
for i_bbox, (_, x2, _, y2) in enumerate(bboxes):
if x2 == nfast:
bboxes[i_bbox][1] = x2 - 1 # update roi_xmax
if y2 == nslow:
bboxes[i_bbox][3] = y2 - 1 # update roi_ymax
# now cache the roi in nanoBragg format ((x1,x2), (y1,y1))
# and also cache the pixels and the coordinates
nanoBragg_rois = [] # special nanoBragg format
xrel, yrel, roi_img = [], [], []
for i_roi, (x1, x2, y1, y2) in enumerate(bboxes):
nanoBragg_rois.append(((x1, x2), (y1, y2)))
yr, xr = np_indices((y2 - y1 + 1, x2 - x1 + 1))
xrel.append(xr)
yrel.append(yr)
pid = panel_ids[i_roi]
roi_img.append(img_in_photons[pid, y1:y2 + 1, x1:x2 + 1])
# make sure to clear that damn memory
img = None
img_in_photons = None
del img # not sure if needed here..
del img_in_photons
# peak at the memory usage of this rank
# mem = getrusage(RUSAGE_SELF).ru_maxrss # peak mem usage in KB
# mem = mem / 1e6 # convert to GB
mem = self._usage()
# print "RANK %d: %.2g total pixels in %d/%d bboxes (file %d / %d); MemUsg=%2.2g GB" \
# % (rank, Ntot, len(bboxes), n_bboxes_total, img_num +1, len(my_shots), mem)
self.all_pix += Ntot
# accumulate per-shot information
self.global_image_id[img_num] = None # TODO
self.all_spot_roi[img_num] = bboxes
self.all_abc_inits[img_num] = tilt_abc
self.all_panel_ids[img_num] = panel_ids
self.all_ucell_mans[img_num] = ucell_man
self.all_spectra[img_num] = spectrum
self.all_crystal_models[img_num] = C
self.log_of_init_crystal_scales[
img_num] = log_init_crystal_scale # these should be the log of the initial crystal scale
self.all_crystal_scales[img_num] = xtal_scale_truth
self.all_crystal_GT[img_num] = C_tru
self.all_xrel[img_num] = xrel
self.all_yrel[img_num] = yrel
self.all_nanoBragg_rois[img_num] = nanoBragg_rois
self.all_roi_imgs[img_num] = roi_img
self.all_fnames[img_num] = npz_path
self.all_proc_fnames[img_num] = h.filename
self.all_Hi[img_num] = Hi
self.all_Hi_asu[img_num] = Hi_asu
self.all_proc_idx[img_num] = proc_file_idx
self.all_shot_idx[img_num] = shot_idx # this is the index of the shot in the process*h5 file
# NOTE all originZ for each panel are the same in the simulated data.. Not necessarily true for real data
shot_originZ = self.SIM.detector[0].get_origin()[2]
self.shot_originZ_init[img_num] = shot_originZ
print(img_num)
for h in self.my_open_files.values():
h.close()