def exercise_map_to_asu(sg_symbol):
sg_type = sgtbx.space_group_type(sg_symbol)
index_abs_range = (4,4,4)
for anomalous_flag in (False,True):
m = miller.index_generator(
sg_type, anomalous_flag, index_abs_range).to_array()
a = flex.double()
p = flex.double()
c = flex.hendrickson_lattman()
for i in range(m.size()):
a.append(random.random())
p.append(random.random() * 2)
c.append([random.random() for j in range(4)])
f = flex.polar(a, p)
p = [p, p*(180/math.pi)]
m_random = flex.miller_index()
p_random = [flex.double(), flex.double()]
c_random = flex.hendrickson_lattman()
f_random = flex.complex_double()
for i,h_asym in enumerate(m):
h_eq = miller.sym_equiv_indices(sg_type.group(), h_asym)
i_eq = random.randrange(h_eq.multiplicity(anomalous_flag))
h_i = h_eq(i_eq)
m_random.append(h_i.h())
for deg in (False,True):
p_random[deg].append(h_i.phase_eq(p[deg][i], deg))
f_random.append(h_i.complex_eq(f[i]))
c_random.append(h_i.hendrickson_lattman_eq(c[i]))
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, f_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,f_asym in enumerate(f):
assert abs(f_asym - f_random[i]) < 1.e-6
m_random_copy = m_random.deep_copy()
a_random = a.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, a_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,a_asym in enumerate(a):
assert a_asym == a_random[i]
for deg in (False,True):
m_random_copy = m_random.deep_copy()
miller.map_to_asu(
sg_type, anomalous_flag, m_random_copy, p_random[deg], deg)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,p_asym in enumerate(p[deg]):
assert scitbx.math.phase_error(p_asym, p_random[deg][i], deg) < 1.e-5
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, c_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,c_asym in enumerate(c):
for j in range(4):
assert abs(c_asym[j] - c_random[i][j]) < 1.e-5
def get_hkl_xyz_isigi(mtz_file):
m = mtz.object(mtz_file)
r = get_phi_range(m)
sg = m.space_group()
xdet_col = m.get_column('XDET')
ydet_col = m.get_column('YDET')
rot_col = m.get_column('ROT')
i_col = m.get_column('I')
sigi_col = m.get_column('SIGI')
mi_s = m.extract_miller_indices()
map_to_asu(sg.type(), False, mi_s)
xdet_s = xdet_col.extract_values(not_a_number_substitute=0.0)
ydet_s = ydet_col.extract_values(not_a_number_substitute=0.0)
rot_s = rot_col.extract_values(not_a_number_substitute=0.0)
i_s = i_col.extract_values(not_a_number_substitute=0.0)
sigi_s = sigi_col.extract_values(not_a_number_substitute=0.0)
hkl_xyz_isigi = []
# N.B. swapping X, Y here
for j in range(mi_s.size()):
hkl_xyz_isigi.append(
((mi_s[j][0], mi_s[j][1], mi_s[j][2]),
(ydet_s[j], xdet_s[j], rot_s[j] / r), (i_s[j], sigi_s[j])))
return hkl_xyz_isigi
def get_hkl_xyz_isigi(mtz_file):
m = mtz.object(mtz_file)
r = get_phi_range(m)
sg = m.space_group()
xdet_col = m.get_column('XDET')
ydet_col = m.get_column('YDET')
rot_col = m.get_column('ROT')
i_col = m.get_column('I')
sigi_col = m.get_column('SIGI')
mi_s = m.extract_miller_indices()
map_to_asu(sg.type(), False, mi_s)
xdet_s = xdet_col.extract_values(not_a_number_substitute = 0.0)
ydet_s = ydet_col.extract_values(not_a_number_substitute = 0.0)
rot_s = rot_col.extract_values(not_a_number_substitute = 0.0)
i_s = i_col.extract_values(not_a_number_substitute = 0.0)
sigi_s = sigi_col.extract_values(not_a_number_substitute = 0.0)
hkl_xyz_isigi = []
# N.B. swapping X, Y here
for j in range(mi_s.size()):
hkl_xyz_isigi.append(((mi_s[j][0], mi_s[j][1], mi_s[j][2]),
(ydet_s[j], xdet_s[j], rot_s[j] / r),
(i_s[j], sigi_s[j])))
return hkl_xyz_isigi
def exercise_map_to_asu(sg_symbol):
sg_type = sgtbx.space_group_type(sg_symbol)
index_abs_range = (4,4,4)
for anomalous_flag in (False,True):
m = miller.index_generator(
sg_type, anomalous_flag, index_abs_range).to_array()
a = flex.double()
p = flex.double()
c = flex.hendrickson_lattman()
for i in xrange(m.size()):
a.append(random.random())
p.append(random.random() * 2)
c.append([random.random() for j in xrange(4)])
f = flex.polar(a, p)
p = [p, p*(180/math.pi)]
m_random = flex.miller_index()
p_random = [flex.double(), flex.double()]
c_random = flex.hendrickson_lattman()
f_random = flex.complex_double()
for i,h_asym in enumerate(m):
h_eq = miller.sym_equiv_indices(sg_type.group(), h_asym)
i_eq = random.randrange(h_eq.multiplicity(anomalous_flag))
h_i = h_eq(i_eq)
m_random.append(h_i.h())
for deg in (False,True):
p_random[deg].append(h_i.phase_eq(p[deg][i], deg))
f_random.append(h_i.complex_eq(f[i]))
c_random.append(h_i.hendrickson_lattman_eq(c[i]))
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, f_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,f_asym in enumerate(f):
assert abs(f_asym - f_random[i]) < 1.e-6
m_random_copy = m_random.deep_copy()
a_random = a.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, a_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,a_asym in enumerate(a):
assert a_asym == a_random[i]
for deg in (False,True):
m_random_copy = m_random.deep_copy()
miller.map_to_asu(
sg_type, anomalous_flag, m_random_copy, p_random[deg], deg)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,p_asym in enumerate(p[deg]):
assert scitbx.math.phase_error(p_asym, p_random[deg][i], deg) < 1.e-5
m_random_copy = m_random.deep_copy()
miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, c_random)
for i,h_asym in enumerate(m):
assert h_asym == m_random_copy[i]
for i,c_asym in enumerate(c):
for j in xrange(4):
assert abs(c_asym[j] - c_random[i][j]) < 1.e-5
def _compute_rij_wij(self, use_cache=True, use_super=False):
if use_super:
# for testing
return super()._compute_rij_wij(use_cache=use_cache)
def _compute_one_row(args):
"""
Call the compute_one_row method of CC, which is a compute_rij_wij_detail
instance. Args is a tuple that we unpack because easy_mp.parallel_map
can only pass a single argument.
"""
CC, i, NN = args
rij_row, rij_col, rij_data, wij_row, wij_col, wij_data = [
list(x) for x in CC.compute_one_row(self._lattices.size, i)
]
rij = sparse.coo_matrix((rij_data, (rij_row, rij_col)),
shape=(NN, NN))
wij = sparse.coo_matrix((wij_data, (wij_row, wij_col)),
shape=(NN, NN))
return rij, wij
n_lattices = self._lattices.size
n_sym_ops = len(self.sym_ops)
NN = n_lattices * n_sym_ops
lower_i = flex.int()
upper_i = flex.int()
for lidx in range(self._lattices.size):
LL, UU = self._lattice_lower_upper_index(lidx)
lower_i.append(int(LL))
if UU is None: UU = self._data.data().size()
upper_i.append(int(UU))
indices = {}
space_group_type = self._data.space_group().type()
from xfel.merging import compute_rij_wij_detail
CC = compute_rij_wij_detail(lower_i, upper_i, self._data.data(),
self._min_pairs)
for sym_op in self.sym_ops:
cb_op = sgtbx.change_of_basis_op(sym_op)
indices_reindexed = cb_op.apply(self._data.indices())
miller.map_to_asu(space_group_type, False, indices_reindexed)
indices[cb_op.as_xyz()] = indices_reindexed
CC.set_indices(cb_op, indices_reindexed)
assert self._nproc == 1
results = map(_compute_one_row,
[(CC, i, NN) for i in range(n_lattices)])
rij_matrix = None
wij_matrix = None
for (rij, wij) in results:
if rij_matrix is None:
rij_matrix = rij
wij_matrix = wij
else:
rij_matrix += rij
wij_matrix += wij
self.rij_matrix = rij_matrix.todense().astype(np.float64)
self.wij_matrix = wij_matrix.todense().astype(np.float64)
return self.rij_matrix, self.wij_matrix
def reindex(self, reindex_operation):
"""Reindex the reflections by the given reindexing operation."""
R = rt_mx(reindex_operation).inverse()
# first construct mapping table - native, anomalous
map_native = {}
hkls = flex.miller_index()
for hkl in self._merged_reflections:
Fhkl = R * hkl
Rhkl = nint(Fhkl[0]), nint(Fhkl[1]), nint(Fhkl[2])
hkls.append(Rhkl)
map_to_asu(self._mf.get_space_group().type(), False, hkls)
for j, hkl in enumerate(self._merged_reflections):
map_native[hkl] = hkls[j]
map_anomalous = {}
hkls = flex.miller_index()
for hkl in self._merged_reflections_anomalous:
Fhkl = R * hkl
Rhkl = nint(Fhkl[0]), nint(Fhkl[1]), nint(Fhkl[2])
hkls.append(Rhkl)
map_to_asu(self._mf.get_space_group().type(), True, hkls)
for j, hkl in enumerate(self._merged_reflections_anomalous):
map_anomalous[hkl] = hkls[j]
# then remap the actual measurements
merged_reflections = {}
for hkl in self._merged_reflections:
Rhkl = map_native[hkl]
merged_reflections[Rhkl] = self._merged_reflections[hkl]
self._merged_reflections = merged_reflections
merged_reflections = {}
for hkl in self._merged_reflections_anomalous:
Rhkl = map_anomalous[hkl]
merged_reflections[Rhkl] = self._merged_reflections_anomalous[hkl]
self._merged_reflections_anomalous = merged_reflections
unmerged_reflections = {}
for hkl in self._unmerged_reflections:
Rhkl = map_native[hkl]
unmerged_reflections[Rhkl] = self._unmerged_reflections[hkl]
self._unmerged_reflections = unmerged_reflections
def load_xds_or_dials(i, file_in, scout):
print "Reading %5d %s" % (i, file_in
if input_type == "xds_ascii" else file_in[1])
if input_type == "xds_ascii":
tmp = get_data_from_xac(params, file_in)
else:
tmp = get_data_from_dials(params, file_in)
miller.map_to_asu(sgtype, params.anomalous_flag, tmp.indices())
k, b = None, None
if scale_ref is not None:
k, b, cc = scale_data(tmp.indices(), tmp.data(), scale_ref,
params.scaling.parameter,
params.scaling.calc_cc)
scout.append("%s %.4e %.4e %.4f\n" %
(file_in if input_type == "xds_ascii" else file_in[1],
k, b, cc))
#tmp.iobs *= k
#tmp.sigma_iobs *= k
#if b == b: # nan if not calculated
# d_star_sq = tmp.symm.unit_cell().d_star_sq(tmp.indices)
# tmp.iobs *= flex.exp(-b*d_star_sq)
# tmp.sigma_iobs *= flex.exp(-b*d_star_sq)
return i, tmp, k, b
def reduce_reflections_to_asu(space_group_number, indices):
"""Reduce reflection indices to asymmetric unit."""
from cctbx.sgtbx import space_group, space_group_symbols
from cctbx.array_family import flex
from cctbx.miller import map_to_asu
sg = space_group(space_group_symbols(space_group_number).hall())
miller = flex.miller_index(indices)
map_to_asu(sg.type(), False, miller)
return [hkl for hkl in miller]
def reduce_reflections_to_asu(space_group_number, indices): '''Reduce reflection indices to asymmetric unit.''' from cctbx.sgtbx import space_group, space_group_symbols from cctbx.array_family import flex from cctbx.miller import map_to_asu sg = space_group(space_group_symbols(space_group_number).hall()) miller = flex.miller_index(indices) map_to_asu(sg.type(), False, miller) return [hkl for hkl in miller]
def exercise_change_basis_in_place():
from cctbx import sgtbx
space_group_info = sgtbx.space_group_info(symbol='P4')
unit_cell = space_group_info.any_compatible_unit_cell(volume=1000)
miller_set = crystal.symmetry(
unit_cell=unit_cell,
space_group_info=space_group_info).build_miller_set(
d_min=1, anomalous_flag=True)
miller_set = miller_set.expand_to_p1().customized_copy(
space_group_info=miller_set.space_group_info())
m = mtz.object()
m.set_title('This is a title')
m.set_space_group_info(miller_set.space_group_info())
x = m.add_crystal('XTAL', 'CCTBX', miller_set.unit_cell().parameters())
d = x.add_dataset('TEST', 1)
indices = miller_set.indices()
original_indices = indices.deep_copy()
# map the miller indices to one hemisphere (i.e. just I+)
miller.map_to_asu(m.space_group().type(), False, indices)
h, k, l = [i.iround() for i in indices.as_vec3_double().parts()]
m.adjust_column_array_sizes(len(h))
m.set_n_reflections(len(h))
# assign H, K, L
d.add_column('H', 'H').set_values(h.as_double().as_float())
d.add_column('K', 'H').set_values(k.as_double().as_float())
d.add_column('L', 'H').set_values(l.as_double().as_float())
d.add_column('M_ISYM', 'Y').set_values(flex.float(len(indices)))
b = m.add_batch()
b.set_cell(flex.float(unit_cell.parameters()))
b.set_umat(
flex.float(
(-0.9542511701583862, -0.1780465543270111, -0.2402169108390808,
-0.13100790977478027, 0.9711279273033142, -0.19936780631542206,
0.26877808570861816, -0.1587766408920288, -0.9500254392623901)))
m.change_basis_in_place(sgtbx.change_of_basis_op('-h,k,-l'))
assert approx_equal(
b.umat(),
(0.9542511701583862, 0.1780465543270111, 0.2402169108390808,
-0.13100790977478027, 0.9711279273033142, -0.19936780631542206,
-0.26877808570861816, 0.1587766408920288, 0.9500254392623901))
def compare(a, b, sgtype):
ind_a = a.indices()
ind_b = b.indices()
from cctbx.miller import map_to_asu
map_to_asu(sgtype, False, ind_a)
map_to_asu(sgtype, False, ind_b)
_a, _b = a.common_sets(b, assert_is_similar_symmetry=False)
if _a.size() < 5:
return 0.0, _a.size()
return cc(_a.data(), _b.data()), _a.size()
def read_xds_integrate(xds_integrate_file):
# N.B. in here need to reduce indices to asymmetric unit
sg_num = 0
r = 0.0
for record in open(xds_integrate_file):
if not '!' in record[:1]:
break
if 'SPACE_GROUP_NUMBER' in record:
sg_num = int(record.split()[-1])
if 'OSCILLATION_RANGE' in record:
r = float(record.split()[-1])
if not sg_num:
raise RuntimeError('spacegroup missing')
if not r:
raise RuntimeError('rotation missing')
sg = space_group(space_group_symbols(sg_num).hall())
observations = []
hkls = flex.miller_index()
xyzs = []
isigmas = []
for record in open(xds_integrate_file):
if '!' in record[:1]:
continue
values = record.split()
hkls.append([int(h) for h in values[:3]])
xyzs.append((float(values[5]), float(values[6]), float(values[7])))
isigmas.append([float(x) for x in values[3:5]])
map_to_asu(sg.type(), False, hkls)
for hkl, xyz, isigma in zip(hkls, xyzs, isigmas):
observations.append((hkl, xyz, isigma))
return observations
def read_xds_integrate(xds_integrate_file):
# N.B. in here need to reduce indices to asymmetric unit
sg_num = 0
r = 0.0
for record in open(xds_integrate_file):
if not '!' in record[:1]:
break
if 'SPACE_GROUP_NUMBER' in record:
sg_num = int(record.split()[-1])
if 'OSCILLATION_RANGE' in record:
r = float(record.split()[-1])
if not sg_num:
raise RuntimeError, 'spacegroup missing'
if not r:
raise RuntimeError, 'rotation missing'
sg = space_group(space_group_symbols(sg_num).hall())
observations = []
hkls = flex.miller_index()
xyzs = []
isigmas = []
for record in open(xds_integrate_file):
if '!' in record[:1]:
continue
values = record.split()
hkls.append(map(int, values[:3]))
xyzs.append((float(values[5]), float(values[6]), float(values[7])))
isigmas.append(map(float, values[3:5]))
map_to_asu(sg.type(), False, hkls)
for hkl, xyz, isigma in zip(hkls, xyzs, isigmas):
observations.append((hkl, xyz, isigma))
return observations
def add_asu_miller_indices_column(self, reflections):
'''Add a "symmetry-reduced hkl" column to the reflection table'''
if reflections.size() == 0:
return
import copy
# Build target symmetry. The exact experiment unit cell values don't matter for converting HKLs to asu HKLs.
target_unit_cell = self.params.scaling.unit_cell
target_space_group_info = self.params.scaling.space_group
target_symmetry = symmetry(unit_cell=target_unit_cell, space_group_info=target_space_group_info)
target_space_group = target_symmetry.space_group()
# generate and add an asu hkl column
if 'miller_index_asymmetric' not in reflections:
reflections['miller_index_asymmetric'] = copy.deepcopy(reflections['miller_index'])
miller.map_to_asu(target_space_group.type(),
not self.params.merging.merge_anomalous,
reflections['miller_index_asymmetric'])
def add_asu_miller_indices_column(self, experiments, reflections):
'''Add a "symmetry-reduced hkl" column to the reflection table'''
if reflections.size() == 0:
return
from cctbx import miller
from cctbx.crystal import symmetry
import copy
# build target space group
target_unit_cell = self.params.scaling.unit_cell
target_space_group_info = self.params.scaling.space_group
target_symmetry = symmetry(unit_cell=target_unit_cell, space_group_info=target_space_group_info)
target_space_group = target_symmetry.space_group()
# generate and add an asu hkl column
reflections['miller_index_asymmetric'] = copy.deepcopy(reflections['miller_index'])
miller.map_to_asu(target_space_group.type(),
not self.params.merging.merge_anomalous,
reflections['miller_index_asymmetric'])
def load_xds(i, xac, scout):
print "Reading %5d %s" %(i, xac)
tmp = get_data_from_xac(params, xac)
miller.map_to_asu(sgtype,
params.anomalous_flag,
tmp.indices)
k, b = None, None
if scale_ref is not None:
k, b, cc = scale_data(tmp.indices, tmp.iobs, scale_ref, params.scaling.parameter, params.scaling.calc_cc)
scout.append("%s %.4e %.4e %.4f\n" % (xac, k, b, cc))
#tmp.iobs *= k
#tmp.sigma_iobs *= k
#if b == b: # nan if not calculated
# d_star_sq = tmp.symm.unit_cell().d_star_sq(tmp.indices)
# tmp.iobs *= flex.exp(-b*d_star_sq)
# tmp.sigma_iobs *= flex.exp(-b*d_star_sq)
return i, tmp, k, b
def reindex_refl_by_coset(refl,data,symms,uuids,co,anomalous_flag = False, verbose=True):
if verbose:
cache_miller = refl["miller_index"].deep_copy()
cache_asu = refl["miller_index_asymmetric"].deep_copy()
for icoset,partition in enumerate(co.partitions):
if icoset==0: continue # no change of basis
cb_op = change_of_basis_op(partition[0])
mi_new = cb_op.apply(refl["miller_index"])
mi_asu_new = mi_new.deep_copy()
miller.map_to_asu(symms[0].space_group().info().type(), anomalous_flag, mi_asu_new)
# now select only those expts assigned to that coset
good_refls = flex.bool(len(refl), False)
all_expt_id = list(data["experiment"])
all_coset = list(data["coset"]) # would like to understand how to use pandas rather than Python list
for iexpt in range(len(symms)):
iexpt_id = uuids[iexpt]
this_coset = all_coset[ all_expt_id.index(iexpt_id) ]
if this_coset == icoset:
good_refls |= refl["exp_id"] == iexpt_id
re_millers = mi_new.select(good_refls)
re_asu = mi_asu_new.select(good_refls)
refl["miller_index"].set_selected(good_refls, re_millers)
refl["miller_index_asymmetric"].set_selected(good_refls, re_asu)
if verbose:
for x in range(len(refl)):
print ("%3d"%x,refl["exp_id"][x],
all_coset[ all_expt_id.index(refl["exp_id"][x]) ],
"%10s"%(change_of_basis_op(co.partitions[ all_coset[ all_expt_id.index(refl["exp_id"][x]) ] ][0]).as_hkl()),
"(%4d%4d%4d)"%(cache_miller[x]), "(%4d%4d%4d)"%(cache_asu[x]),
"(%4d%4d%4d)"%(refl["miller_index"][x]), "(%4d%4d%4d)"%(refl["miller_index_asymmetric"][x]))
return refl
def map_hkl_list(Hi_lst, anomalous_flag=True, symbol="P43212"):
sg_type = sgtbx.space_group_info(symbol=symbol).type()
Hi_flex = flex.miller_index(tuple(map(tuple, Hi_lst)))
miller.map_to_asu(sg_type, anomalous_flag, Hi_flex)
return list(Hi_flex)
fast = flex.miller_index()
data = flex.double()
for r in reflections:
# only copy across the ones which should be present
if sg.is_sys_absent([h, k, l]):
continue
o, h, k, l = r
data.append(o)
fast.append([h, k, l])
map_to_asu(sg.type(), False, fast, data)
# now unpack to original data structures
for j in range(data.size()):
d = data[j]
hkl = fast[j]
new_reflections.append((d, hkl[0], hkl[1], hkl[2]))
# now do something with these symmetry reduced reflections...
outfile = '%s.sym' % os.path.split(hkl_file)[-1][:-4]
print 'Writing reflections to %s' % outfile
fout = open(outfile, 'w')
def export_xds_ascii(integrated_data,
experiment_list,
hklout,
summation=False,
include_partials=False,
keep_partials=False,
var_model=(1, 0)):
'''Export data from integrated_data corresponding to experiment_list to
an XDS_ASCII.HKL formatted text file.'''
from dials.array_family import flex
# for the moment assume (and assert) that we will convert data from exactly
# one lattice...
assert (len(experiment_list) == 1)
# select reflections that are assigned to an experiment (i.e. non-negative id)
integrated_data = integrated_data.select(integrated_data['id'] >= 0)
assert max(integrated_data['id']) == 0
if not summation:
assert ('intensity.prf.value' in integrated_data)
if 'intensity.prf.variance' in integrated_data:
selection = integrated_data.get_flags(integrated_data.flags.integrated,
all=True)
else:
selection = integrated_data.get_flags(
integrated_data.flags.integrated_sum)
integrated_data = integrated_data.select(selection)
selection = integrated_data['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
integrated_data.del_selected(selection)
logger.info('Removing %d reflections with negative variance' % \
selection.count(True))
if 'intensity.prf.variance' in integrated_data:
selection = integrated_data['intensity.prf.variance'] <= 0
if selection.count(True) > 0:
integrated_data.del_selected(selection)
logger.info('Removing %d profile reflections with negative variance' % \
selection.count(True))
if include_partials:
integrated_data = sum_partial_reflections(integrated_data)
integrated_data = scale_partial_reflections(integrated_data)
if 'partiality' in integrated_data:
selection = integrated_data['partiality'] < 0.99
if selection.count(True) > 0 and not keep_partials:
integrated_data.del_selected(selection)
logger.info('Removing %d incomplete reflections' % \
selection.count(True))
experiment = experiment_list[0]
# sort data before output
nref = len(integrated_data['miller_index'])
indices = flex.size_t_range(nref)
import copy
unique = copy.deepcopy(integrated_data['miller_index'])
from cctbx.miller import map_to_asu
map_to_asu(experiment.crystal.get_space_group().type(), False, unique)
perm = sorted(indices, key=lambda k: unique[k])
integrated_data = integrated_data.select(flex.size_t(perm))
from scitbx import matrix
from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame
assert (not experiment.goniometer is None)
unit_cell = experiment.crystal.get_unit_cell()
from scitbx.array_family import flex
from math import sqrt
assert (not experiment.scan is None)
image_range = experiment.scan.get_image_range()
phi_start, phi_range = experiment.scan.get_image_oscillation(
image_range[0])
# gather the required information for the reflection file
nref = len(integrated_data['miller_index'])
zdet = flex.double(integrated_data['xyzcal.px'].parts()[2])
miller_index = integrated_data['miller_index']
I = None
sigI = None
# export including scale factors
if 'lp' in integrated_data:
lp = integrated_data['lp']
else:
lp = flex.double(nref, 1.0)
if 'dqe' in integrated_data:
dqe = integrated_data['dqe']
else:
dqe = flex.double(nref, 1.0)
scl = lp / dqe
# profile correlation
if 'profile.correlation' in integrated_data:
prof_corr = 100.0 * integrated_data['profile.correlation']
else:
prof_corr = flex.double(nref, 100.0)
# partiality
if 'partiality' in integrated_data:
partiality = 100 * integrated_data['partiality']
else:
prof_corr = flex.double(nref, 100.0)
if summation:
I = integrated_data['intensity.sum.value'] * scl
V = integrated_data['intensity.sum.variance'] * scl * scl
assert V.all_gt(0)
V = var_model[0] * (V + var_model[1] * I * I)
sigI = flex.sqrt(V)
else:
I = integrated_data['intensity.prf.value'] * scl
V = integrated_data['intensity.prf.variance'] * scl * scl
assert V.all_gt(0)
V = var_model[0] * (V + var_model[1] * I * I)
sigI = flex.sqrt(V)
fout = open(hklout, 'w')
# first write the header - in the "standard" coordinate frame...
panel = experiment.detector[0]
fast = panel.get_fast_axis()
slow = panel.get_slow_axis()
Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0))
print 'Coordinate change:'
print '%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n' % Rd.elems
fast = Rd * fast
slow = Rd * slow
qx, qy = panel.get_pixel_size()
nx, ny = panel.get_image_size()
distance = matrix.col(Rd * panel.get_origin()).dot(
matrix.col(Rd * panel.get_normal()))
org = Rd * (matrix.col(panel.get_origin()) -
distance * matrix.col(panel.get_normal()))
orgx = -org.dot(fast) / qx
orgy = -org.dot(slow) / qy
UB = Rd * matrix.sqr(experiment.crystal.get_A())
real_space_ABC = UB.inverse().elems
axis = Rd * experiment.goniometer.get_rotation_axis()
beam = Rd * experiment.beam.get_s0()
cell_fmt = '%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f'
axis_fmt = '%9.3f %9.3f %9.3f'
fout.write('\n'.join([
'!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL\'S_LAW=TRUE',
'!Generated by dials.export',
'!DATA_RANGE= %d %d' % image_range,
'!ROTATION_AXIS= %9.6f %9.6f %9.6f' % axis.elems,
'!OSCILLATION_RANGE= %f' % phi_range,
'!STARTING_ANGLE= %f' % phi_start,
'!STARTING_FRAME= %d' % image_range[0],
'!SPACE_GROUP_NUMBER= %d' %
experiment.crystal.get_space_group().type().number(),
'!UNIT_CELL_CONSTANTS= %s' % (cell_fmt % unit_cell.parameters()),
'!UNIT_CELL_A-AXIS= %s' % (axis_fmt % real_space_ABC[0:3]),
'!UNIT_CELL_B-AXIS= %s' % (axis_fmt % real_space_ABC[3:6]),
'!UNIT_CELL_C-AXIS= %s' % (axis_fmt % real_space_ABC[6:9]),
'!X-RAY_WAVELENGTH= %f' % experiment.beam.get_wavelength(),
'!INCIDENT_BEAM_DIRECTION= %f %f %f' % beam.elems,
'!NX= %d NY= %d QX= %f QY= %f' % (nx, ny, qx, qy),
'!ORGX= %9.2f ORGY= %9.2f' % (orgx, orgy),
'!DETECTOR_DISTANCE= %8.3f' % distance,
'!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f' % fast.elems,
'!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f' % slow.elems,
'!VARIANCE_MODEL= %7.3e %7.3e' % var_model,
'!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12', '!ITEM_H=1', '!ITEM_K=2',
'!ITEM_L=3', '!ITEM_IOBS=4', '!ITEM_SIGMA(IOBS)=5', '!ITEM_XD=6',
'!ITEM_YD=7', '!ITEM_ZD=8', '!ITEM_RLP=9', '!ITEM_PEAK=10',
'!ITEM_CORR=11', '!ITEM_PSI=12', '!END_OF_HEADER', ''
]))
# then write the data records
s0 = Rd * matrix.col(experiment.beam.get_s0())
for j in range(nref):
x, y, z = integrated_data['xyzcal.px'][j]
phi = phi_start + z * phi_range
h, k, l = miller_index[j]
X = (UB * (h, k, l)).rotate(axis, phi, deg=True)
s = s0 + X
g = s.cross(s0).normalize()
f = (s - s0).normalize()
# find component of beam perpendicular to f, e
e = -(s + s0).normalize()
if h == k and k == l:
u = (h, -h, 0)
else:
u = (k - l, l - h, h - k)
q = (matrix.col(u).transpose() *
UB.inverse()).normalize().transpose().rotate(axis, phi, deg=True)
psi = q.angle(g, deg=True)
if q.dot(e) < 0:
psi *= -1
fout.write('%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n' %
(h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j],
prof_corr[j], psi))
fout.write('!END_OF_DATA\n')
fout.close()
logger.info('Output %d reflections to %s' % (nref, hklout))
def reindex(self, reindex_operation):
'''Reindex the reflections by the given reindexing operation.'''
R = rt_mx(reindex_operation).inverse()
# first construct mapping table - native, anomalous
map_native = { }
hkls = flex.miller_index()
for hkl in self._merged_reflections:
Fhkl = R * hkl
Rhkl = nint(Fhkl[0]), nint(Fhkl[1]), nint(Fhkl[2])
hkls.append(Rhkl)
map_to_asu(self._mf.get_space_group().type(), False, hkls)
for j, hkl in enumerate(self._merged_reflections):
map_native[hkl] = hkls[j]
map_anomalous = { }
hkls = flex.miller_index()
for hkl in self._merged_reflections_anomalous:
Fhkl = R * hkl
Rhkl = nint(Fhkl[0]), nint(Fhkl[1]), nint(Fhkl[2])
hkls.append(Rhkl)
map_to_asu(self._mf.get_space_group().type(), True, hkls)
for j, hkl in enumerate(self._merged_reflections_anomalous):
map_anomalous[hkl] = hkls[j]
# then remap the actual measurements
merged_reflections = { }
for hkl in self._merged_reflections:
Rhkl = map_native[hkl]
merged_reflections[Rhkl] = self._merged_reflections[hkl]
self._merged_reflections = merged_reflections
merged_reflections = { }
for hkl in self._merged_reflections_anomalous:
Rhkl = map_anomalous[hkl]
merged_reflections[Rhkl] = self._merged_reflections_anomalous[hkl]
self._merged_reflections_anomalous = merged_reflections
unmerged_reflections = { }
for hkl in self._unmerged_reflections:
Rhkl = map_native[hkl]
unmerged_reflections[Rhkl] = self._unmerged_reflections[hkl]
self._unmerged_reflections = unmerged_reflections
return
def __init__(
self,
rtable,
elist,
calculate_variances=False,
keep_singles=False,
uncertainty="sigma",
outfile=None,
):
"""
Generate z-scores and a normal probability plot from a DIALS
reflection_table and a dxtbx ExperimentList, containing the observations
and the corresponding experiments, respectively.
:param rtable: A reflection table object, containing at least the columns
* ``miller_index``
* ``intensity.sum.value``
* ``intensity.sum.variance``
* ``xyzobs.px.value``
:type rtable: dials.array_family_flex_ext.reflection_table
:param elist: A corresponding experiment list.
:type elist: dxtbx.model.ExperimentList
:param calculate_variances: Choose whether to calculate weighted
aggregate variances. Doing so incurs a performance penalty.
Defaults to False.
:type calculate_variances: bool
:param keep_singles: Choose whether to keep multiplicity-1 reflections.
Defaults to False.
:type keep_singles: bool
:param uncertainty: Measure of spread to use in normalising the
z-scores, i.e. z = (I - <I>) / uncertainty.
Possible values for uncertainty:
* 'sigma': Use measured sigma values;
* 'stddev': Use sample standard deviations calculated as
square-root of unbiased weighted sample variances
of symmetry-equivalent reflection intensities;
Defaults to 'sigma'.
:type uncertainty: str
:param outfile: Filename root for output PNG plots.
Defaults to None.
:type: outfile: str
"""
if not isinstance(rtable, flex.reflection_table) or not isinstance(
elist, ExperimentList):
raise TypeError(
"Must be called with a reflection table and an experiment list."
)
rtable = rtable.copy()
# Discard unindexed reflections (only necessary because of
# https://github.com/dials/dials/issues/615 —
# TODO remove the line below when issue #615 is fixed).
rtable.del_selected(rtable["id"] == -1)
rtable["miller_index.asu"] = rtable["miller_index"]
# Divide reflections by the space group to which they have been indexed.
self.rtables = {
expt.crystal.get_space_group().make_tidy():
flex.reflection_table()
for expt in elist
}
for expt, sel in rtable.iterate_experiments_and_indices(elist):
sg = expt.crystal.get_space_group().make_tidy()
self.rtables[sg].extend(rtable.select(sel))
# Map Miller indices to asymmetric unit.
for space_group, rtable in self.rtables.items():
# TODO Handle anomalous flag sensibly. Currently assumes not anomalous.
miller.map_to_asu(space_group.type(), False,
rtable["miller_index.asu"])
# Calculate normal probability plot data.
self._multiplicity_mean_error_stddev(
calculate_variances=calculate_variances, keep_singles=keep_singles)
self._make_z(uncertainty)
self._probplot_data()
self.rtable = flex.reflection_table()
for rtable in self.rtables.values():
self.rtable.extend(rtable)
if not outfile:
outfile = ""
self.outfile = outfile
def _compute_rij_wij(self, use_cache=True):
"""Compute the rij_wij matrix.
Rij is a symmetric matrix of size (n x m, n x m), where n is the number of
datasets and m is the number of symmetry operations.
It is composed of (m, m) blocks of size (n, n), where each block contains the
correlation coefficients between cb_op_k applied to datasets 1..N with
cb_op_kk applied to datasets 1.. N.
If `use_cache=True`, then an optimisation is made to reflect the fact some elements
of the matrix are equivalent, i.e.:
CC[(a, cb_op_k), (b, cb_op_kk)] == CC[(a,), (b, cb_op_k.inverse() * cb_op_kk)]
"""
n_lattices = len(self._lattices)
n_sym_ops = len(self.sym_ops)
# Pre-calculate miller indices after application of each cb_op. Only calculate
# this once per cb_op instead of on-the-fly every time we need it.
indices = {}
epsilons = {}
space_group_type = self._data.space_group().type()
for cb_op in self.sym_ops:
cb_op = sgtbx.change_of_basis_op(cb_op)
indices_reindexed = cb_op.apply(self._data.indices())
miller.map_to_asu(space_group_type, False, indices_reindexed)
cb_op_str = cb_op.as_xyz()
indices[cb_op_str] = np.array([
h.iround().as_numpy_array()
for h in indices_reindexed.as_vec3_double().parts()
]).transpose()
epsilons[cb_op_str] = self._patterson_group.epsilon(
indices_reindexed).as_numpy_array()
intensities = self._data.data().as_numpy_array()
# Map indices to an array of flat 1d indices which can later be used for
# matching pairs of indices
offset = -np.min(np.concatenate(list(indices.values())), axis=0)
dims = np.max(np.concatenate(list(indices.values())),
axis=0) + offset + 1
for cb_op, hkl in indices.items():
indices[cb_op] = np.ravel_multi_index((hkl + offset).T, dims)
# Create an empty 2D array of shape (m * n, L), where m is the number of sym
# ops, n is the number of lattices, and L is the number of unique miller indices
all_intensities = np.empty((n_sym_ops * n_lattices, np.prod(dims)))
# Populate all_intensities with intensity values, filling absent intensities
# with np.nan
all_intensities.fill(np.nan)
slices = np.append(self._lattices, intensities.size)
slices = list(map(slice, slices[:-1], slices[1:]))
for i, (mil_ind,
eps) in enumerate(zip(indices.values(), epsilons.values())):
for j, selection in enumerate(slices):
# map (i, j) to a column in all_intensities
column = np.ravel_multi_index((i, j), (n_sym_ops, n_lattices))
epsilon_equals_one = eps[selection] == 1
valid_mil_ind = mil_ind[selection][epsilon_equals_one]
valid_intensities = intensities[selection][epsilon_equals_one]
all_intensities[column, valid_mil_ind] = valid_intensities
# Ideally we would use `np.ma.corrcoef` here, but it is broken, so use
# pd.DataFrame.corr() instead (see numpy/numpy#15601)
rij = (pd.DataFrame(all_intensities).T.dropna(how="all").corr(
min_periods=self._min_pairs).values)
# Set any NaN correlation coefficients to zero
np.nan_to_num(rij, copy=False)
# Cosym does not make use of the on-diagonal correlation coefficients
np.fill_diagonal(rij, 0)
if self._weights:
wij = np.zeros_like(rij)
right_up = np.triu_indices_from(wij, k=1)
# For each correlation coefficient, set the weight equal to the size of
# the sample used to calculate that coefficient
pairwise_combos = itertools.combinations(
np.isfinite(all_intensities), 2)
sample_size = lambda x, y: np.count_nonzero(x & y)
wij[right_up] = list(
itertools.starmap(sample_size, pairwise_combos))
if self._weights == "standard_error":
# Set each weights as the reciprocal of the standard error on the
# corresponding correlation coefficient
# http://www.sjsu.edu/faculty/gerstman/StatPrimer/correlation.pdf
with np.errstate(divide="ignore", invalid="ignore"):
reciprocal_se = np.sqrt(
(wij[right_up] - 2) / (1 - np.square(rij[right_up])))
wij[right_up] = np.where(wij[right_up] > 2, reciprocal_se, 0)
# Symmetrise the wij matrix
wij += wij.T
else:
wij = None
return rij, wij
# for hkl in detectable_rays['miller_index']:
# seen_hkl_multiplicity[hkl] += 1
expt = experiments[0]
spacegroup = expt.crystal.get_space_group()
unit_cell = expt.crystal.get_unit_cell()
possible_hkl = self.list_possible_reflections(spacegroup, unit_cell, dmin, dmax)
possible_hkl_flex = flex.miller_index(possible_hkl)
hkl_to_id = { hkl: id for (id, hkl) in enumerate(possible_hkl) }
id_to_hkl = [ hkl for (id, hkl) in enumerate(possible_hkl) ]
# find mapping of reciprocal space onto reciprocal asymmetric unit and its inverse
from cctbx.miller import map_to_asu
asu_hkl_flex = flex.miller_index(possible_hkl)
map_to_asu(spacegroup.type(), False, asu_hkl_flex)
# TODO: Treat anomalous signal?
map_hkl_to_symmhkl = {r: rs for (r, rs) in zip(possible_hkl, list(asu_hkl_flex))}
map_symmhkl_to_hkl = {}
for k, v in map_hkl_to_symmhkl.iteritems():
map_symmhkl_to_hkl[v] = map_symmhkl_to_hkl.get(v, [])
map_symmhkl_to_hkl[v].append(k)
unique_asu_indices = set(asu_hkl_flex)
completeness_limit = len(unique_asu_indices)
# scoring function: hkl_id -> shared_bin_id
scoring_unique_reflection = { hkl: unique_id for (unique_id, hkl) in enumerate(possible_hkl) }
scoring_unique_reflection = { hkl_to_id[h]: id for (h, id) in scoring_unique_reflection.iteritems() }
scoring_symmetry_related_reflection = { hkl_to_id[hkl]: hkl_to_id[asu] for (hkl, asu) in map_hkl_to_symmhkl.iteritems() }
def exercise_unmerged(space_group_info):
import random
from cctbx import sgtbx
# shuffle the
space_group = sgtbx.space_group('P 1', no_expand=True)
perm = list(range(len(space_group_info.group())))
random.shuffle(perm)
for i in perm:
space_group.expand_smx(space_group_info.group()[i])
unit_cell = space_group_info.any_compatible_unit_cell(volume=1000)
for cb_op in (None,
space_group.info().change_of_basis_op_to_primitive_setting(),
unit_cell.change_of_basis_op_to_niggli_cell()):
miller_set = crystal.symmetry(
unit_cell=unit_cell,
space_group=space_group).build_miller_set(d_min=1,
anomalous_flag=True)
miller_set = miller_set.expand_to_p1().customized_copy(
space_group_info=miller_set.space_group_info())
if cb_op is not None:
miller_set = miller_set.change_basis(cb_op)
m = mtz.object()
m.set_title('This is a title')
m.set_space_group_info(miller_set.space_group_info())
x = m.add_crystal('XTAL', 'CCTBX', miller_set.unit_cell().parameters())
d = x.add_dataset('TEST', 1)
indices = miller_set.indices()
original_indices = indices.deep_copy()
# map the miller indices to one hemisphere (i.e. just I+)
miller.map_to_asu(m.space_group().type(), False, indices)
h, k, l = [i.iround() for i in indices.as_vec3_double().parts()]
m.adjust_column_array_sizes(len(h))
m.set_n_reflections(len(h))
# assign H, K, L
d.add_column('H', 'H').set_values(h.as_double().as_float())
d.add_column('K', 'H').set_values(k.as_double().as_float())
d.add_column('L', 'H').set_values(l.as_double().as_float())
d.add_column('M_ISYM', 'Y').set_values(flex.float(len(indices)))
m.replace_original_index_miller_indices(original_indices)
assert (m.extract_original_index_miller_indices() == original_indices
).count(False) == 0
# check the indices in the mtz file are actually in the asu
extracted_indices = m.extract_miller_indices()
miller.map_to_asu(m.space_group().type(), False, extracted_indices)
assert (
extracted_indices == m.extract_miller_indices()).count(False) == 0
# test change_basis_in_place if appropriate for current space group
import cctbx.sgtbx.cosets
cb_op_to_niggli_cell \
= miller_set.change_of_basis_op_to_niggli_cell()
minimum_cell_symmetry = crystal.symmetry.change_basis(
miller_set.crystal_symmetry(), cb_op=cb_op_to_niggli_cell)
lattice_group = sgtbx.lattice_symmetry.group(
minimum_cell_symmetry.unit_cell(), max_delta=3.0)
lattice_group.expand_inv(sgtbx.tr_vec((0, 0, 0)))
lattice_group.make_tidy()
cosets = sgtbx.cosets.left_decomposition_point_groups_only(
g=lattice_group,
h=minimum_cell_symmetry.reflection_intensity_symmetry(
anomalous_flag=True).space_group(
).build_derived_acentric_group().make_tidy())
possible_twin_laws = cosets.best_partition_representatives(
cb_op=cb_op_to_niggli_cell.inverse(),
omit_first_partition=True,
omit_negative_determinants=True)
for twin_law in possible_twin_laws:
cb_op = sgtbx.change_of_basis_op(twin_law.as_xyz())
#print cb_op.as_hkl()
# forward direction
m.change_basis_in_place(cb_op)
assert (m.extract_original_index_miller_indices() == cb_op.apply(
original_indices)).count(False) == 0
## check the indices in the mtz file are actually in the asu
#extracted_indices = m.extract_miller_indices()
#miller.map_to_asu(m.space_group().type(), False, extracted_indices)
#assert (extracted_indices == m.extract_miller_indices()).count(False) == 0
# and back again
m.change_basis_in_place(cb_op.inverse())
assert (m.extract_original_index_miller_indices() ==
original_indices).count(False) == 0
def export_xds_ascii(integrated_data, experiment_list, hklout, summation=False,
include_partials=False, keep_partials=False, var_model=(1,0)):
'''Export data from integrated_data corresponding to experiment_list to
an XDS_ASCII.HKL formatted text file.'''
from dials.array_family import flex
import math
# for the moment assume (and assert) that we will convert data from exactly
# one lattice...
assert(len(experiment_list) == 1)
# select reflections that are assigned to an experiment (i.e. non-negative id)
integrated_data = integrated_data.select(integrated_data['id'] >= 0)
assert max(integrated_data['id']) == 0
if not summation:
assert('intensity.prf.value' in integrated_data)
if 'intensity.prf.variance' in integrated_data:
selection = integrated_data.get_flags(
integrated_data.flags.integrated,
all=True)
else:
selection = integrated_data.get_flags(
integrated_data.flags.integrated_sum)
integrated_data = integrated_data.select(selection)
selection = integrated_data['intensity.sum.variance'] <= 0
if selection.count(True) > 0:
integrated_data.del_selected(selection)
logger.info('Removing %d reflections with negative variance' % \
selection.count(True))
if 'intensity.prf.variance' in integrated_data:
selection = integrated_data['intensity.prf.variance'] <= 0
if selection.count(True) > 0:
integrated_data.del_selected(selection)
logger.info('Removing %d profile reflections with negative variance' % \
selection.count(True))
if include_partials:
integrated_data = sum_partial_reflections(integrated_data)
integrated_data = scale_partial_reflections(integrated_data)
if 'partiality' in integrated_data:
selection = integrated_data['partiality'] < 0.99
if selection.count(True) > 0 and not keep_partials:
integrated_data.del_selected(selection)
logger.info('Removing %d incomplete reflections' % \
selection.count(True))
experiment = experiment_list[0]
# sort data before output
nref = len(integrated_data['miller_index'])
indices = flex.size_t_range(nref)
import copy
unique = copy.deepcopy(integrated_data['miller_index'])
from cctbx.miller import map_to_asu
map_to_asu(experiment.crystal.get_space_group().type(), False, unique)
perm = sorted(indices, key=lambda k: unique[k])
integrated_data = integrated_data.select(flex.size_t(perm))
from scitbx import matrix
from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame
assert (not experiment.goniometer is None)
unit_cell = experiment.crystal.get_unit_cell()
from scitbx.array_family import flex
from math import floor, sqrt
assert(not experiment.scan is None)
image_range = experiment.scan.get_image_range()
phi_start, phi_range = experiment.scan.get_image_oscillation(image_range[0])
# gather the required information for the reflection file
nref = len(integrated_data['miller_index'])
zdet = flex.double(integrated_data['xyzcal.px'].parts()[2])
miller_index = integrated_data['miller_index']
I = None
sigI = None
# export including scale factors
if 'lp' in integrated_data:
lp = integrated_data['lp']
else:
lp = flex.double(nref, 1.0)
if 'dqe' in integrated_data:
dqe = integrated_data['dqe']
else:
dqe = flex.double(nref, 1.0)
scl = lp / dqe
# profile correlation
if 'profile.correlation' in integrated_data:
prof_corr = 100.0 * integrated_data['profile.correlation']
else:
prof_corr = flex.double(nref, 100.0)
# partiality
if 'partiality' in integrated_data:
partiality = 100 * integrated_data['partiality']
else:
prof_corr = flex.double(nref, 100.0)
if summation:
I = integrated_data['intensity.sum.value'] * scl
V = integrated_data['intensity.sum.variance'] * scl * scl
assert V.all_gt(0)
V = var_model[0] * (V + var_model[1] * I * I)
sigI = flex.sqrt(V)
else:
I = integrated_data['intensity.prf.value'] * scl
V = integrated_data['intensity.prf.variance'] * scl * scl
assert V.all_gt(0)
V = var_model[0] * (V + var_model[1] * I * I)
sigI = flex.sqrt(V)
fout = open(hklout, 'w')
# first write the header - in the "standard" coordinate frame...
panel = experiment.detector[0]
fast = panel.get_fast_axis()
slow = panel.get_slow_axis()
Rd = align_reference_frame(fast, (1,0,0), slow, (0,1,0))
print 'Coordinate change:'
print '%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n' % Rd.elems
fast = Rd * fast
slow = Rd * slow
qx, qy = panel.get_pixel_size()
nx, ny = panel.get_image_size()
distance = matrix.col(Rd * panel.get_origin()).dot(
matrix.col(Rd * panel.get_normal()))
org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col(
panel.get_normal()))
orgx = - org.dot(fast) / qx
orgy = - org.dot(slow) / qy
UB = Rd * matrix.sqr(experiment.crystal.get_A())
real_space_ABC = UB.inverse().elems
axis = Rd * experiment.goniometer.get_rotation_axis()
beam = Rd * experiment.beam.get_s0()
cell_fmt = '%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f'
axis_fmt = '%9.3f %9.3f %9.3f'
fout.write('\n'.join([
'!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL\'S_LAW=TRUE',
'!Generated by dials.export',
'!DATA_RANGE= %d %d' % image_range,
'!ROTATION_AXIS= %9.6f %9.6f %9.6f' % axis.elems,
'!OSCILLATION_RANGE= %f' % phi_range,
'!STARTING_ANGLE= %f' % phi_start,
'!STARTING_FRAME= %d' % image_range[0],
'!SPACE_GROUP_NUMBER= %d' % experiment.crystal.get_space_group().type().number(),
'!UNIT_CELL_CONSTANTS= %s' % (cell_fmt % unit_cell.parameters()),
'!UNIT_CELL_A-AXIS= %s' % (axis_fmt % real_space_ABC[0:3]),
'!UNIT_CELL_B-AXIS= %s' % (axis_fmt % real_space_ABC[3:6]),
'!UNIT_CELL_C-AXIS= %s' % (axis_fmt % real_space_ABC[6:9]),
'!X-RAY_WAVELENGTH= %f' % experiment.beam.get_wavelength(),
'!INCIDENT_BEAM_DIRECTION= %f %f %f' % beam.elems,
'!NX= %d NY= %d QX= %f QY= %f' % (nx, ny, qx, qy),
'!ORGX= %9.2f ORGY= %9.2f' % (orgx, orgy),
'!DETECTOR_DISTANCE= %8.3f' % distance,
'!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f' % fast.elems,
'!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f' % slow.elems,
'!VARIANCE_MODEL= %7.3e %7.3e' % var_model,
'!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12',
'!ITEM_H=1',
'!ITEM_K=2',
'!ITEM_L=3',
'!ITEM_IOBS=4',
'!ITEM_SIGMA(IOBS)=5',
'!ITEM_XD=6',
'!ITEM_YD=7',
'!ITEM_ZD=8',
'!ITEM_RLP=9',
'!ITEM_PEAK=10',
'!ITEM_CORR=11',
'!ITEM_PSI=12',
'!END_OF_HEADER',
'']))
# then write the data records
s0 = Rd * matrix.col(experiment.beam.get_s0())
for j in range(nref):
x, y, z = integrated_data['xyzcal.px'][j]
phi = phi_start + z * phi_range
h, k, l = miller_index[j]
X = (UB * (h, k, l)).rotate(axis, phi, deg=True)
s = s0 + X
g = s.cross(s0).normalize()
f = (s - s0).normalize()
# find component of beam perpendicular to f, e
e = - (s + s0).normalize()
if h == k and k == l:
u = (h, -h, 0)
else:
u = (k - l, l - h, h - k)
q = (matrix.col(u).transpose() * UB.inverse()).normalize(
).transpose().rotate(axis, phi, deg=True)
psi = q.angle(g, deg=True)
if q.dot(e) < 0:
psi *= -1
fout.write('%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n' %
(h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi))
fout.write('!END_OF_DATA\n')
fout.close()
logger.info('Output %d reflections to %s' % (nref, hklout))
return
def exercise_unmerged(space_group_info):
import random
from cctbx import sgtbx
# shuffle the
space_group = sgtbx.space_group("P 1", no_expand=True)
perm = list(range(len(space_group_info.group())))
random.shuffle(perm)
for i in perm:
space_group.expand_smx(space_group_info.group()[i])
unit_cell = space_group_info.any_compatible_unit_cell(volume=1000)
for cb_op in (
None,
space_group.info().change_of_basis_op_to_primitive_setting(),
unit_cell.change_of_basis_op_to_niggli_cell(),
):
miller_set = crystal.symmetry(unit_cell=unit_cell, space_group=space_group).build_miller_set(
d_min=1, anomalous_flag=True
)
miller_set = miller_set.expand_to_p1().customized_copy(space_group_info=miller_set.space_group_info())
if cb_op is not None:
miller_set = miller_set.change_basis(cb_op)
m = mtz.object()
m.set_title("This is a title")
m.set_space_group_info(miller_set.space_group_info())
x = m.add_crystal("XTAL", "CCTBX", miller_set.unit_cell().parameters())
d = x.add_dataset("TEST", 1)
indices = miller_set.indices()
original_indices = indices.deep_copy()
# map the miller indices to one hemisphere (i.e. just I+)
miller.map_to_asu(m.space_group().type(), False, indices)
h, k, l = [i.iround() for i in indices.as_vec3_double().parts()]
m.adjust_column_array_sizes(len(h))
m.set_n_reflections(len(h))
# assign H, K, L
d.add_column("H", "H").set_values(h.as_double().as_float())
d.add_column("K", "H").set_values(k.as_double().as_float())
d.add_column("L", "H").set_values(l.as_double().as_float())
d.add_column("M_ISYM", "Y").set_values(flex.float(len(indices)))
m.replace_original_index_miller_indices(original_indices)
assert (m.extract_original_index_miller_indices() == original_indices).count(False) == 0
# check the indices in the mtz file are actually in the asu
extracted_indices = m.extract_miller_indices()
miller.map_to_asu(m.space_group().type(), False, extracted_indices)
assert (extracted_indices == m.extract_miller_indices()).count(False) == 0
# test change_basis_in_place if appropriate for current space group
import cctbx.sgtbx.cosets
cb_op_to_niggli_cell = miller_set.change_of_basis_op_to_niggli_cell()
minimum_cell_symmetry = crystal.symmetry.change_basis(miller_set.crystal_symmetry(), cb_op=cb_op_to_niggli_cell)
lattice_group = sgtbx.lattice_symmetry.group(minimum_cell_symmetry.unit_cell(), max_delta=3.0)
lattice_group.expand_inv(sgtbx.tr_vec((0, 0, 0)))
lattice_group.make_tidy()
cosets = sgtbx.cosets.left_decomposition_point_groups_only(
g=lattice_group,
h=minimum_cell_symmetry.reflection_intensity_symmetry(anomalous_flag=True)
.space_group()
.build_derived_acentric_group()
.make_tidy(),
)
possible_twin_laws = cosets.best_partition_representatives(
cb_op=cb_op_to_niggli_cell.inverse(), omit_first_partition=True, omit_negative_determinants=True
)
for twin_law in possible_twin_laws:
cb_op = sgtbx.change_of_basis_op(twin_law.as_xyz())
# print cb_op.as_hkl()
# forward direction
m.change_basis_in_place(cb_op)
assert (m.extract_original_index_miller_indices() == cb_op.apply(original_indices)).count(False) == 0
## check the indices in the mtz file are actually in the asu
# extracted_indices = m.extract_miller_indices()
# miller.map_to_asu(m.space_group().type(), False, extracted_indices)
# assert (extracted_indices == m.extract_miller_indices()).count(False) == 0
# and back again
m.change_basis_in_place(cb_op.inverse())
assert (m.extract_original_index_miller_indices() == original_indices).count(False) == 0
def _compute_rij_wij(self, use_cache=True):
"""Compute the rij_wij matrix.
Rij is a symmetric matrix of size (n x m, n x m), where n is the number of
datasets and m is the number of symmetry operations.
It is composed of (m, m) blocks of size (n, n), where each block contains the
correlation coefficients between cb_op_k applied to datasets 1..N with
cb_op_kk applied to datasets 1.. N.
If `use_cache=True`, then an optimisation is made to reflect the fact some elements
of the matrix are equivalent, i.e.:
CC[(a, cb_op_k), (b, cb_op_kk)] == CC[(a,), (b, cb_op_k.inverse() * cb_op_kk)]
"""
n_lattices = len(self._lattices)
# Pre-calculate miller indices after application of each cb_op. Only calculate
# this once per cb_op instead of on-the-fly every time we need it.
indices = {}
space_group_type = self._data.space_group().type()
for cb_op in self.sym_ops:
cb_op = sgtbx.change_of_basis_op(cb_op)
indices_reindexed = cb_op.apply(self._data.indices())
miller.map_to_asu(space_group_type, False, indices_reindexed)
indices[cb_op.as_xyz()] = indices_reindexed
def _compute_rij_matrix_one_row_block(i):
rij_cache = {}
n_sym_ops = len(self.sym_ops)
NN = n_lattices * n_sym_ops
rij_row = []
rij_col = []
rij_data = []
if self._weights is not None:
wij_row = []
wij_col = []
wij_data = []
else:
wij = None
i_lower, i_upper = self._lattice_lower_upper_index(i)
intensities_i = self._data.data()[i_lower:i_upper]
for j in range(n_lattices):
j_lower, j_upper = self._lattice_lower_upper_index(j)
intensities_j = self._data.data()[j_lower:j_upper]
for k, cb_op_k in enumerate(self.sym_ops):
cb_op_k = sgtbx.change_of_basis_op(cb_op_k)
indices_i = indices[cb_op_k.as_xyz()][i_lower:i_upper]
for kk, cb_op_kk in enumerate(self.sym_ops):
if i == j and k == kk:
# don't include correlation of dataset with itself
continue
cb_op_kk = sgtbx.change_of_basis_op(cb_op_kk)
ik = i + (n_lattices * k)
jk = j + (n_lattices * kk)
key = (i, j, str(cb_op_k.inverse() * cb_op_kk))
if use_cache and key in rij_cache:
cc, n = rij_cache[key]
else:
indices_j = indices[cb_op_kk.as_xyz()][j_lower:j_upper]
matches = miller.match_indices(indices_i, indices_j)
pairs = matches.pairs()
isel_i = pairs.column(0)
isel_j = pairs.column(1)
isel_i = isel_i.select(
self._patterson_group.epsilon(indices_i.select(isel_i))
== 1
)
isel_j = isel_j.select(
self._patterson_group.epsilon(indices_j.select(isel_j))
== 1
)
corr = flex.linear_correlation(
intensities_i.select(isel_i),
intensities_j.select(isel_j),
)
if corr.is_well_defined():
cc = corr.coefficient()
n = corr.n()
rij_cache[key] = (cc, n)
else:
cc = None
n = None
if (
n is None
or cc is None
or (self._min_pairs is not None and n < self._min_pairs)
):
continue
if self._weights == "count":
wij_row.extend([ik, jk])
wij_col.extend([jk, ik])
wij_data.extend([n, n])
elif self._weights == "standard_error":
assert n > 2
# http://www.sjsu.edu/faculty/gerstman/StatPrimer/correlation.pdf
se = math.sqrt((1 - cc ** 2) / (n - 2))
wij = 1 / se
wij_row.extend([ik, jk])
wij_col.extend([jk, ik])
wij_data.extend([wij, wij])
rij_row.append(ik)
rij_col.append(jk)
rij_data.append(cc)
rij = sparse.coo_matrix((rij_data, (rij_row, rij_col)), shape=(NN, NN))
if self._weights is not None:
wij = sparse.coo_matrix((wij_data, (wij_row, wij_col)), shape=(NN, NN))
return rij, wij
args = [(i,) for i in range(n_lattices)]
results = easy_mp.parallel_map(
_compute_rij_matrix_one_row_block,
args,
processes=self._nproc,
iterable_type=easy_mp.posiargs,
method="multiprocessing",
)
rij_matrix = None
wij_matrix = None
for i, (rij, wij) in enumerate(results):
if rij_matrix is None:
rij_matrix = rij
else:
rij_matrix += rij
if wij is not None:
if wij_matrix is None:
wij_matrix = wij
else:
wij_matrix += wij
rij_matrix = rij_matrix.todense().astype(np.float64)
if wij_matrix is not None:
wij_matrix = wij_matrix.todense().astype(np.float64)
return rij_matrix, wij_matrix
def StrategyEvaluator(self, experiments, evaluation_function_factory, dmin,
dmax):
# TODO: Don't take experiments, rather take one set of experiment components and a strategy equivalent list
def calculate_miller_rings(detector):
import time
t = time.clock()
mrings = {}
for scan_axis in ["phi", "omega"]:
mrings[scan_axis] = {}
for other_axis in range(0, 360, 10):
gonio = goniometer_factory.make_kappa_goniometer(
alpha=54.7356, # fixed magic angle
kappa=0, # fixed 0 kappa
phi=0 if scan_axis == "phi" else other_axis,
omega=0 if scan_axis == "omega" else other_axis,
direction="-y",
scan_axis=scan_axis,
)
ring = tools.determine_miller_ring_sectors(
detector, gonio, s0, possible_hkl_flex, crystal_A)
mrings[scan_axis][other_axis] = ring
print(time.clock() - t)
print(list(mrings.iterkeys()))
print([list(x.iterkeys()) for x in mrings.itervalues()])
t = time.clock()
sc = scorings
all_scorings = {
"%s:%s:%s" % (scan, str(x1), str(x2)): tools.geometric_scoring(
[hkl_to_id[hkl] for hkl in mrings[scan][x1][x2]],
sc)["total"]
for scan in mrings.iterkeys()
for x1 in mrings[scan].iterkeys() for x2 in range(0, 36)
}
# print list(all_scorings.itervalues())
print(
len(list(all_scorings.itervalues())),
min(all_scorings.itervalues()),
max(all_scorings.itervalues()),
sum(all_scorings.itervalues()) /
len(list(all_scorings.itervalues())),
)
print(time.clock() - t)
sys.exit(0)
def calculate_observations(detector, goniometer, oscillation):
calculate_miller_rings(detector)
import time
t = time.clock()
r = tools.determine_miller_ring_sectors(detector, goniometer, s0,
possible_hkl_flex,
crystal_A)
sc = scorings
print(
map(
lambda x: tools.geometric_scoring(
[hkl_to_id[hkl] for hkl in x], sc)["total"],
r,
))
sc = tools.geometric_scoring([hkl_to_id[hkl] for hkl in r[4]],
scorings)["scorings"]
sc = tools.geometric_scoring([hkl_to_id[hkl] for hkl in r[5]],
sc)["scorings"]
sc = tools.geometric_scoring([hkl_to_id[hkl] for hkl in r[6]],
sc)["scorings"]
print(
map(
lambda x: tools.geometric_scoring(
[hkl_to_id[hkl] for hkl in x], sc)["total"],
r,
))
print(time.clock() - t)
print(map(len, r))
sys.exit(0)
# TODO: To test 2theta rotation code use detector.py detector factory
# TODO: and create two different two_theta detectors and try to rotate one onto the other
# ----: Reflection only counted once within oscillation, even when multiple times in diffraction condition
# ----: Is it?! Unsure.
# SOLVED: half-true. Not problematic here.
def export_mtz(observed_hkls, experiment, filename):
if experiment.goniometer:
axis = experiment.goniometer.get_rotation_axis()
else:
axis = 0.0, 0.0, 0.0
s0 = experiment.beam.get_s0()
wavelength = experiment.beam.get_wavelength()
from scitbx import matrix
panel = experiment.detector[0]
pixel_size = panel.get_pixel_size()
cb_op_to_ref = (experiment.crystal.get_space_group().info().
change_of_basis_op_to_reference_setting())
experiment.crystal = experiment.crystal.change_basis(cb_op_to_ref)
from iotbx import mtz
from scitbx.array_family import flex
import itertools
m = mtz.object()
m.set_title("from dials.scratch.mg.strategy_i19")
m.set_space_group_info(experiment.crystal.get_space_group().info())
nrefcount = sum(observed_hkls.itervalues())
nref = max(observed_hkls.itervalues())
for batch in range(1, nref + 1):
o = m.add_batch().set_num(batch).set_nbsetid(1).set_ncryst(1)
o.set_time1(0.0).set_time2(0.0).set_title("Batch %d" % batch)
o.set_ndet(1).set_theta(flex.float((0.0, 0.0))).set_lbmflg(0)
o.set_alambd(wavelength).set_delamb(0.0).set_delcor(0.0)
o.set_divhd(0.0).set_divvd(0.0)
o.set_so(flex.float(s0)).set_source(flex.float((0, 0, -1)))
o.set_bbfac(0.0).set_bscale(1.0)
o.set_sdbfac(0.0).set_sdbscale(0.0).set_nbscal(0)
_unit_cell = experiment.crystal.get_unit_cell()
_U = experiment.crystal.get_U()
o.set_cell(flex.float(_unit_cell.parameters()))
o.set_lbcell(flex.int((-1, -1, -1, -1, -1, -1)))
o.set_umat(flex.float(_U.transpose().elems))
mosaic = experiment.crystal.get_mosaicity()
o.set_crydat(
flex.float([
mosaic, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0
]))
o.set_lcrflg(0)
o.set_datum(flex.float((0.0, 0.0, 0.0)))
# detector size, distance
o.set_detlm(
flex.float([
0.0,
panel.get_image_size()[0],
0.0,
panel.get_image_size()[1],
0,
0,
0,
0,
]))
o.set_dx(flex.float([panel.get_directed_distance(), 0.0]))
# goniometer axes and names, and scan axis number, and number of axes, missets
o.set_e1(flex.float(axis))
o.set_e2(flex.float((0.0, 0.0, 0.0)))
o.set_e3(flex.float((0.0, 0.0, 0.0)))
o.set_gonlab(flex.std_string(("AXIS", "", "")))
o.set_jsaxs(1)
o.set_ngonax(1)
o.set_phixyz(flex.float((0.0, 0.0, 0.0, 0.0, 0.0, 0.0)))
phi_start, phi_range = 0.0, 0.0
o.set_phistt(phi_start)
o.set_phirange(phi_range)
o.set_phiend(phi_start + phi_range)
o.set_scanax(flex.float(axis))
# number of misorientation angles
o.set_misflg(0)
# crystal axis closest to rotation axis (why do I want this?)
o.set_jumpax(0)
# type of data - 1; 2D, 2; 3D, 3; Laue
o.set_ldtype(2)
# now create the actual data structures - first keep a track of the columns
# H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH
# LP MPART FLAG BGPKRATIOS
from cctbx.array_family import flex as cflex # implicit import
# now go for it and make an MTZ file...
x = m.add_crystal("XTAL", "DIALS", unit_cell.parameters())
d = x.add_dataset("FROMDIALS", wavelength)
# now add column information...
type_table = {
"IPR": "J",
"BGPKRATIOS": "R",
"WIDTH": "R",
"I": "J",
"H": "H",
"K": "H",
"MPART": "I",
"L": "H",
"BATCH": "B",
"M_ISYM": "Y",
"SIGI": "Q",
"FLAG": "I",
"XDET": "R",
"LP": "R",
"YDET": "R",
"SIGIPR": "Q",
"FRACTIONCALC": "R",
"ROT": "R",
}
m.adjust_column_array_sizes(nrefcount)
m.set_n_reflections(nrefcount)
# assign H, K, L, M_ISYM space
for column in "H", "K", "L", "M_ISYM":
d.add_column(column, type_table[column]).set_values(
flex.float(nrefcount, 0.0))
batchnums = (_ for (x, n) in observed_hkls.iteritems()
for _ in range(1, n + 1))
d.add_column("BATCH",
type_table["BATCH"]).set_values(flex.float(batchnums))
d.add_column("FRACTIONCALC",
type_table["FRACTIONCALC"]).set_values(
flex.float(nrefcount, 3.0))
m.replace_original_index_miller_indices(
cb_op_to_ref.apply(
cflex.miller_index([
_ for (x, n) in observed_hkls.iteritems()
for _ in itertools.repeat(x, n)
])))
m.write(filename)
return m
def evaluate_concrete_strategy(hkls, evaluation_function):
print("%5d reflections fall on detector during sweep" % len(hkls))
# print "%5d unique reflections fall on detector during sweep (without symmetry relations)" % len(set(list(hkls)))
# for r in detectable_rays.rows():
# hkl = r['miller_index']
# if (abs(hkl[0]) == 11) and (abs(hkl[1]) == 5) and (abs(hkl[2]) < 5):
# print "%12s -> %10s observed at angle %.2f on image %.1f" % (hkl, map_hkl_to_symmhkl[hkl], r['phi'],
# expt.scan.get_image_index_from_angle(r['phi'] - (2 * 3.1415926535), deg=False))
symmetry_mapped_hkls = set(
[map_hkl_to_symmhkl[hkl] for hkl in hkls])
print(
"%5d unique reflections fall on detector during sweep (including symmetry relations)"
% len(symmetry_mapped_hkls))
completeness = 100 * len(symmetry_mapped_hkls) / completeness_limit
multiplicity = len(hkls) / len(symmetry_mapped_hkls)
score = evaluation_function(hkls)
print(
"Estimated sweep completeness: %5.1f %% sweep multiplicity : %.1f sweep score: %.1f"
% (completeness, multiplicity, score))
return {
"completeness": completeness,
"multiplicity": multiplicity,
"score": score,
}
# # count the number of observations per reflection to construct an evaluation function
# seen_hkl_multiplicity = {x: 0 for x in possible_hkl}
# for hkl in detectable_rays['miller_index']:
# seen_hkl_multiplicity[hkl] += 1
expt = experiments[0]
spacegroup = expt.crystal.get_space_group()
unit_cell = expt.crystal.get_unit_cell()
possible_hkl = self.list_possible_reflections(spacegroup, unit_cell,
dmin, dmax)
possible_hkl_flex = flex.miller_index(possible_hkl)
hkl_to_id = {hkl: id for (id, hkl) in enumerate(possible_hkl)}
id_to_hkl = [hkl for (id, hkl) in enumerate(possible_hkl)]
# find mapping of reciprocal space onto reciprocal asymmetric unit and its inverse
from cctbx.miller import map_to_asu
asu_hkl_flex = flex.miller_index(possible_hkl)
map_to_asu(spacegroup.type(), False, asu_hkl_flex)
# TODO: Treat anomalous signal?
map_hkl_to_symmhkl = {
r: rs
for (r, rs) in zip(possible_hkl, list(asu_hkl_flex))
}
map_symmhkl_to_hkl = {}
for k, v in map_hkl_to_symmhkl.iteritems():
map_symmhkl_to_hkl[v] = map_symmhkl_to_hkl.get(v, [])
map_symmhkl_to_hkl[v].append(k)
unique_asu_indices = set(asu_hkl_flex)
completeness_limit = len(unique_asu_indices)
# scoring function: hkl_id -> shared_bin_id
scoring_unique_reflection = {
hkl: unique_id
for (unique_id, hkl) in enumerate(possible_hkl)
}
scoring_unique_reflection = {
hkl_to_id[h]: id
for (h, id) in scoring_unique_reflection.iteritems()
}
scoring_symmetry_related_reflection = {
hkl_to_id[hkl]: hkl_to_id[asu]
for (hkl, asu) in map_hkl_to_symmhkl.iteritems()
}
scorings = {
"unique": (scoring_unique_reflection, [0] * len(possible_hkl)),
"symmetry":
(scoring_symmetry_related_reflection, [0] * len(possible_hkl)),
}
print(
"%5d unique reflections possible in %s (ignoring anomalous signal)"
% (completeness_limit, spacegroup.type().lookup_symbol()))
print()
# Determine the detectable reflections given some data collection sweep
s0 = expt.beam.get_s0()
rotation_axis = expt.goniometer.get_rotation_axis()
# Obtain the A matrix with all diffractometer angles set to 0.
crystal_R = matrix.sqr(expt.goniometer.get_fixed_rotation())
crystal_A = crystal_R.inverse() * expt.crystal.get_A()
def run_strategies(strategies):
# starting from scratch
observed_hkls = {}
total_score = 0.0
degrees = 0
from itertools import izip, count
for (run, strategy) in izip(count(1), strategies):
print()
print("Sweep %d:" % run)
evaluation_function = evaluation_function_factory(
possible_hkl, observed_hkls, map_hkl_to_symmhkl,
map_symmhkl_to_hkl)
goniometer = goniometer_factory.make_kappa_goniometer(
alpha=54.7,
kappa=strategy["kappa"],
phi=strategy["phi"],
omega=strategy["omega"],
direction="-y",
scan_axis=strategy["scan_axis"],
)
degrees += strategy["scan"]
from math import radians
if strategy["scan_axis"] == "omega":
oscillation_range = (
radians(strategy["omega"]),
radians(strategy["omega"] + strategy["scan"]),
)
else:
oscillation_range = (
radians(strategy["phi"]),
radians(strategy["phi"] + strategy["scan"]),
)
# repeat sweep: expt.goniometer
proposed_hkls = calculate_observations(expt.detector,
goniometer,
oscillation_range)
strategy_results = evaluate_concrete_strategy(
proposed_hkls, evaluation_function)
combined_observations = self.add_seen_multiplicity(
observed_hkls, proposed_hkls)
num_observed_hkls = len(
set([
map_hkl_to_symmhkl[hkl]
for hkl in combined_observations.keys()
]))
count_hkl_observations = sum(combined_observations.values())
completeness = 100 * num_observed_hkls / completeness_limit
multiplicity = count_hkl_observations / num_observed_hkls
total_score += strategy_results["score"]
print(
"Estimated total completeness: %5.1f %% total multiplicity : %.1f total score: %.1f"
% (completeness, multiplicity, total_score))
# keep the repeat sweep and continue
observed_hkls = combined_observations
export_mtz(observed_hkls, expt, "mtzout.mtz")
return {
"completeness": completeness,
"multiplicity": multiplicity,
"score": total_score,
"degrees": degrees,
}
return run_strategies
def _export_experiment(filename,
integrated_data,
experiment,
params,
var_model=(1, 0)):
# type: (str, flex.reflection_table, dxtbx.model.Experiment, libtbx.phil.scope_extract, Tuple)
"""Export a single experiment to an XDS_ASCII.HKL format file.
Args:
filename: The file to write to
integrated_data: The reflection table, pre-selected to one experiment
experiment: The experiment list entry to export
params: The PHIL configuration object
var_model:
"""
# export for xds_ascii should only be for non-scaled reflections
assert any(i in integrated_data
for i in ["intensity.sum.value", "intensity.prf.value"])
# Handle requesting profile intensities (default via auto) but no column
if "profile" in params.intensity and "intensity.prf.value" not in integrated_data:
raise Sorry(
"Requested profile intensity data but only summed present. Use intensity=sum."
)
integrated_data = filter_reflection_table(
integrated_data,
intensity_choice=params.intensity,
partiality_threshold=params.mtz.partiality_threshold,
combine_partials=params.mtz.combine_partials,
min_isigi=params.mtz.min_isigi,
filter_ice_rings=params.mtz.filter_ice_rings,
d_min=params.mtz.d_min,
)
# calculate the scl = lp/dqe correction for outputting but don't apply it as
# it has already been applied in filter_reflection_table
(
integrated_data,
scl,
) = FilteringReductionMethods.calculate_lp_qe_correction_and_filter(
integrated_data)
# sort data before output
nref = len(integrated_data["miller_index"])
indices = flex.size_t_range(nref)
unique = copy.deepcopy(integrated_data["miller_index"])
map_to_asu(experiment.crystal.get_space_group().type(), False, unique)
perm = sorted(indices, key=lambda k: unique[k])
integrated_data = integrated_data.select(flex.size_t(perm))
if experiment.goniometer is None:
print(
"Warning: No goniometer. Experimentally exporting with (1 0 0) axis"
)
unit_cell = experiment.crystal.get_unit_cell()
if experiment.scan is None:
print(
"Warning: No Scan. Experimentally exporting no-oscillation values")
image_range = (1, 1)
phi_start, phi_range = 0.0, 0.0
else:
image_range = experiment.scan.get_image_range()
phi_start, phi_range = experiment.scan.get_image_oscillation(
image_range[0])
# gather the required information for the reflection file
nref = len(integrated_data["miller_index"])
miller_index = integrated_data["miller_index"]
# profile correlation
if "profile.correlation" in integrated_data:
prof_corr = 100.0 * integrated_data["profile.correlation"]
else:
prof_corr = flex.double(nref, 100.0)
# partiality
if "partiality" in integrated_data:
partiality = 100 * integrated_data["partiality"]
else:
prof_corr = flex.double(nref, 100.0)
if "intensity.sum.value" in integrated_data:
I = integrated_data["intensity.sum.value"]
V = integrated_data["intensity.sum.variance"]
assert V.all_gt(0)
V = var_model[0] * (V + var_model[1] * I * I)
sigI = flex.sqrt(V)
else:
I = integrated_data["intensity.prf.value"]
V = integrated_data["intensity.prf.variance"]
assert V.all_gt(0)
V = var_model[0] * (V + var_model[1] * I * I)
sigI = flex.sqrt(V)
fout = open(filename, "w")
# first write the header - in the "standard" coordinate frame...
panel = experiment.detector[0]
fast = panel.get_fast_axis()
slow = panel.get_slow_axis()
Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0))
print("Coordinate change:")
print("%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n" %
Rd.elems)
fast = Rd * fast
slow = Rd * slow
qx, qy = panel.get_pixel_size()
nx, ny = panel.get_image_size()
distance = matrix.col(Rd * panel.get_origin()).dot(
matrix.col(Rd * panel.get_normal()))
org = Rd * (matrix.col(panel.get_origin()) -
distance * matrix.col(panel.get_normal()))
orgx = -org.dot(fast) / qx
orgy = -org.dot(slow) / qy
UB = Rd * matrix.sqr(experiment.crystal.get_A())
real_space_ABC = UB.inverse().elems
if experiment.goniometer is not None:
axis = Rd * experiment.goniometer.get_rotation_axis()
else:
axis = Rd * (1, 0, 0)
beam = Rd * experiment.beam.get_s0()
cell_fmt = "%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f"
axis_fmt = "%9.3f %9.3f %9.3f"
fout.write("\n".join([
"!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL'S_LAW=TRUE",
"!Generated by dials.export",
"!DATA_RANGE= %d %d" % image_range,
"!ROTATION_AXIS= %9.6f %9.6f %9.6f" % axis.elems,
"!OSCILLATION_RANGE= %f" % phi_range,
"!STARTING_ANGLE= %f" % phi_start,
"!STARTING_FRAME= %d" % image_range[0],
"!SPACE_GROUP_NUMBER= %d" %
experiment.crystal.get_space_group().type().number(),
"!UNIT_CELL_CONSTANTS= %s" % (cell_fmt % unit_cell.parameters()),
"!UNIT_CELL_A-AXIS= %s" % (axis_fmt % real_space_ABC[0:3]),
"!UNIT_CELL_B-AXIS= %s" % (axis_fmt % real_space_ABC[3:6]),
"!UNIT_CELL_C-AXIS= %s" % (axis_fmt % real_space_ABC[6:9]),
"!X-RAY_WAVELENGTH= %f" % experiment.beam.get_wavelength(),
"!INCIDENT_BEAM_DIRECTION= %f %f %f" % beam.elems,
"!NX= %d NY= %d QX= %f QY= %f" % (nx, ny, qx, qy),
"!ORGX= %9.2f ORGY= %9.2f" % (orgx, orgy),
"!DETECTOR_DISTANCE= %8.3f" % distance,
"!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f" % fast.elems,
"!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f" % slow.elems,
"!VARIANCE_MODEL= %7.3e %7.3e" % var_model,
"!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12",
"!ITEM_H=1",
"!ITEM_K=2",
"!ITEM_L=3",
"!ITEM_IOBS=4",
"!ITEM_SIGMA(IOBS)=5",
"!ITEM_XD=6",
"!ITEM_YD=7",
"!ITEM_ZD=8",
"!ITEM_RLP=9",
"!ITEM_PEAK=10",
"!ITEM_CORR=11",
"!ITEM_PSI=12",
"!END_OF_HEADER",
"",
]))
# then write the data records
s0 = Rd * matrix.col(experiment.beam.get_s0())
for j in range(nref):
x, y, z = integrated_data["xyzcal.px"][j]
phi = phi_start + z * phi_range
h, k, l = miller_index[j]
X = (UB * (h, k, l)).rotate(axis, phi, deg=True)
s = s0 + X
g = s.cross(s0).normalize()
# find component of beam perpendicular to f, e
e = -(s + s0).normalize()
if h == k and k == l:
u = (h, -h, 0)
else:
u = (k - l, l - h, h - k)
q = ((matrix.col(u).transpose() *
UB.inverse()).normalize().transpose().rotate(axis, phi,
deg=True))
psi = q.angle(g, deg=True)
if q.dot(e) < 0:
psi *= -1
fout.write("%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n" % (
h,
k,
l,
I[j],
sigI[j],
x,
y,
z,
scl[j],
partiality[j],
prof_corr[j],
psi,
))
fout.write("!END_OF_DATA\n")
fout.close()
logger.info("Output %d reflections to %s" % (nref, filename))
def run(args):
phil = iotbx.phil.process_command_line(
args = args, master_string = master_phil)
work_params = phil.work.extract()
if ("--help" in args) :
libtbx.phil.parse(master_phil.show())
return
if ((work_params.d_min is None) or
(work_params.data is None) or
((work_params.model is None) and
work_params.scaling.algorithm != "mark1")):
raise Usage("cxi.merge "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((work_params.rescale_with_average_cell) and
(not work_params.set_average_unit_cell)) :
raise Usage("If rescale_with_average_cell=True, you must also specify "+
"set_average_unit_cell=True.")
miller_set = symmetry(
unit_cell = work_params.target_unit_cell,
space_group_info = work_params.target_space_group
).build_miller_set(
anomalous_flag = not work_params.merge_anomalous,
d_min = work_params.d_min)
from xfel.cxi.merging.general_fcalc import random_structure
i_model = random_structure(work_params)
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(
miller_set = miller_set,
i_model = i_model,
params = work_params)
scaler.read_all()
sg = miller_set.space_group()
pg = sg.build_derived_laue_group()
rational_ops = []
for symop in pg:
rational_ops.append((matrix.sqr(symop.r().transpose().as_rational()),
symop.r().as_hkl()))
# miller_set.show_summary()
uc = work_params.target_unit_cell
hkl_asu = scaler.observations["hkl_id"]
imageno = scaler.observations["frame_id"]
intensi = scaler.observations["i"]
sigma_i = scaler.observations["sigi"]
lookup = scaler.millers["merged_asu_hkl"]
origH = scaler.observations["H"]
origK = scaler.observations["K"]
origL = scaler.observations["L"]
from cctbx.miller import map_to_asu
sgtype = miller_set.space_group_info().type()
aflag = miller_set.anomalous_flag()
from cctbx.array_family import flex
# FIXME in here perform the mapping to ASU for both the original and other
# index as an array-wise manipulation to make things a bunch faster...
# however this also uses a big chunk of RAM... FIXME also in here use
# cb_op.apply(indices) to get the indices reindexed...
original_indices = flex.miller_index()
for x in xrange(len(scaler.observations["hkl_id"])):
original_indices.append(lookup[hkl_asu[x]])
from cctbx.sgtbx import change_of_basis_op
I23 = change_of_basis_op('k, -h, l')
other_indices = I23.apply(original_indices)
map_to_asu(sgtype, aflag, original_indices)
map_to_asu(sgtype, aflag, other_indices)
# FIXME would be useful in here to have a less expensive way of finding the
# symmetry operation which gave the map to the ASU - perhaps best way is to
# make a new C++ map_to_asu which records this.
# FIXME in here recover the original frame structure of the data to
# logical frame objetcs - N.B. the frame will need to be augmented to test
# alternative indexings
# construct table of start / end indices for frames: now using Python
# range indexing
starts = [0]
ends = []
for x in xrange(1, len(scaler.observations["hkl_id"])):
if imageno[x] != imageno[x - 1]:
ends.append(x)
starts.append(x)
ends.append(len(scaler.observations["hkl_id"]))
keep_start = []
keep_end = []
for j, se in enumerate(zip(starts, ends)):
print 'processing frame %d: %d to %d' % (j, se[0], se[1])
s, e = se
isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
dmin = 100.0
for x in xrange(s, e):
d = uc.d(lookup[hkl_asu[x]])
if d < dmin:
dmin = d
if isig > 6.0 and dmin < 3.2:
keep_start.append(s)
keep_end.append(e)
starts = keep_start
ends = keep_end
print 'Keeping %d frames' % len(starts)
# then start running the comparison code
frames = []
for s, e in zip(starts, ends):
# FIXME need this from remap to ASU
misym = [0 for x in range(s, e)]
indices = [original_indices[x] for x in range(s, e)]
other = [other_indices[x] for x in range(s, e)]
intensities = intensi[s:e]
sigmas = sigma_i[s:e]
frames.append(Frame(uc, indices, other, intensities, sigmas))
reference = FrameFromReferenceMTZ()
fout = open('cc_reference.log', 'w')
for j, f in enumerate(frames):
_cc = reference.cc(f)
_oo = reference.cc_other(f)
print '%d %d %d %d %f %d %f' % (j, starts[j], ends[j], _cc[0], _cc[1],
_oo[0], _oo[1])
fout.write('%d %d %d %d %f %d %f\n' % (j, starts[j], ends[j],
_cc[0], _cc[1], _oo[0], _oo[1]))
fout.close()
return
mtz_dataset.add_miller_array(miller_array=i_obs.select(~remove_sel), column_root_label="ICUT")
mtz_dataset.add_miller_array(miller_array=i_obs.select(remove_sel), column_root_label="IREMOVED")
if f_obs is not None:
mtz_dataset.add_miller_array(miller_array=f_obs.select(~remove_sel), column_root_label="FCUT")
mtz_dataset.add_miller_array(miller_array=f_obs.select(remove_sel), column_root_label="FREMOVED")
mtz_dataset.mtz_object().write(file_name=params.hklout)
if params.xds_ascii is not None:
# XXX Need to check unit cell compatiblity
from yamtbx.dataproc.xds.xds_ascii import XDS_ASCII
from cctbx import miller
xa = XDS_ASCII(params.xds_ascii, sys.stdout)
miller.map_to_asu(xa.symm.space_group_info().type(), False, xa.indices)
removed_indices = i_obs.indices().select(remove_sel)
out = open("removed_positions.dat", "w")
for hkl, x, y, z, i, sigi in zip(xa.indices, xa.xd, xa.yd, xa.zd, xa.iobs, xa.sigma_iobs):
if sigi <= 0:
print "sigi<=0", x, y, z, i, sigi
continue
if hkl in removed_indices:
print >>out, x, y, z, i, sigi
if params.hklref is not None:
#from eval_Rfree_factors_with_common_reflections import get_flag
from cctbx.array_family import flex
calc_r = lambda f_obs, f_model: flex.sum(flex.abs(f_obs.data() - f_model.data())) / flex.sum(f_obs.data())
hklref_arrays = iotbx.mtz.object(params.hklref).as_miller_arrays()
def run(args):
phil = iotbx.phil.process_command_line(
args = args, master_string = master_phil)
work_params = phil.work.extract()
if ("--help" in args) :
libtbx.phil.parse(master_phil.show())
return
if ((work_params.d_min is None) or
(work_params.data is None) or
((work_params.model is None) and
work_params.scaling.algorithm != "mark1")):
raise Usage("cxi.merge "
"d_min=4.0 "
"data=~/scratch/r0220/006/strong/ "
"model=3bz1_3bz2_core.pdb")
if ((work_params.rescale_with_average_cell) and
(not work_params.set_average_unit_cell)) :
raise Usage("If rescale_with_average_cell=True, you must also specify "+
"set_average_unit_cell=True.")
miller_set = symmetry(
unit_cell = work_params.target_unit_cell,
space_group_info = work_params.target_space_group
).build_miller_set(
anomalous_flag = not work_params.merge_anomalous,
d_min = work_params.d_min)
from xfel.cxi.merging.general_fcalc import random_structure
i_model = random_structure(work_params)
# ---- Augment this code with any special procedures for x scaling
scaler = xscaling_manager(
miller_set = miller_set,
i_model = i_model,
params = work_params)
scaler.read_all()
sg = miller_set.space_group()
pg = sg.build_derived_laue_group()
rational_ops = []
for symop in pg:
rational_ops.append((matrix.sqr(symop.r().transpose().as_rational()),
symop.r().as_hkl()))
# miller_set.show_summary()
uc = work_params.target_unit_cell
hkl_asu = scaler.observations["hkl_id"]
imageno = scaler.observations["frame_id"]
intensi = scaler.observations["i"]
sigma_i = scaler.observations["sigi"]
lookup = scaler.millers["merged_asu_hkl"]
origH = scaler.observations["H"]
origK = scaler.observations["K"]
origL = scaler.observations["L"]
from cctbx.miller import map_to_asu
sgtype = miller_set.space_group_info().type()
aflag = miller_set.anomalous_flag()
from cctbx.array_family import flex
# FIXME in here perform the mapping to ASU for both the original and other
# index as an array-wise manipulation to make things a bunch faster...
# however this also uses a big chunk of RAM... FIXME also in here use
# cb_op.apply(indices) to get the indices reindexed...
original_indices = flex.miller_index()
for x in xrange(len(scaler.observations["hkl_id"])):
original_indices.append(lookup[hkl_asu[x]])
from cctbx.sgtbx import change_of_basis_op
I23 = change_of_basis_op('k, -h, l')
other_indices = I23.apply(original_indices)
map_to_asu(sgtype, aflag, original_indices)
map_to_asu(sgtype, aflag, other_indices)
# FIXME would be useful in here to have a less expensive way of finding the
# symmetry operation which gave the map to the ASU - perhaps best way is to
# make a new C++ map_to_asu which records this.
# FIXME in here recover the original frame structure of the data to
# logical frame objetcs - N.B. the frame will need to be augmented to test
# alternative indexings
# construct table of start / end indices for frames: now using Python
# range indexing
starts = [0]
ends = []
for x in xrange(1, len(scaler.observations["hkl_id"])):
if imageno[x] != imageno[x - 1]:
ends.append(x)
starts.append(x)
ends.append(len(scaler.observations["hkl_id"]))
keep_start = []
keep_end = []
for j, se in enumerate(zip(starts, ends)):
print 'processing frame %d: %d to %d' % (j, se[0], se[1])
s, e = se
isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s)
dmin = 100.0
for x in xrange(s, e):
d = uc.d(lookup[hkl_asu[x]])
if d < dmin:
dmin = d
if isig > 6.0 and dmin < 3.2:
keep_start.append(s)
keep_end.append(e)
starts = keep_start
ends = keep_end
print 'Keeping %d frames' % len(starts)
# then start running the comparison code
frames = []
for s, e in zip(starts, ends):
# FIXME need this from remap to ASU
misym = [0 for x in range(s, e)]
indices = [original_indices[x] for x in range(s, e)]
other = [other_indices[x] for x in range(s, e)]
intensities = intensi[s:e]
sigmas = sigma_i[s:e]
frames.append(Frame(uc, indices, other, intensities, sigmas))
cycle = 0
total_nref = sum([len(f.get_indices()) for f in frames])
# pre-scale the data - first determine average ln(k), B; then apply
kbs = [f.kb() for f in frames]
mn_k = sum([kb[0] for kb in kbs]) / len(kbs)
mn_B = sum([kb[1] for kb in kbs]) / len(kbs)
for f in frames:
f.scale_to_kb(mn_k, mn_B)
while True:
print 'Analysing %d frames' % len(frames)
print 'Cycle %d' % cycle
cycle += 1
print 'Power spectrum'
fn = frame_numbers(frames)
for j in sorted(fn):
print '%4d %4d' % (j, fn[j])
nref_cycle = sum([len(f.get_indices()) for f in frames])
assert(nref_cycle == total_nref)
# first work on the original indices
import numpy
common_reflections = numpy.zeros((len(frames), len(frames)),
dtype = numpy.short)
obs = { }
# for other hand add -j
for j, f in enumerate(frames):
indices = set(f.get_indices())
for i in indices:
_i = tuple(i)
if not _i in obs:
obs[_i] = []
obs[_i].append(j)
for hkl in obs:
obs[hkl].sort()
for j, f1 in enumerate(obs[hkl][:-1]):
for f2 in obs[hkl][j + 1:]:
if f1 * f2 > 0:
common_reflections[(abs(f1), abs(f2))] += 1
cmn_rfl_list = []
for f1 in range(len(frames)):
for f2 in range(f1 + 1, len(frames)):
if common_reflections[(f1, f2)] > 10:
cmn_rfl_list.append((common_reflections[(f1, f2)], f1, f2))
cmn_rfl_list.sort()
cmn_rfl_list.reverse()
joins = []
used = []
for n, f1, f2 in cmn_rfl_list:
if f1 in used or f2 in used:
continue
_cc = frames[f1].cc(frames[f2])
# really only need to worry about f2 which will get merged...
# merging multiple files together should be OK provided they are
# correctly sorted (though the order should not matter anyhow?)
# anyhow they are sorted anyway... ah as f2 > f1 then just sorting
# the list by f2 will make sure the data cascase correctly.
# p-value small (3% ish) for cc > 0.6 for > 10 observations -
# necessary as will be correlated due to Wilson curves though
# with B factor < 10 this is less of an issue
if _cc[0] > 10 and _cc[1] > 0.6:
print '%4d %.3f' % _cc, f1, f2
joins.append((f2, f1))
used.append(f2)
if not joins:
print 'No pairs found'
break
joins.sort()
joins.reverse()
for j2, j1 in joins:
rmerge = frames[j1].merge(frames[j2])
if rmerge:
print 'R: %4d %4d %6.3f' % (j1, j2, rmerge)
else:
print 'R: %4d %4d ------' % (j1, j2)
all_joins = [j for j in joins]
# then do the same for the alternative indices
other_reflections = numpy.zeros((len(frames), len(frames)),
dtype = numpy.short)
obs = { }
# for other hand add -j
for j, f in enumerate(frames):
indices = set(f.get_indices())
for i in indices:
_i = tuple(i)
if not _i in obs:
obs[_i] = []
obs[_i].append(j)
indices = set(f.get_other())
for i in indices:
_i = tuple(i)
if not _i in obs:
obs[_i] = []
obs[_i].append(-j)
for hkl in obs:
obs[hkl].sort()
for j, f1 in enumerate(obs[hkl][:-1]):
for f2 in obs[hkl][j + 1:]:
if f1 * f2 < 0:
other_reflections[(abs(f1), abs(f2))] += 1
oth_rfl_list = []
for f1 in range(len(frames)):
for f2 in range(f1 + 1, len(frames)):
if other_reflections[(f1, f2)] > 10:
oth_rfl_list.append((other_reflections[(f1, f2)], f1, f2))
joins = []
oth_rfl_list.sort()
oth_rfl_list.reverse()
for n, f1, f2 in oth_rfl_list:
if f1 in used or f2 in used:
continue
_cc = frames[f1].cc_other(frames[f2])
# really only need to worry about f2 which will get merged...
# merging multiple files together should be OK provided they are
# correctly sorted (though the order should not matter anyhow?)
# anyhow they are sorted anyway... ah as f2 > f1 then just sorting
# the list by f2 will make sure the data cascase correctly.
# p-value small (3% ish) for cc > 0.6 for > 10 observations -
# necessary as will be correlated due to Wilson curves though
# with B factor < 10 this is less of an issue
if _cc[0] > 10 and _cc[1] > 0.6:
print '%4d %.3f' % _cc, f1, f2
joins.append((f2, f1))
used.append(f2)
all_joins += joins
if not all_joins:
break
joins.sort()
joins.reverse()
for j2, j1 in joins:
frames[j2].reindex()
rmerge = frames[j1].merge(frames[j2])
if rmerge:
print 'R: %4d %4d %6.3f' % (j1, j2, rmerge)
else:
print 'R: %4d %4d ------' % (j1, j2)
continue
frames.sort()
print 'Biggest few: #frames; #unique refl'
j = -1
while frames[j].get_frames() > 1:
print frames[j].get_frames(), frames[j].get_unique_indices()
frames[j].output_as_scalepack(sg, 'scalepack-%d.sca' % j)
j -= 1
return
def _compute_rij_wij(self, use_cache=True):
"""Compute the rij_wij matrix."""
n_lattices = self._lattices.size()
n_sym_ops = len(self._sym_ops)
NN = n_lattices * n_sym_ops
self.rij_matrix = flex.double(flex.grid(NN, NN), 0.0)
if self._weights is None:
self.wij_matrix = None
else:
self.wij_matrix = flex.double(flex.grid(NN, NN), 0.0)
indices = {}
space_group_type = self._data.space_group().type()
for cb_op in self._sym_ops:
cb_op = sgtbx.change_of_basis_op(cb_op)
indices_reindexed = cb_op.apply(self._data.indices())
miller.map_to_asu(space_group_type, False, indices_reindexed)
indices[cb_op.as_xyz()] = indices_reindexed
def _compute_rij_matrix_one_row_block(i):
rij_cache = {}
n_sym_ops = len(self._sym_ops)
NN = n_lattices * n_sym_ops
from scipy import sparse
rij_row = []
rij_col = []
rij_data = []
if self._weights is not None:
wij_row = []
wij_col = []
wij_data = []
else:
wij = None
i_lower, i_upper = self._lattice_lower_upper_index(i)
intensities_i = self._data.data()[i_lower:i_upper]
for j in range(n_lattices):
j_lower, j_upper = self._lattice_lower_upper_index(j)
intensities_j = self._data.data()[j_lower:j_upper]
for k, cb_op_k in enumerate(self._sym_ops):
cb_op_k = sgtbx.change_of_basis_op(cb_op_k)
indices_i = indices[cb_op_k.as_xyz()][i_lower:i_upper]
for kk, cb_op_kk in enumerate(self._sym_ops):
if i == j and k == kk:
# don't include correlation of dataset with itself
continue
cb_op_kk = sgtbx.change_of_basis_op(cb_op_kk)
ik = i + (n_lattices * k)
jk = j + (n_lattices * kk)
key = (i, j, str(cb_op_k.inverse() * cb_op_kk))
if use_cache and key in rij_cache:
cc, n = rij_cache[key]
else:
indices_j = indices[
cb_op_kk.as_xyz()][j_lower:j_upper]
matches = miller.match_indices(
indices_i, indices_j)
pairs = matches.pairs()
isel_i = pairs.column(0)
isel_j = pairs.column(1)
isel_i = isel_i.select(
self._patterson_group.epsilon(
indices_i.select(isel_i)) == 1)
isel_j = isel_j.select(
self._patterson_group.epsilon(
indices_j.select(isel_j)) == 1)
corr = flex.linear_correlation(
intensities_i.select(isel_i),
intensities_j.select(isel_j),
)
if corr.is_well_defined():
cc = corr.coefficient()
n = corr.n()
rij_cache[key] = (cc, n)
else:
cc = None
n = None
if n < self._min_pairs:
continue
if cc is not None and n is not None:
if self._weights == "count":
wij_row.extend([ik, jk])
wij_col.extend([jk, ik])
wij_data.extend([n, n])
elif self._weights == "standard_error":
assert n > 2
# http://www.sjsu.edu/faculty/gerstman/StatPrimer/correlation.pdf
se = math.sqrt((1 - cc**2) / (n - 2))
wij = 1 / se
wij_row.extend([ik, jk])
wij_col.extend([jk, ik])
wij_data.extend([wij, wij])
rij_row.append(ik)
rij_col.append(jk)
rij_data.append(cc)
rij = sparse.coo_matrix((rij_data, (rij_row, rij_col)),
shape=(NN, NN))
if self._weights is not None:
wij = sparse.coo_matrix((wij_data, (wij_row, wij_col)),
shape=(NN, NN))
return rij, wij
from libtbx import easy_mp
args = [(i, ) for i in range(n_lattices)]
results = easy_mp.parallel_map(
_compute_rij_matrix_one_row_block,
args,
processes=self._nproc,
iterable_type=easy_mp.posiargs,
method="multiprocessing",
)
rij_matrix = None
wij_matrix = None
for i, (rij, wij) in enumerate(results):
if rij_matrix is None:
rij_matrix = rij
else:
rij_matrix += rij
if wij is not None:
if wij_matrix is None:
wij_matrix = wij
else:
wij_matrix += wij
self.rij_matrix = flex.double(rij_matrix.todense())
if wij_matrix is not None:
import numpy as np
self.wij_matrix = flex.double(wij_matrix.todense().astype(
np.float64))
return self.rij_matrix, self.wij_matrix
def export_xds_ascii(integrated_data,
experiment_list,
params,
var_model=(1, 0)):
"""Export data from integrated_data corresponding to experiment_list to
an XDS_ASCII.HKL formatted text file."""
from dials.array_family import flex
# for the moment assume (and assert) that we will convert data from exactly
# one lattice...
assert len(experiment_list) == 1
# select reflections that are assigned to an experiment (i.e. non-negative id)
integrated_data = integrated_data.select(integrated_data["id"] >= 0)
assert max(integrated_data["id"]) == 0
# export for xds_ascii should only be for non-scaled reflections
assert any([
i in integrated_data
for i in ["intensity.sum.value", "intensity.prf.value"]
])
integrated_data = filter_reflection_table(
integrated_data,
intensity_choice=params.intensity,
partiality_threshold=params.mtz.partiality_threshold,
combine_partials=params.mtz.combine_partials,
min_isigi=params.mtz.min_isigi,
filter_ice_rings=params.mtz.filter_ice_rings,
d_min=params.mtz.d_min,
)
# calculate the scl = lp/dqe correction for outputting but don't apply it as
# it has already been applied in filter_reflection_table
integrated_data, scl = FilteringReductionMethods.calculate_lp_qe_correction_and_filter(
integrated_data)
experiment = experiment_list[0]
# sort data before output
nref = len(integrated_data["miller_index"])
indices = flex.size_t_range(nref)
import copy
unique = copy.deepcopy(integrated_data["miller_index"])
from cctbx.miller import map_to_asu
map_to_asu(experiment.crystal.get_space_group().type(), False, unique)
perm = sorted(indices, key=lambda k: unique[k])
integrated_data = integrated_data.select(flex.size_t(perm))
from scitbx import matrix
from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame
assert not experiment.goniometer is None
unit_cell = experiment.crystal.get_unit_cell()
from scitbx.array_family import flex
assert not experiment.scan is None
image_range = experiment.scan.get_image_range()
phi_start, phi_range = experiment.scan.get_image_oscillation(
image_range[0])
# gather the required information for the reflection file
nref = len(integrated_data["miller_index"])
zdet = flex.double(integrated_data["xyzcal.px"].parts()[2])
miller_index = integrated_data["miller_index"]
# profile correlation
if "profile.correlation" in integrated_data:
prof_corr = 100.0 * integrated_data["profile.correlation"]
else:
prof_corr = flex.double(nref, 100.0)
# partiality
if "partiality" in integrated_data:
partiality = 100 * integrated_data["partiality"]
else:
prof_corr = flex.double(nref, 100.0)
if "intensity.sum.value" in integrated_data:
I = integrated_data["intensity.sum.value"]
V = integrated_data["intensity.sum.variance"]
assert V.all_gt(0)
V = var_model[0] * (V + var_model[1] * I * I)
sigI = flex.sqrt(V)
else:
I = integrated_data["intensity.prf.value"]
V = integrated_data["intensity.prf.variance"]
assert V.all_gt(0)
V = var_model[0] * (V + var_model[1] * I * I)
sigI = flex.sqrt(V)
fout = open(params.xds_ascii.hklout, "w")
# first write the header - in the "standard" coordinate frame...
panel = experiment.detector[0]
fast = panel.get_fast_axis()
slow = panel.get_slow_axis()
Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0))
print("Coordinate change:")
print("%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n" %
Rd.elems)
fast = Rd * fast
slow = Rd * slow
qx, qy = panel.get_pixel_size()
nx, ny = panel.get_image_size()
distance = matrix.col(Rd * panel.get_origin()).dot(
matrix.col(Rd * panel.get_normal()))
org = Rd * (matrix.col(panel.get_origin()) -
distance * matrix.col(panel.get_normal()))
orgx = -org.dot(fast) / qx
orgy = -org.dot(slow) / qy
UB = Rd * matrix.sqr(experiment.crystal.get_A())
real_space_ABC = UB.inverse().elems
axis = Rd * experiment.goniometer.get_rotation_axis()
beam = Rd * experiment.beam.get_s0()
cell_fmt = "%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f"
axis_fmt = "%9.3f %9.3f %9.3f"
fout.write("\n".join([
"!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL'S_LAW=TRUE",
"!Generated by dials.export",
"!DATA_RANGE= %d %d" % image_range,
"!ROTATION_AXIS= %9.6f %9.6f %9.6f" % axis.elems,
"!OSCILLATION_RANGE= %f" % phi_range,
"!STARTING_ANGLE= %f" % phi_start,
"!STARTING_FRAME= %d" % image_range[0],
"!SPACE_GROUP_NUMBER= %d" %
experiment.crystal.get_space_group().type().number(),
"!UNIT_CELL_CONSTANTS= %s" % (cell_fmt % unit_cell.parameters()),
"!UNIT_CELL_A-AXIS= %s" % (axis_fmt % real_space_ABC[0:3]),
"!UNIT_CELL_B-AXIS= %s" % (axis_fmt % real_space_ABC[3:6]),
"!UNIT_CELL_C-AXIS= %s" % (axis_fmt % real_space_ABC[6:9]),
"!X-RAY_WAVELENGTH= %f" % experiment.beam.get_wavelength(),
"!INCIDENT_BEAM_DIRECTION= %f %f %f" % beam.elems,
"!NX= %d NY= %d QX= %f QY= %f" % (nx, ny, qx, qy),
"!ORGX= %9.2f ORGY= %9.2f" % (orgx, orgy),
"!DETECTOR_DISTANCE= %8.3f" % distance,
"!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f" % fast.elems,
"!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f" % slow.elems,
"!VARIANCE_MODEL= %7.3e %7.3e" % var_model,
"!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12",
"!ITEM_H=1",
"!ITEM_K=2",
"!ITEM_L=3",
"!ITEM_IOBS=4",
"!ITEM_SIGMA(IOBS)=5",
"!ITEM_XD=6",
"!ITEM_YD=7",
"!ITEM_ZD=8",
"!ITEM_RLP=9",
"!ITEM_PEAK=10",
"!ITEM_CORR=11",
"!ITEM_PSI=12",
"!END_OF_HEADER",
"",
]))
# then write the data records
s0 = Rd * matrix.col(experiment.beam.get_s0())
for j in range(nref):
x, y, z = integrated_data["xyzcal.px"][j]
phi = phi_start + z * phi_range
h, k, l = miller_index[j]
X = (UB * (h, k, l)).rotate(axis, phi, deg=True)
s = s0 + X
g = s.cross(s0).normalize()
f = (s - s0).normalize()
# find component of beam perpendicular to f, e
e = -(s + s0).normalize()
if h == k and k == l:
u = (h, -h, 0)
else:
u = (k - l, l - h, h - k)
q = ((matrix.col(u).transpose() *
UB.inverse()).normalize().transpose().rotate(axis, phi,
deg=True))
psi = q.angle(g, deg=True)
if q.dot(e) < 0:
psi *= -1
fout.write("%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n" % (
h,
k,
l,
I[j],
sigI[j],
x,
y,
z,
scl[j],
partiality[j],
prof_corr[j],
psi,
))
fout.write("!END_OF_DATA\n")
fout.close()
logger.info("Output %d reflections to %s" %
(nref, params.xds_ascii.hklout))
def exercise_asu():
sg_type = sgtbx.space_group_type("P 41")
asu = sgtbx.reciprocal_space_asu(sg_type)
miller_indices = flex.miller_index(((1,2,3), (3,5,0)))
for h in miller_indices:
h_eq = miller.sym_equiv_indices(sg_type.group(), h)
for i_eq in xrange(h_eq.multiplicity(False)):
h_i = h_eq(i_eq)
for anomalous_flag in (False,True):
a = miller.asym_index(sg_type.group(), asu, h_i.h())
assert a.h() == h
o = a.one_column(anomalous_flag)
assert o.i_column() == 0
t = a.two_column(anomalous_flag)
assert t.h() == h
assert (o.h() != h) == (t.i_column() == 1)
assert not anomalous_flag or (t.i_column() != 0) == h_i.friedel_flag()
assert anomalous_flag or t.i_column() == 0
miller.map_to_asu(sg_type, False, miller_indices)
data = flex.double((0,0))
miller.map_to_asu(sg_type, False, miller_indices, data)
miller.map_to_asu(sg_type, False, miller_indices, data, False)
miller.map_to_asu(sg_type, False, miller_indices, data, True)
data = flex.complex_double((0,0))
miller.map_to_asu(sg_type, False, miller_indices, data)
data = flex.hendrickson_lattman(((1,2,3,4),(2,3,4,5)))
miller.map_to_asu(sg_type, False, miller_indices, data)
for sg_symbol in ("P 41", "P 31 1 2"):
exercise_map_to_asu(sg_symbol)
#
sg_type = sgtbx.space_group_type("P 2")
miller_indices = flex.miller_index(((1,2,3), (-1,-2,-3)))
assert not miller.is_unique_set_under_symmetry(
space_group_type=sg_type,
anomalous_flag=False,
miller_indices=miller_indices)
assert miller.is_unique_set_under_symmetry(
space_group_type=sg_type,
anomalous_flag=True,
miller_indices=miller_indices)
assert list(miller.unique_under_symmetry_selection(
space_group_type=sg_type,
anomalous_flag=False,
miller_indices=miller_indices)) == [0]
assert list(miller.unique_under_symmetry_selection(
space_group_type=sg_type,
anomalous_flag=True,
miller_indices=miller_indices)) == [0,1]
def exercise_asu():
sg_type = sgtbx.space_group_type("P 41")
asu = sgtbx.reciprocal_space_asu(sg_type)
miller_indices = flex.miller_index(((1,2,3), (3,5,0)))
for h in miller_indices:
h_eq = miller.sym_equiv_indices(sg_type.group(), h)
for i_eq in range(h_eq.multiplicity(False)):
h_i = h_eq(i_eq)
for anomalous_flag in (False,True):
a = miller.asym_index(sg_type.group(), asu, h_i.h())
assert a.h() == h
o = a.one_column(anomalous_flag)
assert o.i_column() == 0
t = a.two_column(anomalous_flag)
assert t.h() == h
assert (o.h() != h) == (t.i_column() == 1)
assert not anomalous_flag or (t.i_column() != 0) == h_i.friedel_flag()
assert anomalous_flag or t.i_column() == 0
miller.map_to_asu(sg_type, False, miller_indices)
data = flex.double((0,0))
miller.map_to_asu(sg_type, False, miller_indices, data)
miller.map_to_asu(sg_type, False, miller_indices, data, False)
miller.map_to_asu(sg_type, False, miller_indices, data, True)
data = flex.complex_double((0,0))
miller.map_to_asu(sg_type, False, miller_indices, data)
data = flex.hendrickson_lattman(((1,2,3,4),(2,3,4,5)))
miller.map_to_asu(sg_type, False, miller_indices, data)
for sg_symbol in ("P 41", "P 31 1 2"):
exercise_map_to_asu(sg_symbol)
#
sg_type = sgtbx.space_group_type("P 2")
miller_indices = flex.miller_index(((1,2,3), (-1,-2,-3)))
assert not miller.is_unique_set_under_symmetry(
space_group_type=sg_type,
anomalous_flag=False,
miller_indices=miller_indices)
assert miller.is_unique_set_under_symmetry(
space_group_type=sg_type,
anomalous_flag=True,
miller_indices=miller_indices)
assert list(miller.unique_under_symmetry_selection(
space_group_type=sg_type,
anomalous_flag=False,
miller_indices=miller_indices)) == [0]
assert list(miller.unique_under_symmetry_selection(
space_group_type=sg_type,
anomalous_flag=True,
miller_indices=miller_indices)) == [0,1]
