def compute_f_calc(fmodel, params):
from cctbx import miller
coeffs_partial_set = fmodel.f_obs().structure_factors_from_scatterers(
xray_structure = fmodel.xray_structure).f_calc()
if(hasattr(params,"dev") and params.dev.complete_set_up_to_d_min):
coeffs = fmodel.xray_structure.structure_factors(
d_min = fmodel.f_obs().d_min()).f_calc()
frac_inc = 1.*coeffs_partial_set.data().size()/coeffs.data().size()
n_miss = coeffs.data().size() - coeffs_partial_set.data().size()
if(params.dev.aply_same_incompleteness_to_complete_set_at == "randomly"):
sel = flex.random_bool(coeffs.data().size(), frac_inc)
coeffs = coeffs.select(sel)
elif(params.dev.aply_same_incompleteness_to_complete_set_at == "low"):
coeffs = coeffs.sort()
coeffs = miller.set(
crystal_symmetry = coeffs,
indices = coeffs.indices()[n_miss+1:],
anomalous_flag = coeffs.anomalous_flag()).array(
data = coeffs.data()[n_miss+1:])
elif(params.dev.aply_same_incompleteness_to_complete_set_at == "high"):
coeffs = coeffs.sort(reverse=True)
coeffs = miller.set(
crystal_symmetry = coeffs,
indices = coeffs.indices()[n_miss+1:],
anomalous_flag = coeffs.anomalous_flag()).array(
data = coeffs.data()[n_miss+1:])
else:
coeffs = coeffs_partial_set
return coeffs
def as_miller_arrays(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
from cctbx import miller
from cctbx import crystal
if (crystal_symmetry is None):
crystal_symmetry = crystal.symmetry()
if (base_array_info is None):
base_array_info = miller.array_info(source_type="shelx_hklf")
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=self.indices()).auto_anomalous()
miller_arrays = []
obs = (miller.array(
miller_set=miller_set,
data=self.data(),
sigmas=self.sigmas())
.set_info(base_array_info.customized_copy(labels=["obs", "sigmas"])))
miller_arrays.append(obs)
if (self.alphas() is not None):
miller_arrays.append(miller.array(
miller_set=miller_set,
data=self.alphas())
.set_info(base_array_info.customized_copy(labels=["alphas"])))
return miller_arrays
def exercise():
ma = miller.array(
miller.set(crystal.symmetry(unit_cell=(5,5,5, 90, 90, 90),
space_group=sgtbx.space_group('P 2x')),
indices=flex.miller_index(
[(1,0,0), (0,1,0), (0,0,1),
(-1,0,0), (0,-1,0), (0,0,-1),
(1,1,0), (1,0,1), (0,1,1),
(-1,-1,0), (-1,0,-1), (0,-1,-1),
(1,-1,0), (1,0,-1), (0,1,-1),
(-1,1,0), (-1,0,1), (0,-1,1),
(1,1,1), (-1,1,1), (1,-1,1), (1,1,-1),
(-1,-1,-1), (1,-1,-1), (-1,1,-1), (-1,-1,1)])),
data=flex.complex_double(flex.random_double(26), flex.random_double(26)))
f_at_h = dict(zip(ma.indices(), ma.data()))
for op in ("-x, y+1/2, -z", "x+1/2, -y, z-1/2"):
op = sgtbx.rt_mx(op)
original, transformed = ma.common_sets(
ma.change_basis(sgtbx.change_of_basis_op(op.inverse())))
for h, f in original:
assert f == f_at_h[h]
for h, op_f in transformed:
assert approx_equal(
op_f,
f_at_h[h*op.r()]*exp(1j*2*pi*row(h).dot(col(op.t().as_double()))))
def tst_pgtools():
unit_cell = uctbx.unit_cell('40, 40, 60, 90.0, 90.0, 90.0')
mi = flex.miller_index(((2,4,6), (2,4,8)))
xs = crystal.symmetry(unit_cell, "P 1 2 1")
ms = miller.set(xs, mi)
# Go to the minimum cell, for safety
cob_min_cell = ms.change_of_basis_op_to_minimum_cell()
ms_new = ms.change_basis( cob_min_cell )
lattice_group = sgtbx.lattice_symmetry.group(
ms_new.unit_cell(),
max_delta=5.0)
point_group_low = ms_new.space_group().build_derived_point_group()
point_group_high = lattice_group.build_derived_point_group()
pgtree = pt.point_group_graph(point_group_low,point_group_high)
# find the possible routes from 'P 2' to 'P 4 2 2'
atlas = pgtree.graph.find_all_paths( 'P 1 2 1', 'P 4 2 2')
route_1 = ['P 1 2 1', 'P 4 2 2']
route_2 = ['P 1 2 1', 'P 2 2 2', 'P 4 2 2']
assert route_1 in atlas
assert route_2 in atlas
assert len(atlas)==2
# Now lets 'disqualify' point group 'P 2 2 2'
pgtree.remove_point_group_and_its_super_groups_from_graph(
str(sgtbx.space_group_info(16)))
assert len(pgtree.graph.node_objects)==1
assert pgtree.graph.node_objects.has_key ( 'P 1 2 1' )
def as_miller_array(self,
crystal_symmetry=None,
force_symmetry=False,
merge_equivalents=True,
base_array_info=None):
if (base_array_info is None):
base_array_info = miller.array_info(source_type="scalepack_merge")
crystal_symmetry_from_file = self.crystal_symmetry()
if (self.anomalous):
labels = ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]
else:
labels = ["I", "SIGI"]
return (miller.array(
miller_set=miller.set(
crystal_symmetry=crystal_symmetry_from_file.join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry),
indices=self.miller_indices,
anomalous_flag=self.anomalous),
data=self.i_obs,
sigmas=self.sigmas)
.set_info(base_array_info.customized_copy(
labels=labels,
crystal_symmetry_from_file=crystal_symmetry_from_file))
.set_observation_type_xray_intensity())
def randomize_struture_factors(map_coeffs, number_of_kicks, phases_only=False):
map_coeff_data = None
for kick in xrange(number_of_kicks):
rc, ar, pr = random.choice([(0.1, 0.10, 40),
(0.2, 0.09, 40),
(0.3, 0.08, 30),
(0.4, 0.07, 30),
(0.5, 0.06, 20),
(0.6, 0.05, 20),
(0.7, 0.04, 20),
(0.8, 0.03, 10),
(0.9, 0.02, 10),
(1.0, 0.01, 10)
])
if(phases_only): ar = 0
sel = flex.random_bool(map_coeffs.size(), rc)
mc = map_coeffs.randomize_amplitude_and_phase(
amplitude_error=ar, phase_error_deg=pr, selection=sel)
if(map_coeff_data is None): map_coeff_data = mc.data()
else: map_coeff_data = map_coeff_data + mc.data()
map_coeff_data = map_coeff_data/number_of_kicks
return miller.set(
crystal_symmetry = map_coeffs.crystal_symmetry(),
indices = map_coeffs.indices(),
anomalous_flag = False).array(data = map_coeff_data)
def df2m(df, cell, spgr, data=None, sigmas=None):
"""Constructs a miller.array from the columns specified by data/sigmas in the dataframe,
if both are None, returns just the indices.
needs cell and spgr to generate a symmetry object."""
anomalous_flag = False
if isinstance(df, pd.DataFrame):
try:
sel = df[data].notnull() # select notnull data items for index
except ValueError:
index = df.index
else:
index = df.index[sel]
else:
index = df
indices = flex.miller_index(index)
if data:
data = flex.double(df[data][sel])
if sigmas:
sigmas = flex.double(df[sigmas][sel])
symm = make_symmetry(cell, spgr)
ms = miller.set(
crystal_symmetry=symm, indices=indices, anomalous_flag=anomalous_flag)
return miller.array(ms, data=data, sigmas=sigmas)
def as_mtz(self,scale_factor,prefix):
#This reads in an hkl map and returns a .mtz map
#Read in hkl file and populate miller array
inf = open(self.diffuse_file, 'r')
indices = flex.miller_index()
i_obs = flex.double()
sig_i = flex.double()
for line in inf.readlines():
assert len(line.split())==4
line = line.strip().split()
#####ATTENTION:SCALE FACTOR##############
i_obs_ = float(line[3])/scale_factor #is a uniform scale factor meant to re-size all diffuse intensities (normally too large for scalepack)
sig_i_ = math.sqrt(i_obs_)
#if(abs(i_obs_)>1.e-6): # perhaps you don't want zeros
indices.append([int(line[0]),int(line[1]),int(line[2])])
i_obs.append(i_obs_)
sig_i.append(sig_i_)
inf.close()
# get miller array object
cs = self.symmetry
ma = miller.array(miller_set=miller.set(cs, indices), data=i_obs, sigmas=sig_i)
ma.set_observation_type_xray_intensity()
mtz_dataset = ma.as_mtz_dataset(column_root_label="I")
mtz_dataset.mtz_object().write(prefix + '.mtz')
def as_miller_set(self, anomalous_flag=None):
if anomalous_flag is None:
anomalous_flag = self.anomalous
return miller.set(crystal_symmetry=self.symm,
indices=self.indices,
anomalous_flag=anomalous_flag)
def __init__(self):
self.data_obs1 = flex.double(2,1.0)
self.data_obs2 = flex.double(2,3.0)
self.sigma_obs1 = flex.double(2,0.1)
self.sigma_obs2 = flex.double(2,1)
self.unit_cell = uctbx.unit_cell('20, 30, 40, 90.0, 90.0, 90.0')
#mi = flex.miller_index(((1,2,3), (1,2,3)))
self.mi = flex.miller_index(((1,2,3), (5,6,7)))
self.xs = crystal.symmetry((20,30,40), "P 2 2 2")
self.ms = miller.set(self.xs, self.mi)
self.u = [1,2,3,4,5,6]
self.p_scale = 0.40
#self.u = [0,0,0,0,0,0]
#self.p_scale = 0.00
self.ls_i_wt = scaling.least_squares_on_i_wt(
self.mi,
self.data_obs1,
self.sigma_obs1,
self.data_obs2,
self.sigma_obs2,
self.p_scale,
self.unit_cell,
self.u)
self.ls_i = scaling.least_squares_on_i(
self.mi,
self.data_obs1,
self.sigma_obs1,
self.data_obs2,
self.sigma_obs2,
self.p_scale,
self.unit_cell,
self.u)
self.ls_f_wt = scaling.least_squares_on_f_wt(
self.mi,
self.data_obs1,
self.sigma_obs1,
self.data_obs2,
self.sigma_obs2,
self.p_scale,
self.unit_cell,
self.u)
self.ls_f = scaling.least_squares_on_f(
self.mi,
self.data_obs1,
self.sigma_obs1,
self.data_obs2,
self.sigma_obs2,
self.p_scale,
self.unit_cell,
self.u)
self.tst_ls_f_wt()
self.tst_ls_i_wt()
self.tst_ls_f()
self.tst_ls_i()
self.tst_hes_ls_i_wt()
self.tst_hes_ls_f_wt()
self.tst_hes_ls_i()
self.tst_hes_ls_f()
def cluster_completeness(self, clno, anomalous_flag, calc_redundancy=True):
if clno not in self.clusters:
print "Cluster No. %d not found" % clno
return
cls = self.clusters[clno][3]
msets = map(lambda x: self.miller_sets[self.files[x-1]], cls)
num_idx = sum(map(lambda x: x.size(), msets))
all_idx = flex.miller_index()
all_idx.reserve(num_idx)
for mset in msets: all_idx.extend(mset.indices())
# Calc median cell
cells = numpy.array(map(lambda x: x.unit_cell().parameters(), msets))
median_cell = map(lambda i: numpy.median(cells[:,i]), xrange(6))
symm = msets[0].customized_copy(unit_cell=median_cell)
assert anomalous_flag is not None
# XXX all must belong to the same Laue group and appropriately reindexed..
all_set = miller.set(indices=all_idx,
crystal_symmetry=symm, anomalous_flag=anomalous_flag)
all_set = all_set.resolution_filter(d_min=self.d_min)
# dummy for redundancy calculation. dirty way..
if calc_redundancy:
dummy_array = miller.array(miller_set=all_set, data=flex.int(all_set.size()))
merge = dummy_array.merge_equivalents()
cmpl = merge.array().completeness()
redun = merge.redundancies().as_double().mean()
return cmpl, redun
else:
cmpl = all_set.unique_under_symmetry().completeness()
return cmpl
def merge_obs(indices, iobs, sel, symm, anomalous_flag, d_min, d_max):
#indices = flex.miller_index(indices)
#iobs = flex.double(iobs)
if sel is not None and len(sel) > 0:
sel = flex.bool(sel)
indices = indices.select(sel)
iobs = iobs.select(sel)
miller_set = miller.set(crystal_symmetry=symm,
indices=indices,
anomalous_flag=anomalous_flag)
array = miller.array(miller_set=miller_set,
data=iobs,
).set_observation_type_xray_intensity()
array = array.resolution_filter(d_min=d_min, d_max=d_max)
## New way
#array = array.map_to_asu()
#
#merger = yamtbx_dataproc_crystfel_ext.merge_equivalents_crystfel()
#merger.add_observations(array.indices(), array.data())
#merger.merge()
return array.merge_equivalents(algorithm="crystfel") # if sigmas is None, merge_equivalents_real() is used which simply averages.
def exercise(prefix="tst_models_to_from_chains"):
# inputs
expected_n = 5
xrs = iotbx.pdb.input(source_info=None, lines=pdb_str).xray_structure_simple()
input_file_name = "%s.pdb"%prefix
of = open(input_file_name,"w")
print >> of, pdb_str
of.close()
mi = flex.miller_index(((0,0,1), ))
ms = miller.set(xrs.crystal_symmetry(), mi, anomalous_flag=False)
complete_set = ms.complete_set(d_min=3)
fc = complete_set.structure_factors_from_scatterers(
xray_structure=xrs).f_calc()
# models -> chains
easy_run.call("phenix.models_as_chains %s"%input_file_name)
pdb_inp = iotbx.pdb.input(file_name="chains_"+input_file_name)
h = pdb_inp.construct_hierarchy()
assert len(list(h.chains()))==expected_n
assert len(list(h.models()))==1
xrs_c = pdb_inp.xray_structure_simple()
fc_c = complete_set.structure_factors_from_scatterers(
xray_structure=xrs_c).f_calc()
#
easy_run.call("phenix.chains_as_models chains_%s"%input_file_name)
pdb_inp = iotbx.pdb.input(file_name="models_chains_"+input_file_name)
h = pdb_inp.construct_hierarchy()
assert len(list(h.chains()))==expected_n
assert len(list(h.models()))==expected_n
xrs_m = pdb_inp.xray_structure_simple()
fc_m = complete_set.structure_factors_from_scatterers(
xray_structure=xrs_m).f_calc()
#
assert approx_equal(0, r(fc, fc_c) )
assert approx_equal(0, r(fc, fc_m) )
assert approx_equal(0, r(fc_c, fc_m))
def extend_symmetry(self,scale_factor,prefix):
#This reads in an hkl map and returns a .sca map
#Read in hkl file and populate miller array
inf = open(self.diffuse_file, 'r')
indices = flex.miller_index()
i_obs = flex.double()
sig_i = flex.double()
for line in inf.readlines():
assert len(line.split())==4
line = line.strip().split()
#####ATTENTION:SCALE FACTOR##############
i_obs_ = float(line[3])/scale_factor #is a uniform scale factor meant to re-size all diffuse intensities (normally too large for scalepack)
sig_i_ = math.sqrt(i_obs_)
#if(abs(i_obs_)>1.e-6): # perhaps you don't want zeros
indices.append([int(line[0]),int(line[1]),int(line[2])])
i_obs.append(i_obs_)
sig_i.append(sig_i_)
inf.close()
# get miller array object
cs = self.symmetry
ma = miller.array(miller_set=miller.set(cs, indices), data=i_obs, sigmas=sig_i)
ma.set_observation_type_xray_intensity()
ma_anom = ma.customized_copy(anomalous_flag=False)
ma_p1 = ma_anom.expand_to_p1()
merge.write(file_name= prefix + '.sca', miller_array=ma_p1)
self.p1_map = prefix + '.sca'
def aniso_convert(uc,sg,map,res):
os.system('hkl2vtk %s_friedel.hkl %s_friedel.vtk %s_raw.vtk' %(map,map,map))
os.system('vtk2lat %s_friedel.vtk %s_friedel.lat' %(map,map))
os.system('avgrlt %s_friedel.lat %s_friedel.rf' %(map,map))
os.system('subrflt %s_friedel.rf %s_friedel.lat anisotropic.lat' %(map,map))
os.system('lat2hkl anisotropic.lat anisotropic.hkl')
#Read in hkl file and populate miller array
from cctbx import crystal
inf = open('anisotropic.hkl', "r")
indices = flex.miller_index()
i_obs = flex.double()
#sig_i = flex.double()
for line in inf.readlines():
assert len(line.split())==4
line = line.strip().split()
i_obs_ = float(line[3])#/10000 #10000 is a uniform scale factor meant to re-size all diffuse intensities (normally too large for scalepack)
#sig_i_ = math.sqrt(i_obs_)
#if(abs(i_obs_)>1.e-6): # perhaps you don't want zeros
if i_obs_ != -32768.0:
indices.append([int(line[0]),int(line[1]),int(line[2])])
i_obs.append(i_obs_)
#sig_i.append(sig_i_)
inf.close()
#Get miller array object
cs = crystal.symmetry(unit_cell=(float(uc[0]), float(uc[1]), float(uc[2]), float(uc[3]), float(uc[4]), float(uc[5])), space_group=sg)
miller_set=miller.set(cs, indices)
ma = miller.array(miller_set=miller_set, data=i_obs, sigmas=None)
mtz_dataset = ma.as_mtz_dataset(column_root_label="Amplitude")
mtz_dataset.mtz_object().write('%s_anisotropic.mtz' %map)
def __init__(self, strategies, n_bins=8, degrees_per_bin=5):
from cctbx import crystal, miller
import copy
sg = strategies[0].experiment.crystal.get_space_group() \
.build_derived_reflection_intensity_group(anomalous_flag=True)
cs = crystal.symmetry(
unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg)
for i, strategy in enumerate(strategies):
if i == 0:
predicted = copy.deepcopy(strategy.predicted)
else:
predicted_ = copy.deepcopy(strategy.predicted)
predicted_['dose'] += (flex.max(predicted['dose']) + 1)
predicted.extend(predicted_)
ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True)
ma = miller.array(ms, data=flex.double(ms.size(),1),
sigmas=flex.double(ms.size(), 1))
if 1:
merging = ma.merge_equivalents()
o = merging.array().customized_copy(
data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object()
o.write('predicted.mtz')
d_star_sq = ma.d_star_sq().data()
binner = ma.setup_binner_d_star_sq_step(
d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins)
dose = predicted['dose']
range_width = 1
range_min = flex.min(dose) - range_width
range_max = flex.max(dose)
n_steps = 2 + int((range_max - range_min) - range_width)
binner_non_anom = ma.as_non_anomalous_array().use_binning(
binner)
self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1])
from xia2.Modules.PyChef2 import ChefStatistics
chef_stats = ChefStatistics(
ma.indices(), ma.data(), ma.sigmas(),
ma.d_star_sq().data(), dose, self.n_complete, binner,
ma.space_group(), ma.anomalous_flag(), n_steps)
def fraction_new(completeness):
# Completeness so far at end of image
completeness_end = completeness[1:]
# Completeness so far at start of image
completeness_start = completeness[:-1]
# Fraction of unique reflections observed for the first time on each image
return completeness_end - completeness_start
self.dose = dose
self.ieither_completeness = chef_stats.ieither_completeness()
self.iboth_completeness = chef_stats.iboth_completeness()
self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin
self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
def exercise_systematic(verbose):
from cctbx import miller
from cctbx import crystal
from cctbx.array_family import flex
cs = crystal.symmetry(
unit_cell=(13,15,14,90,90,100),
space_group_symbol="P112")
ms = miller.set(
crystal_symmetry=cs,
indices=flex.miller_index([
(0,0,1),(0,0,-1),
(0,1,1),
(1,0,0),
(-1,-1,-1)]),
anomalous_flag=True).map_to_asu()
cf = ms.centric_flags().data()
assert cf.count(True) == 1
mt = flex.mersenne_twister(seed=0)
ma = ms.array(
data=mt.random_double(size=5)+0.1,
sigmas=mt.random_double(size=5)+0.1)
def recycle(expected_column_data):
mtz_obj = ma.as_mtz_dataset(column_root_label="X").mtz_object()
sio = StringIO()
mtz_obj.show_column_data_human_readable(out=sio)
from libtbx.test_utils import show_diff
if (verbose): sys.stdout.write(sio.getvalue())
assert not show_diff(sio.getvalue(), expected_column_data)
ma_2 = only_element(mtz_obj.as_miller_arrays())
assert_equal_data_and_sigmas(ma, ma_2)
recycle("""\
Column data:
-------------------------------------------------------------------------------
X(+) SIGX(+) X(-) SIGX(-)
0 0 1 0.517022 0.192339 0.820324 0.28626
0 1 1 0.100114 0.445561 None None
1 0 0 0.402333 0.496767 None None
1 1 1 None None 0.246756 0.638817
-------------------------------------------------------------------------------
""")
from cctbx.xray import observation_types
ma.set_observation_type(observation_types.reconstructed_amplitude())
recycle("""\
Column data:
-------------------------------------------------------------------------------
X SIGX DANOX SIGDANOX
ISYMX
0 0 1 0.668673 0.172438 -0.303302 0.344875
0
0 1 1 0.100114 0.445561 None None
1
1 0 0 0.402333 0.496767 None None
0
1 1 1 0.246756 0.638817 None None
2
-------------------------------------------------------------------------------
""")
def verify(sg_fcalc, sg_hl, sg_cns, p1_cns):
sg_phase_integrals = miller.array(
miller_set=miller.set(
crystal_symmetry=sg_fcalc,
indices=sg_cns.miller_indices,
anomalous_flag=sg_fcalc.anomalous_flag()),
data=sg_cns.hl).phase_integrals()
for h, cns_pi, miller_pi in zip(sg_cns.miller_indices,
sg_cns.pi.data,
sg_phase_integrals.data()):
if (abs(cns_pi - miller_pi) > 1.e-2):
print "Error:", h, cns_pi, miller_pi
if (0): return
raise AssertionError
#
p1_phase_integrals = miller.array(
miller_set=miller.set(
crystal_symmetry=sg_fcalc.cell_equivalent_p1(),
indices=p1_cns.miller_indices,
anomalous_flag=sg_fcalc.anomalous_flag()),
data=p1_cns.hl).phase_integrals()
for h, cns_pi, miller_pi in zip(p1_cns.miller_indices,
p1_cns.pi.data,
p1_phase_integrals.data()):
if (abs(cns_pi - miller_pi) > 1.e-2):
print "Error:", h, cns_pi, miller_pi
if (0): return
raise AssertionError
#
space_group = sg_fcalc.space_group()
asu = sg_fcalc.space_group_info().reciprocal_space_asu()
lookup_dict = miller.make_lookup_dict(sg_fcalc.indices())
for p1_i, h in enumerate(p1_cns.miller_indices):
h_asu = miller.asym_index(space_group, asu, h)
h_eq = h_asu.one_column(sg_fcalc.anomalous_flag())
fcalc_asu = h_eq.complex_eq(p1_cns.fcalc.data[p1_i])
hl_asu = h_eq.hendrickson_lattman_eq(p1_cns.hl[p1_i])
sg_i = lookup_dict[h_eq.h()]
assert abs(sg_fcalc.data()[sg_i] - fcalc_asu) < 1.e-2
if (not approx_equal(sg_hl[sg_i], hl_asu, eps=1.e-2)):
print "Error:", sg_fcalc.space_group_info()
print sg_fcalc.indices()[sg_i]
print "i:", sg_hl[sg_i]
print "o:", hl_asu
if (0): return
raise AssertionError
def dummy_miller_set(crystal_symmetry, log_n_reflections):
indices = flex.miller_index(((1,2,3),))
for i in xrange(log_n_reflections):
indices.append(indices)
return miller.set(
crystal_symmetry=crystal_symmetry,
indices=indices,
anomalous_flag=False)
def run(files):
assert len(files) == 2
hkl1 = xds_ascii.XDS_ASCII(files[0], sys.stdout)
hkl2 = xds_ascii.XDS_ASCII(files[1], sys.stdout)
hkl1_points = numpy.column_stack((hkl1.xd, hkl1.yd, hkl1.zd))
tree1 = spatial.cKDTree(hkl1_points)
n_ovl, n_nonovl = 0, 0
novl_indices, novl_i, novl_sigma = flex.miller_index(), flex.double(), flex.double()
for i in xrange(len(hkl2.indices)):
x, y, z = hkl2.xd[i], hkl2.yd[i], hkl2.zd[i]
#if z > 180:
# continue
dists, idxs = tree1.query((x,y,z), k=3, p=1)
overlaps = []
for dist, idx in zip(dists, idxs):
idx = int(idx)
xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx]
if abs(z-zo) < 2.5 and (xo-x)**2+(yo-y)**2 < 15**2: # FIXME MAGIC NUMBER!
overlaps.append((dist,idx))
if len(overlaps) == 0:
novl_indices.append(hkl2.indices[i])
novl_i.append(hkl2.iobs[i])
novl_sigma.append(hkl2.sigma_iobs[i])
n_nonovl += 1
else:
print hkl2.indices[i], x, y, z
for dist, idx in overlaps:
xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx]
print hkl1.indices[idx], xo, yo, zo
print dist, idx
print
print
n_ref = len(hkl2.indices)
print "%.2f%% overlap!" % (100.*(n_ref-n_nonovl)/n_ref)
novl_array = miller.array(miller_set=miller.set(crystal_symmetry=hkl2.symm, indices=novl_indices),
data=novl_i, sigmas=novl_sigma)
stats = dataset_statistics(novl_array, anomalous=False, sigma_filtering="xds")
stats.show(out=sys.stdout)
novl_array = novl_array.customized_copy(anomalous_flag=False).map_to_asu()
novl_array = novl_array.eliminate_sys_absent()
novl_array = novl_array.select(novl_array.sigmas() >= 0)
filtr = filter_intensities_by_sigma(novl_array, "xds")
hklout = os.path.splitext(os.path.basename(files[1]))[0] + "_novl.mtz"
filtr.array_merged.set_observation_type_xray_intensity().as_mtz_dataset(column_root_label="IMEAN").mtz_object().write(hklout)
def column_group(
crystal_symmetry_from_file,
crystal_symmetry,
base_array_info,
primary_column_type,
labels,
group,
observation_type):
assert group.data.size() == group.indices.size()
sigmas = getattr(group, "sigmas", None)
if (sigmas is not None): assert sigmas.size() == group.indices.size()
if (group.anomalous_flag):
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=group.indices,
anomalous_flag=True)
if (miller_set.n_bijvoet_pairs() == 0):
# account for non-sensical files generated via
# ccp4i "import merged data" tab with default parameters
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=group.indices,
anomalous_flag=False)
else:
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=group.indices).auto_anomalous(min_fraction_bijvoet_pairs=2/3.)
result = miller_set.array(
data=group.data,
sigmas=sigmas).set_info(
base_array_info.customized_copy(
labels=labels,
crystal_symmetry_from_file=crystal_symmetry_from_file,
type_hints_from_file=column_type_as_miller_array_type_hints.get(
primary_column_type)))
if (observation_type is not None):
result.set_observation_type(observation_type)
elif (not result.is_complex_array()):
if (primary_column_type in "FG"):
result.set_observation_type_xray_amplitude()
elif (primary_column_type in "JK"):
result.set_observation_type_xray_intensity()
return result
def i_obs(self, anomalous_flag=None):
assert "IOBS" in self.data
assert "SIGMA" in self.data
array_info = miller.array_info(source_type="xds_integrate")#, wavelength=)
return miller.array(miller_set=miller.set(crystal_symmetry=self.crystal_symmetry(),
indices=self.hkl,
anomalous_flag=anomalous_flag),
data=self.data["IOBS"],
sigmas=self.data["SIGMA"]).set_info(array_info).set_observation_type_xray_intensity()
def miller_set (self,
crystal_symmetry=None,
force_symmetry=False) :
crystal_symmetry_from_file = self.crystal_symmetry()
return miller.set(
crystal_symmetry=crystal_symmetry_from_file.join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry),
indices=self.miller_indices,
anomalous_flag=self.anomalous_flag)
def as_normalised_array(miller_array,
asu_contents,
wilson_plot=None):
"""Old python code replaced by faster C++ code."""
from cctbx import statistics
from cctbx import eltbx
if not wilson_plot:
wilson_plot = statistics.wilson_plot(miller_array, asu_contents)
# cache scattering factor info
gaussians = {}
for chemical_type in asu_contents:
gaussians.setdefault(chemical_type, eltbx.xray_scattering.wk1995(
chemical_type).fetch())
stol_sq = miller_array.sin_theta_over_lambda_sq()
epsilons = miller_array.epsilons()
e_sq_minus_1 = 0
n_e_greater_than_2 = 0
normalised_f_obs = flex.double()
space_group = miller_array.space_group()
tau = space_group.n_ltr()
for i in xrange(0,miller_array.size()):
s_sq = stol_sq.data()[i]
f_sq = math.pow(miller_array.data()[i], 2)
epsilon = epsilons.data()[i]
sum_fj_sq = 0
for chemical_type, n_atoms in asu_contents.items():
n_atoms *= space_group.order_z()
f0 = gaussians[chemical_type].at_stol_sq(s_sq)
sum_fj_sq += f0 * f0 * n_atoms
e_sq = f_sq\
/(wilson_plot.wilson_intensity_scale_factor*math.exp(-2*wilson_plot.wilson_b*s_sq)
*epsilon
*tau
*sum_fj_sq)
normalised_f_obs.append(math.sqrt(e_sq))
e_sq_minus_1 += abs(e_sq - 1)
if (e_sq > 4.0): n_e_greater_than_2 += 1
r = Empty()
r.array = miller.array(
miller_set=miller.set(
crystal_symmetry=miller_array.crystal_symmetry(),
indices=miller_array.indices()).auto_anomalous(),
data=normalised_f_obs,
sigmas=miller_array.sigmas())
r.mean_e_sq_minus_1 = e_sq_minus_1/r.array.size()
r.percent_e_sq_gt_2 = (100.0*n_e_greater_than_2)/r.array.size()
return r
def exercise_sys_absent_intensity_distribution(): xs = crystal.symmetry((3,4,5), "F222") mi = flex.miller_index(((1,2,3), (1,1,1), (1,2,2), (0,0,4))) ms = miller.set(xs, mi) data = flex.double((-1,-2,3,4)) sigmas = flex.double((2,3,4,5)) f_obs = miller.array(ms, data=data, sigmas=sigmas).set_observation_type_xray_intensity() dist = statistics.sys_absent_intensity_distribution(f_obs) assert approx_equal(dist.x,(-0.5,0.75)) assert approx_equal(dist.y,(-1,3)) assert approx_equal(dist.indices,((1,2,3), (1,2,2)))
def arrays(self):
ret = {}
for key in self.data:
arr = miller.array(miller_set=miller.set(crystal_symmetry=self.crystal_symmetry(),
indices=self.hkl,
anomalous_flag=False),
data=self.data[key])
ret[key] = arr
return ret
def unit_cell_to_symetry_object(img_dict):
xsym = symmetry(unit_cell=img_dict['current_orientation'][0].unit_cell().niggli_cell(),
space_group_symbol=point_group)
miller_set = miller.set(crystal_symmetry=xsym,
indices=flex.miller_index(img_dict['Millers']),
anomalous_flag=True)
miller_array = miller.array(miller_set,
flex.double(img_dict['Is']),
flex.double(img_dict['sigIs']))
miller_array.set_observation_type_xray_intensity()
miller_array.set_info("Raw data obtained by integration using CrystFEL")
return miller_array
def unmerged_miller_set(self, crystal_symmetry=None, force_symmetry=False):
if (not force_symmetry
or crystal_symmetry is None
or crystal_symmetry.space_group_info() is None):
space_group_info = self.space_group_info()
else:
space_group_info = None
return miller.set(
crystal_symmetry=self.crystal_symmetry().join_symmetry(
other_symmetry=crystal_symmetry,
force=force_symmetry),
indices=self.original_indices,
anomalous_flag=True)
def make_joined_set (miller_arrays) :
if(len(miller_arrays)==0): return None
cs0 = miller_arrays[0].crystal_symmetry()
for ma in miller_arrays:
if(not ma.crystal_symmetry().is_similar_symmetry(cs0)): return None
from cctbx import miller
master_set = miller.set(
crystal_symmetry=miller_arrays[0].crystal_symmetry(),
indices=miller_arrays[0].indices(),
anomalous_flag=False)
master_indices = miller_arrays[0].indices().deep_copy()
for array in miller_arrays[1:] :
current_indices = array.indices()
missing_isel = miller.match_indices(master_indices,
current_indices).singles(1)
missing_indices = current_indices.select(missing_isel)
master_indices.extend(missing_indices)
master_set = miller.set(
crystal_symmetry=miller_arrays[0].crystal_symmetry(),
indices=master_indices,
anomalous_flag=False)
return master_set.map_to_asu().unique_under_symmetry()
def tst_sg_tools():
unit_cell = uctbx.unit_cell('40, 50, 60, 90.0, 90.0, 90.0')
mi = flex.miller_index(((2,4,6), (2,4,8)))
xs = crystal.symmetry(unit_cell, "P 1 1 21")
ms = miller.set(xs, mi)
tmp_choice = pt.space_group_graph_from_cell_and_sg(unit_cell,
xs.space_group())
xs1=crystal.symmetry(uctbx.unit_cell('40.00 60.00 50.00 90.00 90.00 90.00'),
'P 1 21 1')
xs2=crystal.symmetry(uctbx.unit_cell('40.00 50.00 60.00 90.00 90.00 90.00'),
'P 2 2 21')
xs3=crystal.symmetry(uctbx.unit_cell('40.00 60.00 50.00 90.00 90.00 90.00'),
'P 21 21 2')
xs4=crystal.symmetry(uctbx.unit_cell('50.00 60.00 40.00 90.00 90.00 90.00'),
'P 21 21 2')
xs5=crystal.symmetry(uctbx.unit_cell('40.00 50.00 60.00 90.00 90.00 90.00'),
'P 21 21 21')
p222_dict = { str( xs2.unit_cell().parameters())
+ " " + str( xs2.space_group_info() ): None,
str( xs3.unit_cell().parameters())
+ " " + str( xs3.space_group_info() ):None,
str( xs4.unit_cell().parameters())
+ " " + str( xs4.space_group_info() ):None,
str( xs5.unit_cell().parameters())
+ " " + str( xs5.space_group_info() ):None
}
p112 = tmp_choice.pg_graph.graph.node_objects['P 1 1 2'].allowed_xtal_syms
# check the cell parameters
assert approx_equal(p112[0][0].unit_cell().parameters(),
xs1.unit_cell().parameters())
# check the sg
assert p112[0][0].space_group() == xs1.space_group()
# check the change of basis operator
assert approx_equal(xs.change_basis(p112[0][1]).unit_cell().parameters(),
xs1.unit_cell().parameters())
p222 = tmp_choice.pg_graph.graph.node_objects['P 2 2 2'].allowed_xtal_syms
for xs in p222:
comp_string = (str(xs[0].unit_cell().parameters())
+ " " + str(xs[0].space_group_info()))
assert p222_dict.has_key(comp_string)
def run(self, experiments, reflections):
self.logger.log_step_time("REINDEX")
# Get list of twinning operators for this space group
operators = twin_laws(self.params.scaling.i_model).operators
if not operators:
self.logger.log("No indexing ambiguity. Skipping this step.")
return experiments, reflections
self.logger.log("Resolving indexing ambiguity using operators h,k,l, %s"%", ".join( \
[op.operator.r().as_hkl() for op in operators]))
operators = [
sgtbx.change_of_basis_op(op.operator.r().as_hkl())
for op in operators
]
result = flex.reflection_table()
scaler = experiment_scaler(self.params, self.mpi_helper, self.logger)
model_intensities = self.params.scaling.i_model
target_symm = symmetry(
unit_cell=self.params.scaling.unit_cell,
space_group_info=self.params.scaling.space_group)
def get_correlation(cb_op=None):
""" Helper function to get CC to the reference given an operator """
# Build a miller array for the experiment reflections
exp_miller_indices = miller.set(
target_symm, exp_reflections['miller_index_asymmetric'], True)
exp_intensities = miller.array(
exp_miller_indices, exp_reflections['intensity.sum.value'],
flex.sqrt(exp_reflections['intensity.sum.variance']))
if cb_op:
exp_intensities = exp_intensities.change_basis(
cb_op).map_to_asu()
# Extract an array of HKLs from the model to match the experiment HKLs
matching_indices = miller.match_multi_indices(
miller_indices_unique=model_intensities.indices(),
miller_indices=exp_intensities.indices())
# Least squares
scaling_result = scaler.fit_experiment_to_reference(
model_intensities, exp_intensities, matching_indices)
return scaling_result.correlation if scaling_result.correlation is not None else -1
# Test each experiment to see if an operator gives a better CC to the reference, and if it does, apply it
for expt_id, experiment in enumerate(experiments):
exp_reflections = reflections.select(
reflections['exp_id'] == experiment.identifier)
all_correlations = []
best_correlation = get_correlation()
all_correlations.append(best_correlation)
best_op = None
for cb_op in operators:
test_correlation = get_correlation(cb_op)
all_correlations.append(test_correlation)
if test_correlation > best_correlation:
best_correlation = test_correlation
best_op = cb_op
if best_op:
exp_miller_indices = miller.set(
target_symm, exp_reflections['miller_index'],
True).change_basis(best_op)
exp_reflections[
'miller_index_asymmetric'] = exp_miller_indices.map_to_asu(
).indices()
exp_reflections['miller_index'] = exp_miller_indices.indices()
experiment.crystal = MosaicCrystalSauter2014(
experiment.crystal.change_basis(best_op)
) # need to use wrapper because of cctbx/dxtbx#5
result.extend(exp_reflections)
self.logger.log(
"Expt %d, reindexing op correlations: %s" %
(expt_id, ", ".join(["%6.3f" % c for c in all_correlations])))
self.logger.log_step_time("REINDEX", True)
self.logger.log("Memory usage: %d MB" % get_memory_usage())
return experiments, result
# Load the cctbx.xfel.merge output for estimation of initial params
if args.mtzinput:
# TODO: make this so that one can selec on label (e.g. fobs)
F = any_reflection_file(
args.merge).as_miller_arrays()[0].as_amplitude_array()
else:
F = utils.open_flex(args.merge) #.as_amplitude_array()
Fmap = {h: amp for h, amp in zip(F.indices(), F.data())}
merge_Hi = list(set(map_hkl_list(Fmap.keys(), symbol=args.symbol)))
symm = symmetry(unit_cell=(a, b, c, al, be, ga),
space_group_symbol=args.symbol)
# diffBragg resolutions
mset = miller.set(symm, flex.miller_index(u_all_Hi), anomalous_flag=True)
reso = mset.d_spacings().data()
reso = np.array(reso)[:, None]
# merged resolutions
msetm = miller.set(symm, flex.miller_index(merge_Hi), anomalous_flag=True)
resom = msetm.d_spacings().data()
resom = np.array(resom)[:, None]
tree = cKDTree(resom)
o, knearest = tree.query(reso, k=5)
u_all_amp = []
for i_h, h in enumerate(u_all_Hi):
if h in Fmap:
in_array = True
def run(args):
import libtbx.load_env
usage = "dev.dials.estimate_resolution_limit indexed.expt indexed.refl [options]"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
if len(experiments) == 0:
parser.print_help()
return
elif len(experiments) > 1:
raise Sorry("More than one experiment present")
experiment = experiments[0]
assert len(reflections) == 1
reflections = reflections[0]
intensities = reflections["intensity.sum.value"]
variances = reflections["intensity.sum.variance"]
if "intensity.prf.value" in reflections:
intensities = reflections["intensity.prf.value"]
variances = reflections["intensity.prf.variance"]
sel = variances > 0
intensities = intensities.select(sel)
variances = variances.select(sel)
sigmas = flex.sqrt(variances)
indices = reflections["miller_index"].select(sel)
from cctbx import crystal, miller
crystal_symmetry = crystal.symmetry(
space_group=experiment.crystal.get_space_group(),
unit_cell=experiment.crystal.get_unit_cell(),
)
miller_set = miller.set(crystal_symmetry=crystal_symmetry,
anomalous_flag=True,
indices=indices)
miller_array = miller.array(
miller_set=miller_set, data=intensities,
sigmas=sigmas).set_observation_type_xray_intensity()
# miller_array.setup_binner(n_bins=50, reflections_per_bin=100)
miller_array.setup_binner(auto_binning=True, n_bins=20)
result = miller_array.i_over_sig_i(use_binning=True)
result.show()
from cctbx import uctbx
d_star_sq_centre = result.binner.bin_centers(2)
i_over_sig_i = flex.double(
[d if d is not None else 0 for d in result.data[1:-1]])
sel = i_over_sig_i > 0
d_star_sq_centre = d_star_sq_centre.select(sel)
i_over_sig_i = i_over_sig_i.select(sel)
log_i_over_sig_i = flex.log(i_over_sig_i)
weights = result.binner.counts()[1:-1].as_double().select(sel)
fit = flex.linear_regression(d_star_sq_centre,
log_i_over_sig_i,
weights=weights)
m = fit.slope()
c = fit.y_intercept()
import math
y_cutoff = math.log(params.i_sigi_cutoff)
x_cutoff = (y_cutoff - c) / m
estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff)
print("estimated d_min: %.2f" % estimated_d_min)
if params.plot:
from matplotlib import pyplot
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(list(d_star_sq_centre),
list(log_i_over_sig_i),
label=r"ln(I/sigI)")
ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()],
color="red")
ax.plot([x_cutoff, x_cutoff],
pyplot.ylim(),
color="grey",
linestyle="dashed")
ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff],
color="grey",
linestyle="dashed")
ax.set_xlabel("d_star_sq")
ax.set_ylabel("ln(I/sigI)")
ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis
xticks = ax.get_xticks()
xlim = ax.get_xlim()
xticks_d = [
uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks
]
xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks]
ax_.set_xticks(xticks)
ax_.set_xlim(ax.get_xlim())
ax_.set_xlabel(r"Resolution ($\AA$)")
ax_.set_xticklabels(["%.1f" % d for d in xticks_d])
pyplot.savefig("estimate_resolution_limit.png")
pyplot.clf()
def demo():
#
# create toy lists of Miller indices
#
crystal_symmetry = crystal.symmetry(unit_cell=(13, 14, 15, 90, 90, 90),
space_group_symbol="P212121")
miller_set_a = miller.set(crystal_symmetry=crystal_symmetry,
anomalous_flag=False,
indices=flex.miller_index([(0, -1, 2),
(-1, -2, 3),
(2, 3, 4), (-3, 4, 5),
(4, -5, 6)]))
miller_set_b = miller_set_a.customized_copy(
indices=flex.miller_index([(-5, -6, 7), (0, 1, 2), (3, -4,
5), (-1, -2,
3), (-4, -5,
6)]))
#
# map all indices to the asymmetric unit
#
asu_a = miller_set_a.map_to_asu()
asu_b = miller_set_b.map_to_asu()
for h in asu_a.indices():
print("asu a:", h)
print()
for h in asu_b.indices():
print("asu b:", h)
print()
#
# obtain the common index sets
#
common_a, common_b = asu_a.common_sets(asu_b)
for h in common_a.indices():
print("common a:", h)
print()
for h in common_b.indices():
print("common b:", h)
print()
#
# obtain the "lone" index sets
#
lone_set_a, lone_set_b = asu_a.lone_sets(asu_b)
for h in lone_set_a.indices():
print("lone a:", h)
print()
for h in lone_set_b.indices():
print("lone b:", h)
print()
#
# now the same again, but with data (i.e. miller.array instances)
#
miller_array_a = miller_set_a.array(data=flex.random_double(
size=miller_set_a.indices().size()))
miller_array_b = miller_set_b.array(data=flex.random_double(
size=miller_set_a.indices().size()))
#
# map all indices to the asymmetric unit
#
asu_a = miller_array_a.map_to_asu()
asu_b = miller_array_b.map_to_asu()
asu_a.show_array(prefix="asu a: ")
print()
asu_b.show_array(prefix="asu b: ")
print()
#
# obtain the common index sets
#
common_a, common_b = asu_a.common_sets(asu_b)
common_a.show_array(prefix="common a: ")
print()
common_b.show_array(prefix="common b: ")
print()
#
# obtain the "lone" index sets
#
lone_a, lone_b = asu_a.lone_sets(asu_b)
lone_a.show_array(prefix="lone a: ")
print()
lone_b.show_array(prefix="lone b: ")
print()
print("OK")
def miller_set(self, anomalous_flag=True):
return miller.set(crystal_symmetry=crystal.symmetry(
unit_cell=self.unit_cell, space_group_symbol=self.space_group),
indices=self.miller_indices,
anomalous_flag=anomalous_flag)
def create_datastructures_for_target_mtz(experiments, mtz_file):
"""Read a merged mtz file and extract miller indices, intensities and
variances."""
m = mtz.object(mtz_file)
ind = m.extract_miller_indices()
cols = m.columns()
col_dict = {c.label(): c for c in cols}
r_t = flex.reflection_table()
if "I" in col_dict: # nice and simple
r_t["miller_index"] = ind
r_t["intensity"] = col_dict["I"].extract_values().as_double()
r_t["variance"] = flex.pow2(
col_dict["SIGI"].extract_values().as_double())
elif "IMEAN" in col_dict: # nice and simple
r_t["miller_index"] = ind
r_t["intensity"] = col_dict["IMEAN"].extract_values().as_double()
r_t["variance"] = flex.pow2(
col_dict["SIGIMEAN"].extract_values().as_double())
elif "I(+)" in col_dict: # need to combine I+ and I- together into target Ih
if col_dict["I(+)"].n_valid_values() == 0: # use I(-)
r_t["miller_index"] = ind
r_t["intensity"] = col_dict["I(-)"].extract_values().as_double()
r_t["variance"] = flex.pow2(
col_dict["SIGI(-)"].extract_values().as_double())
elif col_dict["I(-)"].n_valid_values() == 0: # use I(+)
r_t["miller_index"] = ind
r_t["intensity"] = col_dict["I(+)"].extract_values().as_double()
r_t["variance"] = flex.pow2(
col_dict["SIGI(+)"].extract_values().as_double())
else: # Combine both - add together then use Ih table to calculate I and sigma
r_tplus = flex.reflection_table()
r_tminus = flex.reflection_table()
r_tplus["miller_index"] = ind
r_tplus["intensity"] = col_dict["I(+)"].extract_values().as_double(
)
r_tplus["variance"] = flex.pow2(
col_dict["SIGI(+)"].extract_values().as_double())
r_tminus["miller_index"] = ind
r_tminus["intensity"] = col_dict["I(-)"].extract_values(
).as_double()
r_tminus["variance"] = flex.pow2(
col_dict["SIGI(-)"].extract_values().as_double())
r_tplus.extend(r_tminus)
r_tplus.set_flags(flex.bool(r_tplus.size(), False),
r_tplus.flags.bad_for_scaling)
r_tplus = r_tplus.select(r_tplus["variance"] != 0.0)
Ih_table = create_Ih_table([experiments[0]], [r_tplus],
anomalous=True).blocked_data_list[0]
r_t["intensity"] = Ih_table.Ih_values
inv_var = Ih_table.sum_in_groups(Ih_table.weights,
output="per_refl")
r_t["variance"] = 1.0 / inv_var
r_t["miller_index"] = Ih_table.miller_index
else:
raise KeyError(
"Unable to find intensities (tried I, IMEAN, I(+)/I(-))")
logger.info(f"Extracted {r_t.size()} intensities from target mtz")
r_t = r_t.select(r_t["variance"] > 0.0)
if r_t.size() == 0:
raise ValueError(
"No reflections with positive sigma remain after filtering")
r_t["d"] = (miller.set(
crystal_symmetry=crystal.symmetry(
space_group=m.space_group(),
unit_cell=m.crystals()[0].unit_cell()),
indices=r_t["miller_index"],
).d_spacings().data())
r_t.set_flags(flex.bool(r_t.size(), True), r_t.flags.integrated)
exp = Experiment()
exp.crystal = deepcopy(experiments[0].crystal)
exp.identifier = ersatz_uuid4()
r_t.experiment_identifiers()[len(experiments)] = exp.identifier
r_t["id"] = flex.int(r_t.size(), len(experiments))
# create a new KB scaling model for the target and set as scaled to fix scale
# for targeted scaling.
params = Mock()
params.KB.decay_correction.return_value = False
exp.scaling_model = KBScalingModel.from_data(params, [], [])
exp.scaling_model.set_scaling_model_as_scaled(
) # Set as scaled to fix scale.
return exp, r_t
bins = []
for i in range(nbins):
b0, b1 = min_D2 + i * bin_size, min_D2 + (i + 1) * bin_size
print 1 / sqrt(b0), 1 / sqrt(b1)
bins.append((b0, b1))
def get_bin_index(h):
d = unit_cell.d(h)
d2 = 1 / d**2
bin_index = int(floor((d2 - min_D2) / bin_size))
if bin_index == nbins:
bin_index = nbins - 1
return bin_index
cs = intensities.crystal_symmetry()
ms = miller.set(cs, intensities.indices())
ms_asu = ms.map_to_asu()
asu_indices = ms_asu.indices()
hkl = []
batch = []
intensity = []
for i in range(len(asu_indices)):
hkl.append(asu_indices[i])
batch.append(batches.data()[i])
intensity.append(intensities.data()[i])
class Item(object):
def __init__(self, h, batch, intensity):
self.h = h
self.batch = batch
def __init__(self, cif_block, base_array_info=None, wavelengths=None):
crystal_symmetry_builder.__init__(self, cif_block)
if base_array_info is not None:
self.crystal_symmetry = self.crystal_symmetry.join_symmetry(
other_symmetry=base_array_info.crystal_symmetry_from_file,
force=True)
self._arrays = OrderedDict()
if (wavelengths is None):
wavelengths = {}
if base_array_info is None:
base_array_info = miller.array_info(source_type="cif")
refln_containing_loops = self.get_miller_indices_containing_loops()
for self.indices, refln_loop in refln_containing_loops:
self.wavelength_id_array = None
self.crystal_id_array = None
self.scale_group_array = None
wavelength_ids = [None]
crystal_ids = [None]
scale_groups = [None]
for key, value in refln_loop.iteritems():
# need to get these arrays first
if (key.endswith('wavelength_id') or key.endswith('crystal_id')
or key.endswith('scale_group_code')):
data = as_int_or_none_if_all_question_marks(
value, column_name=key)
if data is None:
continue
counts = data.counts()
if key.endswith('wavelength_id'):
wavelength_ids = counts.keys()
if len(counts) == 1: continue
array = miller.array(
miller.set(self.crystal_symmetry,
self.indices).auto_anomalous(), data)
if key.endswith('wavelength_id'):
self.wavelength_id_array = array
wavelength_ids = counts.keys()
elif key.endswith('crystal_id'):
self.crystal_id_array = array
crystal_ids = counts.keys()
elif key.endswith('scale_group_code'):
self.scale_group_array = array
scale_groups = counts.keys()
for label, value in sorted(refln_loop.items()):
for w_id in wavelength_ids:
for crys_id in crystal_ids:
for scale_group in scale_groups:
if 'index_' in label: continue
key = label
labels = [label]
wavelength = None
if (key.endswith('wavelength_id')
or key.endswith('crystal_id')
or key.endswith('scale_group_code')):
w_id = None
crys_id = None
scale_group = None
key_suffix = ''
if w_id is not None:
key_suffix += '_%i' % w_id
labels.insert(0, "wavelength_id=%i" % w_id)
wavelength = wavelengths.get(w_id, None)
if crys_id is not None:
key_suffix += '_%i' % crys_id
labels.insert(0, "crystal_id=%i" % crys_id)
if scale_group is not None:
key_suffix += '_%i' % scale_group
labels.insert(
0, "scale_group_code=%i" % scale_group)
key += key_suffix
sigmas = None
if key in self._arrays: continue
array = self.flex_std_string_as_miller_array(
value,
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
if array is None: continue
if '_sigma' in key:
sigmas_label = label
key = None
for suffix in ('', '_meas', '_calc'):
if sigmas_label.replace(
'_sigma', suffix) in refln_loop:
key = sigmas_label.replace(
'_sigma', suffix) + key_suffix
break
if key is None:
key = sigmas_label + key_suffix
elif key in self._arrays and self._arrays[
key].sigmas() is None:
sigmas = array
array = self._arrays[key]
check_array_sizes(array, sigmas, key,
sigmas_label)
sigmas = as_flex_double(
sigmas, sigmas_label)
array.set_sigmas(sigmas.data())
info = array.info()
array.set_info(
info.customized_copy(
labels=info.labels +
[sigmas_label],
wavelength=wavelength))
continue
elif 'PHWT' in key:
phwt_label = label
fwt_label = label.replace('PHWT', 'FWT')
if fwt_label not in refln_loop: continue
phwt_array = array
if fwt_label in self._arrays:
array = self._arrays[fwt_label]
check_array_sizes(array, phwt_array,
fwt_label, phwt_label)
phases = as_flex_double(
phwt_array, phwt_label)
info = array.info()
array = array.phase_transfer(phases,
deg=True)
array.set_info(
info.customized_copy(
labels=info.labels + [phwt_label]))
self._arrays[fwt_label] = array
continue
elif 'HL_' in key:
hl_letter = key[key.find('HL_') + 3]
hl_key = 'HL_' + hl_letter
key = key.replace(hl_key, 'HL_A')
if key in self._arrays:
continue # this array is already dealt with
hl_labels = [
label.replace(hl_key, 'HL_' + letter)
for letter in 'ABCD'
]
hl_keys = [
key.replace(hl_key, 'HL_' + letter)
for letter in 'ABCD'
]
hl_values = [
cif_block.get(hl_key)
for hl_key in hl_labels
]
if hl_values.count(None) == 0:
selection = self.get_selection(
hl_values[0],
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
hl_values = [
as_double_or_none_if_all_question_marks(
hl.select(selection),
column_name=lab)
for hl, lab in zip(
hl_values, hl_labels)
]
array = miller.array(
miller.set(
self.crystal_symmetry,
self.indices.select(
selection)).auto_anomalous(),
flex.hendrickson_lattman(*hl_values))
labels = labels[:-1] + hl_labels
elif '.B_' in key or '_B_' in key:
if '.B_' in key:
key, key_b = key.replace('.B_', '.A_'), key
label, label_b = label.replace(
'.B_', '.A_'), label
elif '_B_' in key:
key, key_b = key.replace('_B', '_A'), key
label, label_b = label.replace('_B',
'_A'), label
if key in refln_loop and key_b in refln_loop:
b_part = array.data()
if key in self._arrays:
info = self._arrays[key].info()
a_part = self._arrays[key].data()
self._arrays[key] = self._arrays[
key].array(
data=flex.complex_double(
a_part, b_part))
self._arrays[key].set_info(
info.customized_copy(
labels=info.labels + [key_b]))
continue
elif ('phase_' in key and not "_meas" in key
and self.crystal_symmetry.space_group()
is not None):
alt_key1 = label.replace('phase_', 'F_')
alt_key2 = alt_key1 + '_au'
if alt_key1 in refln_loop:
phase_key = label
key = alt_key1 + key_suffix
elif alt_key2 in refln_loop:
phase_key = label
key = alt_key2 + key_suffix
else:
phase_key = None
if phase_key is not None:
phases = array.data()
if key in self._arrays:
array = self._arrays[key]
array = as_flex_double(array, key)
check_array_sizes(
array, phases, key, phase_key)
info = self._arrays[key].info()
self._arrays[
key] = array.phase_transfer(
phases, deg=True)
self._arrays[key].set_info(
info.customized_copy(
labels=info.labels +
[phase_key]))
else:
array = self.flex_std_string_as_miller_array(
refln_loop[label],
wavelength_id=w_id,
crystal_id=crys_id,
scale_group_code=scale_group)
check_array_sizes(
array, phases, key, phase_key)
array.phase_transfer(phases, deg=True)
labels = labels + [label, phase_key]
if base_array_info.labels is not None:
labels = base_array_info.labels + labels
def rstrip_substrings(string, substrings):
for substr in substrings:
if substr == '': continue
if string.endswith(substr):
string = string[:-len(substr)]
return string
# determine observation type
stripped_key = rstrip_substrings(
key, [
key_suffix, '_au', '_meas', '_calc',
'_plus', '_minus'
])
if (stripped_key.endswith('F_squared')
or stripped_key.endswith('intensity')
or stripped_key.endswith('.I')
or stripped_key.endswith('_I')) and (
array.is_real_array()
or array.is_integer_array()):
array.set_observation_type_xray_intensity()
elif (stripped_key.endswith('F')
and (array.is_real_array()
or array.is_integer_array())):
array.set_observation_type_xray_amplitude()
if (array.is_xray_amplitude_array()
or array.is_xray_amplitude_array()):
# e.g. merge_equivalents treats integer arrays differently, so must
# convert integer observation arrays here to be safe
if isinstance(array.data(), flex.int):
array = array.customized_copy(
data=array.data().as_double())
array.set_info(
base_array_info.customized_copy(labels=labels))
if (array.is_xray_amplitude_array()
or array.is_xray_amplitude_array()):
info = array.info()
array.set_info(
info.customized_copy(
wavelength=wavelength))
self._arrays.setdefault(key, array)
for key, array in self._arrays.copy().iteritems():
if (key.endswith('_minus') or '_minus_' in key
or key.endswith('_plus') or '_plus_' in key):
if '_minus' in key:
minus_key = key
plus_key = key.replace('_minus', '_plus')
elif '_plus' in key:
plus_key = key
minus_key = key.replace('_plus', '_minus')
if plus_key in self._arrays and minus_key in self._arrays:
plus_array = self._arrays.pop(plus_key)
minus_array = self._arrays.pop(minus_key)
minus_array = minus_array.customized_copy(
indices=-minus_array.indices()).set_info(
minus_array.info())
array = plus_array.concatenate(
minus_array, assert_is_similar_symmetry=False)
array = array.customized_copy(anomalous_flag=True)
array.set_info(
minus_array.info().customized_copy(labels=list(
OrderedSet(plus_array.info().labels +
minus_array.info().labels))))
array.set_observation_type(plus_array.observation_type())
self._arrays.setdefault(key, array)
if len(self._arrays) == 0:
raise CifBuilderError("No reflection data present in cif block")
def quasi_normalisation(reflection_table, experiment):
"""Calculate normalised intensity (Esq) values for reflections, for the purpose
of selecting subsets based on Esq for scaling. If more involved analyses of
normalised intensities are needed, then it may be necessary to split this
procedure to handle acentric and centric reflections separately."""
logger.info(
"Calculating normalised intensity values to select a reflection \n"
"subset for scaling. \n")
logger.debug(
"Negative intensities are set to zero for the purpose of \n"
"calculating mean intensity values for resolution bins. This is to avoid \n"
"spuriously high E^2 values due to a mean close to zero and should only \n"
"affect the E^2 values of the highest resolution bins. \n")
good_refl_sel = ~reflection_table.get_flags(
reflection_table.flags.bad_for_scaling, all=False)
rt_subset = reflection_table.select(good_refl_sel)
# Scaling subset is data that has not been flagged as bad or excluded
miller_set = miller.set(
crystal_symmetry=experiment.crystal.get_crystal_symmetry(),
indices=rt_subset["miller_index"],
)
# handle negative reflections to minimise effect on mean I values.
rt_subset["intensity_for_norm"] = copy.deepcopy(rt_subset["intensity"])
rt_subset["intensity_for_norm"].set_selected(rt_subset["intensity"] < 0.0,
0.0)
miller_array = miller.array(miller_set,
data=rt_subset["intensity_for_norm"])
n_refl = rt_subset.size()
# set up binning objects
if n_refl > 20000:
n_refl_shells = 20
elif n_refl > 15000:
n_refl_shells = 15
elif n_refl > 10000:
n_refl_shells = 10
else:
logger.info(
"No normalised intensity values were calculated, as an insufficient\n"
"number of reflections were detected. All normalised intensity \n"
"values will be set to 1 to allow use in scaling model determination. \n"
)
reflection_table["Esq"] = flex.double(reflection_table.size(), 1.0)
return reflection_table
d_star_sq = miller_array.d_star_sq().data()
d_star_sq_min = flex.min(d_star_sq)
d_star_sq_max = flex.max(d_star_sq)
span = d_star_sq_max - d_star_sq_min
relative_tolerance = 1e-6
d_star_sq_max += span * relative_tolerance
d_star_sq_min -= span * relative_tolerance
step = (d_star_sq_max - d_star_sq_min) / n_refl_shells
miller_array.setup_binner_d_star_sq_step(
auto_binning=False,
d_max=uctbx.d_star_sq_as_d(d_star_sq_max),
d_min=uctbx.d_star_sq_as_d(d_star_sq_min),
d_star_sq_step=step,
)
normalisations = miller_array.intensity_quasi_normalisations()
normalised_intensities = miller_array.customized_copy(
data=(miller_array.data() / normalisations.data()))
reflection_table["Esq"] = flex.double(reflection_table.size(), 0.0)
reflection_table["Esq"].set_selected(good_refl_sel,
normalised_intensities.data())
return reflection_table
def process_input_array(self):
from cctbx.array_family import flex
array = self.miller_array.deep_copy()
multiplicities = None
if self.merge_equivalents:
if self.settings.show_anomalous_pairs:
from cctbx import miller
merge = array.merge_equivalents()
multiplicities = merge.redundancies()
asu, matches = multiplicities.match_bijvoet_mates()
mult_plus, mult_minus = multiplicities.hemispheres_acentrics()
anom_mult = flex.int(
min(p, m)
for (p, m) in zip(mult_plus.data(), mult_minus.data()))
#flex.min_max_mean_double(anom_mult.as_double()).show()
anomalous_multiplicities = miller.array(
miller.set(asu.crystal_symmetry(),
mult_plus.indices(),
anomalous_flag=False), anom_mult)
anomalous_multiplicities = anomalous_multiplicities.select(
anomalous_multiplicities.data() > 0)
array = anomalous_multiplicities
multiplicities = anomalous_multiplicities
else:
merge = array.merge_equivalents()
array = merge.array()
multiplicities = merge.redundancies()
settings = self.settings
data = array.data()
self.missing_set = oop.null()
if (array.is_xray_intensity_array()):
data.set_selected(data < 0, flex.double(data.size(), 0.))
if (array.is_unique_set_under_symmetry()) and (settings.map_to_asu):
array = array.map_to_asu()
if (multiplicities is not None):
multiplicities = multiplicities.map_to_asu()
if (settings.d_min is not None):
array = array.resolution_filter(d_min=settings.d_min)
if (multiplicities is not None):
multiplicities = multiplicities.resolution_filter(
d_min=settings.d_min)
self.filtered_array = array.deep_copy()
if (settings.expand_anomalous):
array = array.generate_bijvoet_mates()
if (multiplicities is not None):
multiplicities = multiplicities.generate_bijvoet_mates()
if (self.settings.show_missing):
self.missing_set = array.complete_set().lone_set(array)
if self.settings.show_anomalous_pairs:
self.missing_set = self.missing_set.select(
self.missing_set.centric_flags().data(), negate=True)
if (settings.expand_to_p1):
original_symmetry = array.crystal_symmetry()
array = array.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
#array = array.niggli_cell().expand_to_p1()
#self.missing_set = self.missing_set.niggli_cell().expand_to_p1()
self.missing_set = self.missing_set.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
if (multiplicities is not None):
multiplicities = multiplicities.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
data = array.data()
self.r_free_mode = False
if isinstance(data, flex.bool):
self.r_free_mode = True
data_as_float = flex.double(data.size(), 0.0)
data_as_float.set_selected(data == True,
flex.double(data.size(), 1.0))
data = data_as_float
self.data = data.deep_copy()
else:
if isinstance(data, flex.double):
self.data = data.deep_copy()
elif isinstance(data, flex.complex_double):
self.data = flex.abs(data)
elif hasattr(array.data(), "as_double"):
self.data = array.data().as_double()
else:
raise RuntimeError("Unexpected data type: %r" % data)
if (settings.show_data_over_sigma):
if (array.sigmas() is None):
raise Sorry("sigmas not defined.")
sigmas = array.sigmas()
non_zero_sel = sigmas != 0
array = array.select(non_zero_sel)
array = array.customized_copy(data=array.data() /
array.sigmas())
self.data = array.data()
if (multiplicities is not None):
multiplicities = multiplicities.select(non_zero_sel)
self.work_array = array
self.multiplicities = multiplicities
def run(self, experiments, reflections):
self.logger.log_step_time("POSTREFINEMENT")
if not self.params.postrefinement.enable:
self.logger.log("Postrefinement was not done")
if self.mpi_helper.rank == 0:
self.logger.main_log("Postrefinement was not done")
return experiments, reflections
target_symm = symmetry(
unit_cell=self.params.scaling.unit_cell,
space_group_info=self.params.scaling.space_group)
i_model = self.params.scaling.i_model
miller_set = self.params.scaling.miller_set
# Ensure that match_multi_indices() will return identical results
# when a frame's observations are matched against the
# pre-generated Miller set, self.miller_set, and the reference
# data set, self.i_model. The implication is that the same match
# can be used to map Miller indices to array indices for intensity
# accumulation, and for determination of the correlation
# coefficient in the presence of a scaling reference.
assert len(i_model.indices()) == len(miller_set.indices())
assert (i_model.indices() == miller_set.indices()).count(False) == 0
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
experiments_rejected_by_reason = {} # reason:how_many_rejected
for experiment in experiments:
exp_reflections = reflections.select(
reflections['exp_id'] == experiment.identifier)
# Build a miller array for the experiment reflections with original miller indexes
exp_miller_indices_original = miller.set(
target_symm, exp_reflections['miller_index'], True)
observations_original_index = miller.array(
exp_miller_indices_original,
exp_reflections['intensity.sum.value'],
flex.double(
flex.sqrt(exp_reflections['intensity.sum.variance'])))
assert exp_reflections.size() == exp_miller_indices_original.size()
assert observations_original_index.size(
) == exp_miller_indices_original.size()
# Build a miller array for the experiment reflections with asu miller indexes
exp_miller_indices_asu = miller.set(
target_symm, exp_reflections['miller_index_asymmetric'], True)
observations = miller.array(
exp_miller_indices_asu, exp_reflections['intensity.sum.value'],
flex.double(
flex.sqrt(exp_reflections['intensity.sum.variance'])))
matches = miller.match_multi_indices(
miller_indices_unique=miller_set.indices(),
miller_indices=observations.indices())
pair1 = flex.int([pair[1] for pair in matches.pairs()
]) # refers to the observations
pair0 = flex.int([pair[0] for pair in matches.pairs()
]) # refers to the model
assert exp_reflections.size() == exp_miller_indices_original.size()
assert observations_original_index.size(
) == exp_miller_indices_original.size()
# narrow things down to the set that matches, only
observations_pair1_selected = observations.customized_copy(
indices=flex.miller_index(
[observations.indices()[p] for p in pair1]),
data=flex.double([observations.data()[p] for p in pair1]),
sigmas=flex.double([observations.sigmas()[p] for p in pair1]))
observations_original_index_pair1_selected = observations_original_index.customized_copy(
indices=flex.miller_index(
[observations_original_index.indices()[p] for p in pair1]),
data=flex.double(
[observations_original_index.data()[p] for p in pair1]),
sigmas=flex.double(
[observations_original_index.sigmas()[p] for p in pair1]))
I_observed = observations_pair1_selected.data()
MILLER = observations_original_index_pair1_selected.indices()
ORI = crystal_orientation(experiment.crystal.get_A(),
basis_type.reciprocal)
Astar = matrix.sqr(ORI.reciprocal_matrix())
Astar_from_experiment = matrix.sqr(experiment.crystal.get_A())
assert Astar == Astar_from_experiment
WAVE = experiment.beam.get_wavelength()
BEAM = matrix.col((0.0, 0.0, -1. / WAVE))
BFACTOR = 0.
MOSAICITY_DEG = experiment.crystal.get_half_mosaicity_deg()
DOMAIN_SIZE_A = experiment.crystal.get_domain_size_ang()
# calculation of correlation here
I_reference = flex.double(
[i_model.data()[pair[0]] for pair in matches.pairs()])
I_invalid = flex.bool(
[i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()])
use_weights = False # New facility for getting variance-weighted correlation
if use_weights:
# variance weighting
I_weight = flex.double([
1. / (observations_pair1_selected.sigmas()[pair[1]])**2
for pair in matches.pairs()
])
else:
I_weight = flex.double(
len(observations_pair1_selected.sigmas()), 1.)
I_weight.set_selected(I_invalid, 0.)
"""Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement:
include_negatives = True
+ and - reflections both used for Rh distribution for initial estimate of RS parameter
+ and - reflections both used for calc/obs correlation slope for initial estimate of G parameter
+ and - reflections both passed to the refinery and used in the target function (makes sense if
you look at it from a certain point of view)
include_negatives = False
+ and - reflections both used for Rh distribution for initial estimate of RS parameter
+ reflections only used for calc/obs correlation slope for initial estimate of G parameter
+ and - reflections both passed to the refinery and used in the target function (makes sense if
you look at it from a certain point of view)
"""
# RB: By design, for MPI-Merge "include negatives" is implicitly True
SWC = simple_weighted_correlation(I_weight, I_reference,
I_observed)
if self.params.output.log_level == 0:
self.logger.log("Old correlation is: %f" % SWC.corr)
if self.params.postrefinement.algorithm == "rs":
Rhall = flex.double()
for mill in MILLER:
H = matrix.col(mill)
Xhkl = Astar * H
Rh = (Xhkl + BEAM).length() - (1. / WAVE)
Rhall.append(Rh)
Rs = math.sqrt(flex.mean(Rhall * Rhall))
RS = 1. / 10000. # reciprocal effective domain size of 1 micron
RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value
current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.])
parameterization_class = rs_parameterization
refinery = rs_refinery(ORI=ORI,
MILLER=MILLER,
BEAM=BEAM,
WAVE=WAVE,
ICALCVEC=I_reference,
IOBSVEC=I_observed)
elif self.params.postrefinement.algorithm == "eta_deff":
eta_init = 2. * MOSAICITY_DEG * math.pi / 180.
D_eff_init = 2. * DOMAIN_SIZE_A
current = flex.double(
[SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init])
parameterization_class = eta_deff_parameterization
refinery = eta_deff_refinery(ORI=ORI,
MILLER=MILLER,
BEAM=BEAM,
WAVE=WAVE,
ICALCVEC=I_reference,
IOBSVEC=I_observed)
func = refinery.fvec_callable(parameterization_class(current))
functional = flex.sum(func * func)
if self.params.output.log_level == 0:
self.logger.log("functional: %f" % functional)
self.current = current
self.parameterization_class = parameterization_class
self.refinery = refinery
self.observations_pair1_selected = observations_pair1_selected
self.observations_original_index_pair1_selected = observations_original_index_pair1_selected
error_detected = False
try:
self.run_plain()
result_observations_original_index, result_observations, result_matches = self.result_for_cxi_merge(
)
assert result_observations_original_index.size(
) == result_observations.size()
assert result_matches.pairs().size(
) == result_observations_original_index.size()
except (AssertionError, ValueError, RuntimeError) as e:
error_detected = True
reason = repr(e)
if not reason:
reason = "Unknown error"
if not reason in experiments_rejected_by_reason:
experiments_rejected_by_reason[reason] = 1
else:
experiments_rejected_by_reason[reason] += 1
if not error_detected:
new_experiments.append(experiment)
new_exp_reflections = flex.reflection_table()
new_exp_reflections[
'miller_index_asymmetric'] = flex.miller_index(
result_observations.indices())
new_exp_reflections['intensity.sum.value'] = flex.double(
result_observations.data())
new_exp_reflections['intensity.sum.variance'] = flex.double(
flex.pow(result_observations.sigmas(), 2))
new_exp_reflections['exp_id'] = flex.std_string(
len(new_exp_reflections), experiment.identifier)
new_reflections.extend(new_exp_reflections)
'''
# debugging
elif reason.startswith("ValueError"):
self.logger.log("Rejected b/c of value error exp id: %s; unit cell: %s"%(exp_id, str(experiment.crystal.get_unit_cell())) )
'''
# report rejected experiments, reflections
experiments_rejected_by_postrefinement = len(experiments) - len(
new_experiments)
reflections_rejected_by_postrefinement = reflections.size(
) - new_reflections.size()
self.logger.log("Experiments rejected by post-refinement: %d" %
experiments_rejected_by_postrefinement)
self.logger.log("Reflections rejected by post-refinement: %d" %
reflections_rejected_by_postrefinement)
all_reasons = []
for reason, count in experiments_rejected_by_reason.iteritems():
self.logger.log("Experiments rejected due to %s: %d" %
(reason, count))
all_reasons.append(reason)
comm = self.mpi_helper.comm
MPI = self.mpi_helper.MPI
# Collect all rejection reasons from all ranks. Use allreduce to let each rank have all reasons.
all_reasons = comm.allreduce(all_reasons, MPI.SUM)
all_reasons = set(all_reasons)
# Now that each rank has all reasons from all ranks, we can treat the reasons in a uniform way.
total_experiments_rejected_by_reason = {}
for reason in all_reasons:
rejected_experiment_count = 0
if reason in experiments_rejected_by_reason:
rejected_experiment_count = experiments_rejected_by_reason[
reason]
total_experiments_rejected_by_reason[reason] = comm.reduce(
rejected_experiment_count, MPI.SUM, 0)
total_accepted_experiment_count = comm.reduce(len(new_experiments),
MPI.SUM, 0)
if self.mpi_helper.rank == 0:
for reason, count in total_experiments_rejected_by_reason.iteritems(
):
self.logger.main_log(
"Total experiments rejected due to %s: %d" %
(reason, count))
self.logger.main_log("Total experiments accepted: %d" %
total_accepted_experiment_count)
self.logger.log_step_time("POSTREFINEMENT", True)
return new_experiments, new_reflections
def organize_input(self,
observations_pickle,
iparams,
avg_mode,
pickle_filename=None):
"""Given the pickle file, extract and prepare observations object and
the alpha angle (meridional to equatorial).
"""
#get general parameters
if iparams.isoform_name is not None:
if "identified_isoform" not in observations_pickle:
return None, "No identified isoform"
if observations_pickle[
"identified_isoform"] != iparams.isoform_name:
return None, "Identified isoform(%s) is not the requested isoform (%s)" % (
observations_pickle["identified_isoform"],
iparams.isoform_name)
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
img_filename_only = ''
if pickle_filename:
img_filename_only = os.path.basename(pickle_filename)
txt_exception = ' {0:40} ==> '.format(img_filename_only)
#for dials integration pickles - also look for experimentxxx.json
if "miller_index" in observations_pickle:
from dxtbx.model.experiment_list import ExperimentListFactory
exp_json_file = os.path.join(
os.path.dirname(pickle_filename),
img_filename_only.split('_')[0] + '_refined_experiments.json')
if os.path.isfile(exp_json_file):
experiments = ExperimentListFactory.from_json_file(
exp_json_file)
dials_crystal = experiments[0].crystal
detector = experiments[0].detector
beam = experiments[0].beam
crystal_symmetry = crystal.symmetry(
unit_cell=dials_crystal.get_unit_cell().parameters(),
space_group_symbol=iparams.target_space_group)
miller_set_all = miller.set(
crystal_symmetry=crystal_symmetry,
indices=observations_pickle['miller_index'],
anomalous_flag=target_anomalous_flag)
observations = miller_set_all.array(
data=observations_pickle['intensity.sum.value'],
sigmas=flex.sqrt(
observations_pickle['intensity.sum.variance'])
).set_observation_type_xray_intensity()
detector_distance_mm = detector[0].get_distance()
alpha_angle_obs = flex.double([0] * len(observations.data()))
wavelength = beam.get_wavelength()
spot_pred_x_mm = observations_pickle['s1'] #a disguise of s1
spot_pred_y_mm = flex.double([0] * len(observations.data()))
#calculate the crystal orientation
O = sqr(dials_crystal.get_unit_cell().orthogonalization_matrix(
)).transpose()
R = sqr(dials_crystal.get_U()).transpose()
from cctbx.crystal_orientation import crystal_orientation, basis_type
crystal_init_orientation = crystal_orientation(
O * R, basis_type.direct)
else:
txt_exception += exp_json_file + ' not found'
print(txt_exception)
return None, txt_exception
else:
#for cctbx.xfel proceed as usual
observations = observations_pickle["observations"][0]
detector_distance_mm = observations_pickle['distance']
mm_predictions = iparams.pixel_size_mm * (
observations_pickle['mapped_predictions'][0])
xbeam = observations_pickle["xbeam"]
ybeam = observations_pickle["ybeam"]
alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \
for pred in mm_predictions])
spot_pred_x_mm = flex.double(
[pred[0] - xbeam for pred in mm_predictions])
spot_pred_y_mm = flex.double(
[pred[1] - ybeam for pred in mm_predictions])
#Polarization correction
wavelength = observations_pickle["wavelength"]
crystal_init_orientation = observations_pickle[
"current_orientation"][0]
#continue reading...
if iparams.flag_LP_correction and "observations" in observations_pickle:
fx = 1 - iparams.polarization_horizontal_fraction
fy = 1 - fx
if fx > 1.0 or fx < 0:
print(
'Horizontal polarization fraction is not correct. The value must be >= 0 and <= 1'
)
print(
'No polarization correction. Continue with post-refinement'
)
else:
phi_angle_obs = flex.double([math.atan2(pred[1]-ybeam, pred[0]-xbeam) \
for pred in mm_predictions])
bragg_angle_obs = observations.two_theta(wavelength).data()
P = ((fx*((flex.sin(phi_angle_obs)**2)+((flex.cos(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))+\
(fy*((flex.cos(phi_angle_obs)**2)+((flex.sin(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2))))
I_prime = observations.data() / P
sigI_prime = observations.sigmas() / P
observations = observations.customized_copy(
data=flex.double(I_prime), sigmas=flex.double(sigI_prime))
#set observations with target space group - !!! required for correct
#merging due to map_to_asu command.
if iparams.target_crystal_system is not None:
target_crystal_system = iparams.target_crystal_system
else:
target_crystal_system = observations.crystal_symmetry(
).space_group().crystal_system()
lph = lbfgs_partiality_handler()
if iparams.flag_override_unit_cell:
uc_constrained_inp = lph.prep_input(
iparams.target_unit_cell.parameters(), target_crystal_system)
else:
uc_constrained_inp = lph.prep_input(
observations.unit_cell().parameters(), target_crystal_system)
uc_constrained = list(
lph.prep_output(uc_constrained_inp, target_crystal_system))
try:
#apply constrain using the crystal system
miller_set = symmetry(unit_cell=uc_constrained,
space_group_symbol=iparams.target_space_group
).build_miller_set(
anomalous_flag=target_anomalous_flag,
d_min=iparams.merge.d_min)
observations = observations.customized_copy(
anomalous_flag=target_anomalous_flag,
crystal_symmetry=miller_set.crystal_symmetry())
except Exception:
a, b, c, alpha, beta, gamma = uc_constrained
txt_exception += 'Mismatch spacegroup (%6.2f,%6.2f,%6.2f,%6.2f,%6.2f,%6.2f)' % (
a, b, c, alpha, beta, gamma)
print(txt_exception)
return None, txt_exception
#reset systematic absence
sys_absent_negate_flags = flex.bool([
sys_absent_flag[1] == False
for sys_absent_flag in observations.sys_absent_flags()
])
observations = observations.select(sys_absent_negate_flags)
alpha_angle_obs = alpha_angle_obs.select(sys_absent_negate_flags)
spot_pred_x_mm = spot_pred_x_mm.select(sys_absent_negate_flags)
spot_pred_y_mm = spot_pred_y_mm.select(sys_absent_negate_flags)
#remove observations from rejection list
if iparams.rejections:
if pickle_filename in iparams.rejections:
miller_indices_ori_rejected = iparams.rejections[
pickle_filename]
i_sel_flag = flex.bool([True] * len(observations.data()))
cnrej = 0
for miller_index_ori_rejected in miller_indices_ori_rejected:
for i_index_ori, miller_index_ori in enumerate(
observations.indices()):
if miller_index_ori_rejected == miller_index_ori:
i_sel_flag[i_index_ori] = False
cnrej += 1
observations = observations.customized_copy(
indices=observations.indices().select(i_sel_flag),
data=observations.data().select(i_sel_flag),
sigmas=observations.sigmas().select(i_sel_flag))
alpha_angle_obs = alpha_angle_obs.select(i_sel_flag)
spot_pred_x_mm = spot_pred_x_mm.select(i_sel_flag)
spot_pred_y_mm = spot_pred_y_mm.select(i_sel_flag)
#filter resolution
i_sel_res = observations.resolution_filter_selection(
d_max=iparams.merge.d_max, d_min=iparams.merge.d_min)
observations = observations.select(i_sel_res)
alpha_angle_obs = alpha_angle_obs.select(i_sel_res)
spot_pred_x_mm = spot_pred_x_mm.select(i_sel_res)
spot_pred_y_mm = spot_pred_y_mm.select(i_sel_res)
#Filter weak
i_sel = (observations.data() /
observations.sigmas()) > iparams.merge.sigma_min
observations = observations.select(i_sel)
alpha_angle_obs = alpha_angle_obs.select(i_sel)
spot_pred_x_mm = spot_pred_x_mm.select(i_sel)
spot_pred_y_mm = spot_pred_y_mm.select(i_sel)
#filter icering (if on)
if iparams.icering.flag_on:
miller_indices = flex.miller_index()
I_set = flex.double()
sigI_set = flex.double()
alpha_angle_obs_set = flex.double()
spot_pred_x_mm_set = flex.double()
spot_pred_y_mm_set = flex.double()
for miller_index, d, I, sigI, alpha, spot_x, spot_y in zip(
observations.indices(),
observations.d_spacings().data(), observations.data(),
observations.sigmas(), alpha_angle_obs, spot_pred_x_mm,
spot_pred_y_mm):
if d > iparams.icering.d_upper or d < iparams.icering.d_lower:
miller_indices.append(miller_index)
I_set.append(I)
sigI_set.append(sigI)
alpha_angle_obs_set.append(alpha)
spot_pred_x_mm_set.append(spot_x)
spot_pred_y_mm_set.append(spot_y)
observations = observations.customized_copy(indices=miller_indices,
data=I_set,
sigmas=sigI_set)
alpha_angle_obs = alpha_angle_obs_set[:]
spot_pred_x_mm = spot_pred_x_mm_set[:]
spot_pred_y_mm = spot_pred_y_mm_set[:]
#replacing sigI (if set)
if iparams.flag_replace_sigI:
observations = observations.customized_copy(
sigmas=flex.sqrt(observations.data()))
inputs = observations, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation
return inputs, 'OK'
def refine_subgroup(args):
assert len(args) == 4
from dials.command_line.check_indexing_symmetry import (
get_symop_correlation_coefficients,
normalise_intensities,
)
params, subgroup, used_reflections, experiments = args
used_reflections = copy.deepcopy(used_reflections)
triclinic_miller = used_reflections["miller_index"]
cb_op = subgroup["cb_op_inp_best"]
higher_symmetry_miller = cb_op.apply(triclinic_miller)
used_reflections["miller_index"] = higher_symmetry_miller
unrefined_crystal = copy.deepcopy(subgroup.unrefined_crystal)
for expt in experiments:
expt.crystal = unrefined_crystal
from dials.algorithms.indexing.refinement import refine
subgroup.max_cc = None
subgroup.min_cc = None
subgroup.correlation_coefficients = []
subgroup.cc_nrefs = []
with LoggingContext(logging.getLogger(), level=logging.ERROR):
try:
outlier_algorithm = params.refinement.reflections.outlier.algorithm
sel = used_reflections.get_flags(
used_reflections.flags.used_in_refinement)
if sel.all_eq(False):
# Soft outlier rejection if no used_in_refinement flag is set
params.refinement.reflections.outlier.algorithm = "tukey"
iqr_multiplier = (
params.refinement.reflections.outlier.tukey.iqr_multiplier)
params.refinement.reflections.outlier.tukey.iqr_multiplier = (
2 * iqr_multiplier)
sel = ~sel
else:
# Remove reflections not previously used in refinement
params.refinement.reflections.outlier.algorithm = "null"
refinery, refined, outliers = refine(params,
used_reflections.select(sel),
experiments)
params.refinement.reflections.outlier.algorithm = outlier_algorithm
refinery, refined, outliers = refine(params, used_reflections,
refinery.get_experiments())
except RuntimeError as e:
if (str(e) == "scitbx Error: g0 - astry*astry -astrz*astrz <= 0."
or str(e) == "scitbx Error: g1-bstrz*bstrz <= 0."):
subgroup.refined_experiments = None
subgroup.rmsd = None
subgroup.Nmatches = None
else:
raise
else:
dall = refinery.rmsds()
dx = dall[0]
dy = dall[1]
subgroup.rmsd = math.sqrt(dx * dx + dy * dy)
subgroup.Nmatches = len(refinery.get_matches())
subgroup.refined_experiments = refinery.get_experiments()
assert len(subgroup.refined_experiments.crystals()) == 1
subgroup.refined_crystal = subgroup.refined_experiments.crystals(
)[0]
cs = crystal.symmetry(
unit_cell=subgroup.refined_crystal.get_unit_cell(),
space_group=subgroup.refined_crystal.get_space_group(),
)
if "intensity.sum.value" in used_reflections:
# remove refl with -ve variance
sel = used_reflections["intensity.sum.variance"] > 0
good_reflections = used_reflections.select(sel)
from cctbx import miller
ms = miller.set(cs, good_reflections["miller_index"])
ms = ms.array(
good_reflections["intensity.sum.value"] /
flex.sqrt(good_reflections["intensity.sum.variance"]))
if params.normalise:
if params.normalise_bins:
ms = normalise_intensities(
ms, n_bins=params.normalise_bins)
else:
ms = normalise_intensities(ms)
if params.cc_n_bins is not None:
ms.setup_binner(n_bins=params.cc_n_bins)
ccs, nrefs = get_symop_correlation_coefficients(
ms, use_binning=(params.cc_n_bins is not None))
subgroup.correlation_coefficients = ccs
subgroup.cc_nrefs = nrefs
ccs = ccs.select(nrefs > 10)
if len(ccs) > 1:
subgroup.max_cc = flex.max(ccs[1:])
subgroup.min_cc = flex.min(ccs[1:])
return subgroup
def prepare_output(self, results, iparams, avg_mode):
if avg_mode == 'average':
cc_thres = 0
else:
cc_thres = iparams.frame_accept_min_cc
std_filter = iparams.sigma_rejection
if iparams.flag_weak_anomalous:
if avg_mode == 'final':
target_anomalous_flag = iparams.target_anomalous_flag
else:
target_anomalous_flag = False
else:
target_anomalous_flag = iparams.target_anomalous_flag
pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters(
results)
G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean
G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med
G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std
#prepare data for merging
miller_indices_all = flex.miller_index()
miller_indices_ori_all = flex.miller_index()
I_all = flex.double()
sigI_all = flex.double()
G_all = flex.double()
B_all = flex.double()
p_all = flex.double()
rx_all = flex.double()
rs_all = flex.double()
rh_all = flex.double()
SE_all = flex.double()
sin_sq_all = flex.double()
wavelength_all = flex.double()
detector_distance_set = flex.double()
R_init_all = flex.double()
R_final_all = flex.double()
R_xy_init_all = flex.double()
R_xy_final_all = flex.double()
pickle_filename_all = flex.std_string()
filtered_results = []
cn_good_frame, cn_bad_frame_SE, cn_bad_frame_uc, cn_bad_frame_cc, cn_bad_frame_G, cn_bad_frame_re = (
0, 0, 0, 0, 0, 0)
crystal_orientation_dict = {}
for pres in results:
if pres is not None:
pickle_filepath = pres.pickle_filename.split('/')
img_filename = pickle_filepath[len(pickle_filepath) - 1]
flag_pres_ok = True
#check SE, CC, UC, G, B, gamma_e
if math.isnan(pres.G):
flag_pres_ok = False
if math.isnan(pres.SE) or np.isinf(pres.SE):
flag_pres_ok = False
if flag_pres_ok and SE_std > 0:
if abs(pres.SE - SE_med) / SE_std > std_filter:
flag_pres_ok = False
cn_bad_frame_SE += 1
if flag_pres_ok and pres.CC_final < cc_thres:
flag_pres_ok = False
cn_bad_frame_cc += 1
if flag_pres_ok:
if G_std > 0:
if abs(pres.G - G_med) / G_std > std_filter:
flag_pres_ok = False
cn_bad_frame_G += 1
if flag_pres_ok:
if re_std > 0:
if abs(pres.re - re_med) / re_std > std_filter:
flag_pres_ok = False
cn_bad_frame_re += 1
if flag_pres_ok and not good_unit_cell(
pres.uc_params, iparams, iparams.merge.uc_tolerance):
flag_pres_ok = False
cn_bad_frame_uc += 1
data_size = pres.observations.size()
if flag_pres_ok:
cn_good_frame += 1
filtered_results.append(pres)
R_init_all.append(pres.R_init)
R_final_all.append(pres.R_final)
R_xy_init_all.append(pres.R_xy_init)
R_xy_final_all.append(pres.R_xy_final)
miller_indices_all.extend(pres.observations.indices())
miller_indices_ori_all.extend(
pres.observations_original.indices())
I_all.extend(pres.observations.data())
sigI_all.extend(pres.observations.sigmas())
G_all.extend(flex.double([pres.G] * data_size))
B_all.extend(flex.double([pres.B] * data_size))
p_all.extend(pres.partiality)
rs_all.extend(pres.rs_set)
rh_all.extend(pres.rh_set)
sin_sq_all.extend(
pres.observations.two_theta(wavelength=pres.wavelength)
.sin_theta_over_lambda_sq().data())
SE_all.extend(flex.double([pres.SE] * data_size))
wavelength_all.extend(
flex.double([pres.wavelength] * data_size))
detector_distance_set.append(pres.detector_distance_mm)
pickle_filename_all.extend(
flex.std_string([pres.pickle_filename] * data_size))
crystal_orientation_dict[
pres.pickle_filename] = pres.crystal_orientation
#plot stats
self.plot_stats(filtered_results, iparams)
#write out updated crystal orientation as a pickle file
if not iparams.flag_hush:
pickle.dump(crystal_orientation_dict,
open(iparams.run_no + '/' + "crystal.o", "wb"),
pickle.HIGHEST_PROTOCOL)
#calculate average unit cell
uc_mean, uc_med, uc_std = self.calc_mean_unit_cell(filtered_results)
unit_cell_mean = unit_cell(tuple(uc_mean))
#recalculate stats for pr parameters
pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters(
filtered_results)
G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean
G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med
G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std
#from all observations merge them
crystal_symmetry = crystal.symmetry(unit_cell=tuple(uc_mean),
space_group_symbol=str(
iparams.target_space_group))
miller_set_all = miller.set(crystal_symmetry=crystal_symmetry,
indices=miller_indices_all,
anomalous_flag=target_anomalous_flag)
miller_array_all = miller_set_all.array(
data=I_all, sigmas=sigI_all).set_observation_type_xray_intensity()
#sort reflections according to asymmetric-unit symmetry hkl
perm = miller_array_all.sort_permutation(by_value="packed_indices")
miller_indices_all_sort = miller_array_all.indices().select(perm)
miller_indices_ori_all_sort = miller_indices_ori_all.select(perm)
I_obs_all_sort = miller_array_all.data().select(perm)
sigI_obs_all_sort = miller_array_all.sigmas().select(perm)
G_all_sort = G_all.select(perm)
B_all_sort = B_all.select(perm)
p_all_sort = p_all.select(perm)
rs_all_sort = rs_all.select(perm)
wavelength_all_sort = wavelength_all.select(perm)
sin_sq_all_sort = sin_sq_all.select(perm)
SE_all_sort = SE_all.select(perm)
pickle_filename_all_sort = pickle_filename_all.select(perm)
miller_array_uniq = miller_array_all.merge_equivalents().array(
).complete_array(d_min=iparams.merge.d_min, d_max=iparams.merge.d_max)
matches_uniq = miller.match_multi_indices(
miller_indices_unique=miller_array_uniq.indices(),
miller_indices=miller_indices_all_sort)
pair_0 = flex.int([pair[0] for pair in matches_uniq.pairs()])
pair_1 = flex.int([pair[1] for pair in matches_uniq.pairs()])
group_id_list = flex.int(
[pair_0[pair_1[i]] for i in range(len(matches_uniq.pairs()))])
tally = Counter()
for elem in group_id_list:
tally[elem] += 1
cn_group = len(tally)
#preparte txt out stat
txt_out = 'Summary of refinement and merging\n'
txt_out += ' No. good frames: %12.0f\n' % (cn_good_frame)
txt_out += ' No. bad cc frames: %12.0f\n' % (cn_bad_frame_cc)
txt_out += ' No. bad G frames) : %12.0f\n' % (cn_bad_frame_G)
txt_out += ' No. bad unit cell frames: %12.0f\n' % (cn_bad_frame_uc)
txt_out += ' No. bad gamma_e frames: %12.0f\n' % (cn_bad_frame_re)
txt_out += ' No. bad SE: %12.0f\n' % (cn_bad_frame_SE)
txt_out += ' No. observations: %12.0f\n' % (
len(I_obs_all_sort))
txt_out += 'Mean target value (BEFORE: Mean Median (Std.))\n'
txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_init_all), np.median(R_init_all), np.std(R_init_all))
txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_xy_init_all), np.median(R_xy_init_all),
np.std(R_xy_init_all))
txt_out += 'Mean target value (AFTER: Mean Median (Std.))\n'
txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_final_all), np.median(R_final_all), np.std(R_final_all))
txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n' % (
np.mean(R_xy_final_all), np.median(R_xy_final_all),
np.std(R_xy_final_all))
txt_out += ' SE: %12.2f %12.2f (%9.2f)\n' % (
SE_mean, SE_med, SE_std)
txt_out += ' G: %12.3e %12.3e (%9.2e)\n' % (
G_mean, G_med, G_std)
txt_out += ' B: %12.2f %12.2f (%9.2f)\n' % (
B_mean, B_med, B_std)
txt_out += ' Rot.x: %12.2f %12.2f (%9.2f)\n' % (
rotx_mean * 180 / math.pi, rotx_med * 180 / math.pi,
rotx_std * 180 / math.pi)
txt_out += ' Rot.y: %12.2f %12.2f (%9.2f)\n' % (
roty_mean * 180 / math.pi, roty_med * 180 / math.pi,
roty_std * 180 / math.pi)
txt_out += ' gamma_y: %12.5f %12.5f (%9.5f)\n' % (
ry_mean, ry_med, ry_std)
txt_out += ' gamma_z: %12.5f %12.5f (%9.5f)\n' % (
rz_mean, rz_med, rz_std)
txt_out += ' gamma_0: %12.5f %12.5f (%9.5f)\n' % (
r0_mean, r0_med, r0_std)
txt_out += ' gamma_e: %12.5f %12.5f (%9.5f)\n' % (
re_mean, re_med, re_std)
txt_out += ' voigt_nu: %12.5f %12.5f (%9.5f)\n' % (
voigt_nu_mean, voigt_nu_med, voigt_nu_std)
txt_out += ' unit cell\n'
txt_out += ' a: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[0], uc_med[0], uc_std[0])
txt_out += ' b: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[1], uc_med[1], uc_std[1])
txt_out += ' c: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[2], uc_med[2], uc_std[2])
txt_out += ' alpha: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[3], uc_med[3], uc_std[3])
txt_out += ' beta: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[4], uc_med[4], uc_std[4])
txt_out += ' gamma: %12.2f %12.2f (%9.2f)\n' % (
uc_mean[5], uc_med[5], uc_std[5])
txt_out += 'Parmeters from integration (not-refined)\n'
txt_out += ' Wavelength: %12.5f %12.5f (%9.5f)\n' % (
np.mean(wavelength_all), np.median(wavelength_all),
np.std(wavelength_all))
txt_out += ' Detector distance: %12.5f %12.5f (%9.5f)\n' % (
np.mean(detector_distance_set), np.median(detector_distance_set),
np.std(detector_distance_set))
txt_out += '* (standard deviation)\n'
#write out stat. pickle
if not iparams.flag_hush:
stat_dict = {"n_frames_good": [cn_good_frame], \
"n_frames_bad_cc": [cn_bad_frame_cc], \
"n_frames_bad_G": [cn_bad_frame_G], \
"n_frames_bad_uc": [cn_bad_frame_uc], \
"n_frames_bad_gamma_e": [cn_bad_frame_re], \
"n_frames_bad_SE": [cn_bad_frame_SE], \
"n_observations": [len(I_obs_all_sort)], \
"R_start": [np.mean(R_init_all)], \
"R_end": [np.mean(R_final_all)], \
"R_xy_start": [np.mean(R_xy_init_all)], \
"R_xy_end": [np.mean(R_xy_final_all)], \
"mean_gamma_y": [ry_mean], \
"std_gamma_y": [ry_std], \
"mean_gamma_z": [rz_mean], \
"std_gamma_z": [rz_std], \
"mean_gamma_0": [r0_mean], \
"std_gamma_0": [r0_std], \
"mean_gamma_e": [re_mean], \
"std_gamma_e": [re_std], \
"mean_voigt_nu": [voigt_nu_mean], \
"std_voigt_nu": [voigt_nu_std], \
"mean_a": [uc_mean[0]], \
"std_a": [uc_std[0]], \
"mean_b": [uc_mean[1]], \
"std_b": [uc_std[1]], \
"mean_c": [uc_mean[2]], \
"std_c": [uc_std[2]], \
"mean_alpha": [uc_mean[3]], \
"std_alpha": [uc_std[3]], \
"mean_beta": [uc_mean[4]], \
"std_beta": [uc_std[4]], \
"mean_gamma": [uc_mean[5]], \
"std_gamma": [uc_std[5]]}
self.write_stat_pickle(iparams, stat_dict)
return cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, \
I_obs_all_sort, sigI_obs_all_sort,G_all_sort, B_all_sort, \
p_all_sort, rs_all_sort, wavelength_all_sort, sin_sq_all_sort, SE_all_sort, uc_mean, \
np.mean(wavelength_all), pickle_filename_all_sort, txt_out
def refine_subgroup(args):
assert len(args) == 5
from dials.command_line.check_indexing_symmetry \
import get_symop_correlation_coefficients, normalise_intensities
params, subgroup, used_reflections, experiments, refiner_verbosity = args
used_reflections = copy.deepcopy(used_reflections)
triclinic_miller = used_reflections['miller_index']
cb_op = subgroup['cb_op_inp_best']
higher_symmetry_miller = cb_op.apply(triclinic_miller)
used_reflections['miller_index'] = higher_symmetry_miller
unrefined_crystal = copy.deepcopy(subgroup.unrefined_crystal)
for expt in experiments:
expt.crystal = unrefined_crystal
from dials.algorithms.indexing.refinement import refine
subgroup.max_cc = None
subgroup.min_cc = None
subgroup.correlation_coefficients = []
subgroup.cc_nrefs = []
try:
logger = logging.getLogger()
disabled = logger.disabled
logger.disabled = True
iqr_multiplier = params.refinement.reflections.outlier.tukey.iqr_multiplier
params.refinement.reflections.outlier.tukey.iqr_multiplier = 2 * iqr_multiplier
refinery, refined, outliers = refine(params,
used_reflections,
experiments,
verbosity=refiner_verbosity)
params.refinement.reflections.outlier.tukey.iqr_multiplier = iqr_multiplier
refinery, refined, outliers = refine(params,
used_reflections,
refinery.get_experiments(),
verbosity=refiner_verbosity)
except RuntimeError as e:
if (str(e) == "scitbx Error: g0 - astry*astry -astrz*astrz <= 0."
or str(e) == "scitbx Error: g1-bstrz*bstrz <= 0."):
subgroup.refined_crystal = None
subgroup.rmsd = None
subgroup.Nmatches = None
subgroup.scan = None
subgroup.goniometer = None
subgroup.beam = None
subgroup.detector = None
else:
raise
else:
dall = refinery.rmsds()
dx = dall[0]
dy = dall[1]
subgroup.rmsd = math.sqrt(dx * dx + dy * dy)
subgroup.Nmatches = len(refinery.get_matches())
refined_exps = refinery.get_experiments()
subgroup.scan = refined_exps[0].scan
subgroup.goniometer = refined_exps[0].goniometer
subgroup.beam = refined_exps[0].beam
subgroup.detector = refined_exps[0].detector
subgroup.refined_crystal = refined_exps[0].crystal
cs = crystal.symmetry(
unit_cell=subgroup.refined_crystal.get_unit_cell(),
space_group=subgroup.refined_crystal.get_space_group())
if 'intensity.sum.value' in used_reflections:
# remove refl with -ve variance
sel = used_reflections['intensity.sum.variance'] > 0
good_reflections = used_reflections.select(sel)
from cctbx import miller
ms = miller.set(cs, good_reflections['miller_index'])
ms = ms.array(
good_reflections['intensity.sum.value'] /
flex.sqrt(good_reflections['intensity.sum.variance']))
if params.normalise:
if params.normalise_bins:
ms = normalise_intensities(ms,
n_bins=params.normalise_bins)
else:
ms = normalise_intensities(ms)
if params.cc_n_bins is not None:
ms.setup_binner(n_bins=params.cc_n_bins)
ccs, nrefs = get_symop_correlation_coefficients(
ms, use_binning=(params.cc_n_bins is not None))
subgroup.correlation_coefficients = ccs
subgroup.cc_nrefs = nrefs
ccs = ccs.select(nrefs > 10)
if len(ccs) > 1:
subgroup.max_cc = flex.max(ccs[1:])
subgroup.min_cc = flex.min(ccs[1:])
finally:
logger.disabled = disabled
return subgroup
def run(args, command_name = "mmtbx.csv_to_mtz"):
if (len(args) == 0): args = ["--help"]
try:
command_line = (iotbx_option_parser(
usage="%s [reflection_csv_file] [options]" % command_name,
description='Example: %s 1m5u-sf.csv --use_model=1m5u.pdb'%command_name)
.enable_symmetry_comprehensive()
.option(None, "--output_file",
action="store",
default=False,
type="string",
help="Output mtz file name.")
.option(None, "--use_model",
action="store",
default=False,
type="string",
help="Use PDB model to make better guess about reflection data type.")
).process(args=args)
except Exception as e:
if(str(e) != "0"): print(str(e))
sys.exit(0)
crystal_symmetry = command_line.symmetry
if(command_line.symmetry.unit_cell() is None or
command_line.symmetry.space_group_info() is None):
if(command_line.options.use_model):
crystal_symmetry = crystal_symmetry_from_pdb.extract_from(
file_name=command_line.options.use_model)
if(crystal_symmetry.unit_cell() is None or
crystal_symmetry.space_group_info() is None):
raise Sorry(
"Crystal symmetry is not defined. Please use the --symmetry option.\n"
"Type %s without arguments to see more options."%command_name)
if(len(command_line.args) > 1):
print("%d arguments are given from the command line:"% \
len(command_line.args), command_line.args)
raise Sorry("Please specify one reflection csv file.")
file_name = command_line.args[0]
if(not os.path.isfile(file_name)):
raise Sorry("File is not found: %s"%file_name)
data = flex.double()
sigmas = flex.double()
flags = flex.int()
column_ids, columns = parse_csv_file(file_name=file_name)
data_label_root = column_ids[3]
ms = miller.set(crystal_symmetry, flex.miller_index(columns[0]))
for d in columns[1]:
data.append(float(d))
for sig in columns[2]:
sigmas.append(float(sig))
for flag in columns[3]:
flags.append(int(flag))
assert len(data) == len(sigmas)
assert len(flags) == len(data)
ma = miller.array(ms, data, sigmas)
if data_label_root.startswith('F'):
ma.set_observation_type_xray_amplitude()
elif data_label_root.startswith('I'):
ma.set_observation_type_xray_intensity()
else:
ma.set_observation_type_xray_amplitude()
flags_ma = miller.set(
crystal_symmetry = crystal_symmetry,
indices = ma.indices()).array(data = flags)
mtz_dataset = ma.as_mtz_dataset(
column_root_label = data_label_root)
mtz_dataset.add_miller_array(
miller_array = flags_ma,
column_root_label = "R-free-flags")
mtz_object = mtz_dataset.mtz_object()
mtz_object.write(file_name = command_line.options.output_file)
def run(args):
from dials.util.options import OptionParser
import libtbx.load_env
usage = "%s [options] find_spots.json" % (libtbx.env.dispatcher_name)
parser = OptionParser(usage=usage, phil=phil_scope, epilog=help_message)
params, options, args = parser.parse_args(show_diff_phil=True,
return_unhandled=True)
positions = None
if params.positions is not None:
with open(params.positions, 'rb') as f:
positions = flex.vec2_double()
for line in f.readlines():
line = line.replace('(', ' ').replace(')', '').replace(
',', ' ').strip().split()
assert len(line) == 3
i, x, y = [float(l) for l in line]
positions.append((x, y))
assert len(args) == 1
json_file = args[0]
import json
with open(json_file, 'rb') as f:
results = json.load(f)
n_indexed = flex.double()
fraction_indexed = flex.double()
n_spots = flex.double()
n_lattices = flex.double()
crystals = []
image_names = flex.std_string()
for r in results:
n_spots.append(r['n_spots_total'])
image_names.append(str(r['image']))
if 'n_indexed' in r:
n_indexed.append(r['n_indexed'])
n_lattices.append(len(r['lattices']))
for d in r['lattices']:
from dxtbx.model.crystal import CrystalFactory
crystals.append(CrystalFactory.from_dict(d['crystal']))
else:
n_indexed.append(0)
n_lattices.append(0)
if n_indexed.size():
sel = n_spots > 0
fraction_indexed = flex.double(n_indexed.size(), 0)
fraction_indexed.set_selected(
sel,
n_indexed.select(sel) / n_spots.select(sel))
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
blue = '#3498db'
red = '#e74c3c'
marker = 'o'
alpha = 0.5
lw = 0
plot = True
table = True
grid = params.grid
from libtbx import group_args
from dials.algorithms.spot_finding.per_image_analysis \
import plot_stats, print_table
estimated_d_min = flex.double()
d_min_distl_method_1 = flex.double()
d_min_distl_method_2 = flex.double()
n_spots_total = flex.int()
n_spots_no_ice = flex.int()
total_intensity = flex.double()
for d in results:
estimated_d_min.append(d['estimated_d_min'])
d_min_distl_method_1.append(d['d_min_distl_method_1'])
d_min_distl_method_2.append(d['d_min_distl_method_2'])
n_spots_total.append(d['n_spots_total'])
n_spots_no_ice.append(d['n_spots_no_ice'])
total_intensity.append(d['total_intensity'])
stats = group_args(image=image_names,
n_spots_total=n_spots_total,
n_spots_no_ice=n_spots_no_ice,
n_spots_4A=None,
n_indexed=n_indexed,
fraction_indexed=fraction_indexed,
total_intensity=total_intensity,
estimated_d_min=estimated_d_min,
d_min_distl_method_1=d_min_distl_method_1,
d_min_distl_method_2=d_min_distl_method_2,
noisiness_method_1=None,
noisiness_method_2=None)
if plot:
plot_stats(stats)
pyplot.clf()
if table:
print_table(stats)
n_rows = 10
n_rows = min(n_rows, len(n_spots_total))
perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True)
print('Top %i images sorted by number of spots:' % n_rows)
print_table(stats, perm=perm_n_spots_total, n_rows=n_rows)
if flex.max(n_indexed) > 0:
perm_n_indexed = flex.sort_permutation(n_indexed, reverse=True)
print('Top %i images sorted by number of indexed reflections:' %
n_rows)
print_table(stats, perm=perm_n_indexed, n_rows=n_rows)
print("Number of indexed lattices: ", (n_indexed > 0).count(True))
print("Number with valid d_min but failed indexing: ",
((d_min_distl_method_1 > 0) & (d_min_distl_method_2 > 0) &
(estimated_d_min > 0) & (n_indexed == 0)).count(True))
n_bins = 20
spot_count_histogram(n_spots_total,
n_bins=n_bins,
filename='hist_n_spots_total.png',
log=True)
spot_count_histogram(n_spots_no_ice,
n_bins=n_bins,
filename='hist_n_spots_no_ice.png',
log=True)
spot_count_histogram(n_indexed.select(n_indexed > 0),
n_bins=n_bins,
filename='hist_n_indexed.png',
log=False)
if len(crystals):
plot_unit_cell_histograms(crystals)
if params.stereographic_projections and len(crystals):
from dxtbx.datablock import DataBlockFactory
datablocks = DataBlockFactory.from_filenames([image_names[0]],
verbose=False)
assert len(datablocks) == 1
imageset = datablocks[0].extract_imagesets()[0]
s0 = imageset.get_beam().get_s0()
# XXX what if no goniometer?
rotation_axis = imageset.get_goniometer().get_rotation_axis()
indices = ((1, 0, 0), (0, 1, 0), (0, 0, 1))
for i, index in enumerate(indices):
from cctbx import crystal, miller
from scitbx import matrix
miller_indices = flex.miller_index([index])
symmetry = crystal.symmetry(
unit_cell=crystals[0].get_unit_cell(),
space_group=crystals[0].get_space_group())
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
d_spacings = d_spacings.generate_bijvoet_mates()
miller_indices = d_spacings.indices()
# plane normal
d0 = matrix.col(s0).normalize()
d1 = d0.cross(matrix.col(rotation_axis)).normalize()
d2 = d1.cross(d0).normalize()
reference_poles = (d0, d1, d2)
from dials.command_line.stereographic_projection import stereographic_projection
projections = []
for cryst in crystals:
reciprocal_space_points = list(
cryst.get_U() *
cryst.get_B()) * miller_indices.as_vec3_double()
projections.append(
stereographic_projection(reciprocal_space_points,
reference_poles))
#from dials.algorithms.indexing.compare_orientation_matrices import \
# difference_rotation_matrix_and_euler_angles
#R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles(
# crystals[0], cryst)
#print max(euler_angles)
from dials.command_line.stereographic_projection import plot_projections
plot_projections(projections,
filename='projections_%s.png' % ('hkl'[i]))
pyplot.clf()
def plot_grid(values,
grid,
file_name,
cmap=pyplot.cm.Reds,
vmin=None,
vmax=None,
invalid='white'):
values = values.as_double()
# At DLS, fast direction appears to be largest direction
if grid[0] > grid[1]:
values.reshape(flex.grid(reversed(grid)))
values = values.matrix_transpose()
else:
values.reshape(flex.grid(grid))
Z = values.as_numpy_array()
#f, (ax1, ax2) = pyplot.subplots(2)
f, ax1 = pyplot.subplots(1)
mesh1 = ax1.pcolormesh(values.as_numpy_array(),
cmap=cmap,
vmin=vmin,
vmax=vmax)
mesh1.cmap.set_under(color=invalid, alpha=None)
mesh1.cmap.set_over(color=invalid, alpha=None)
#mesh2 = ax2.contour(Z, cmap=cmap, vmin=vmin, vmax=vmax)
#mesh2 = ax2.contourf(Z, cmap=cmap, vmin=vmin, vmax=vmax)
ax1.set_aspect('equal')
ax1.invert_yaxis()
#ax2.set_aspect('equal')
#ax2.invert_yaxis()
pyplot.colorbar(mesh1, ax=ax1)
#pyplot.colorbar(mesh2, ax=ax2)
pyplot.savefig(file_name, dpi=600)
pyplot.clf()
def plot_positions(values,
positions,
file_name,
cmap=pyplot.cm.Reds,
vmin=None,
vmax=None,
invalid='white'):
values = values.as_double()
assert positions.size() >= values.size()
positions = positions[:values.size()]
if vmin is None:
vmin = flex.min(values)
if vmax is None:
vmax = flex.max(values)
x, y = positions.parts()
dx = flex.abs(x[1:] - x[:-1])
dy = flex.abs(y[1:] - y[:-1])
dx = dx.select(dx > 0)
dy = dy.select(dy > 0)
scale = 1 / flex.min(dx)
#print scale
x = (x * scale).iround()
y = (y * scale).iround()
from libtbx.math_utils import iceil
z = flex.double(
flex.grid(iceil(flex.max(y)) + 1,
iceil(flex.max(x)) + 1), -2)
#print z.all()
for x_, y_, z_ in zip(x, y, values):
z[y_, x_] = z_
plot_grid(z.as_1d(),
z.all(),
file_name,
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=invalid)
return
if grid is not None or positions is not None:
if grid is not None:
positions = tuple(reversed(grid))
plotter = plot_grid
else:
plotter = plot_positions
cmap = pyplot.get_cmap(params.cmap)
plotter(n_spots_total,
positions,
'grid_spot_count_total.png',
cmap=cmap,
invalid=params.invalid)
plotter(n_spots_no_ice,
positions,
'grid_spot_count_no_ice.png',
cmap=cmap,
invalid=params.invalid)
plotter(total_intensity,
positions,
'grid_total_intensity.png',
cmap=cmap,
invalid=params.invalid)
if flex.max(n_indexed) > 0:
plotter(n_indexed,
positions,
'grid_n_indexed.png',
cmap=cmap,
invalid=params.invalid)
plotter(fraction_indexed,
positions,
'grid_fraction_indexed.png',
cmap=cmap,
vmin=0,
vmax=1,
invalid=params.invalid)
for i, d_min in enumerate(
(estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)):
from cctbx import uctbx
d_star_sq = uctbx.d_as_d_star_sq(d_min)
d_star_sq.set_selected(d_star_sq == 1, 0)
vmin = flex.min(d_star_sq.select(d_star_sq > 0))
vmax = flex.max(d_star_sq)
vmin = flex.min(d_min.select(d_min > 0))
vmax = flex.max(d_min)
cmap = pyplot.get_cmap('%s_r' % params.cmap)
d_min.set_selected(d_min <= 0, vmax)
if i == 0:
plotter(d_min,
positions,
'grid_d_min.png',
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=params.invalid)
else:
plotter(d_min,
positions,
'grid_d_min_method_%i.png' % i,
cmap=cmap,
vmin=vmin,
vmax=vmax,
invalid=params.invalid)
if flex.max(n_indexed) > 0:
pyplot.hexbin(n_spots,
n_indexed,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(n_spots, n_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
xlim = pyplot.xlim()
ylim = pyplot.ylim()
pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('# spots')
pyplot.ylabel('# indexed')
pyplot.savefig('n_spots_vs_n_indexed.png')
pyplot.clf()
pyplot.hexbin(n_spots,
fraction_indexed,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('Fraction indexed')
pyplot.savefig('n_spots_vs_fraction_indexed.png')
pyplot.clf()
pyplot.hexbin(n_indexed,
fraction_indexed,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_indexed, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# indexed')
pyplot.ylabel('Fraction indexed')
pyplot.savefig('n_indexed_vs_fraction_indexed.png')
pyplot.clf()
pyplot.hexbin(n_spots,
n_lattices,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, n_lattices, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('# lattices')
pyplot.savefig('n_spots_vs_n_lattices.png')
pyplot.clf()
#pyplot.scatter(
# estimated_d_min, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(estimated_d_min,
d_min_distl_method_1,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(estimated_d_min), max(d_min_distl_method_1))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('estimated_d_min')
pyplot.ylabel('d_min_distl_method_1')
pyplot.savefig('d_min_vs_distl_method_1.png')
pyplot.clf()
#pyplot.scatter(
# estimated_d_min, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(estimated_d_min,
d_min_distl_method_2,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(estimated_d_min), max(d_min_distl_method_2))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('estimated_d_min')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('d_min_vs_distl_method_2.png')
pyplot.clf()
#pyplot.scatter(
# d_min_distl_method_1, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.hexbin(d_min_distl_method_1,
d_min_distl_method_2,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.gca().set_aspect('equal')
xlim = pyplot.xlim()
ylim = pyplot.ylim()
m = max(max(d_min_distl_method_1), max(d_min_distl_method_2))
pyplot.plot([0, m], [0, m], c=red)
pyplot.xlim(0, xlim[1])
pyplot.ylim(0, ylim[1])
pyplot.xlabel('d_min_distl_method_1')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('distl_method_1_vs_distl_method_2.png')
pyplot.clf()
pyplot.hexbin(n_spots,
estimated_d_min,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, estimated_d_min, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('estimated_d_min')
pyplot.savefig('n_spots_vs_d_min.png')
pyplot.clf()
pyplot.hexbin(n_spots,
d_min_distl_method_1,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('d_min_distl_method_1')
pyplot.savefig('n_spots_vs_distl_method_1.png')
pyplot.clf()
pyplot.hexbin(n_spots,
d_min_distl_method_2,
bins='log',
cmap=pyplot.cm.jet,
gridsize=50)
pyplot.colorbar()
#pyplot.scatter(
#n_spots, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw)
pyplot.xlim(0, pyplot.xlim()[1])
pyplot.ylim(0, pyplot.ylim()[1])
pyplot.xlabel('# spots')
pyplot.ylabel('d_min_distl_method_2')
pyplot.savefig('n_spots_vs_distl_method_2.png')
pyplot.clf()
def exercise(space_group_info,
anomalous_flag,
n_scatterers=8,
d_min=2,
verbose=0):
structure = random_structure.xray_structure(space_group_info,
elements=["const"] *
n_scatterers)
f_calc = structure.structure_factors(
d_min=d_min, anomalous_flag=anomalous_flag).f_calc()
f = abs(f_calc)
fs = miller.array(miller_set=f, data=f.data(), sigmas=flex.sqrt(f.data()))
assert fs.is_unique_set_under_symmetry()
for a in (f, fs):
for algorithm in ["gaussian", "shelx"]:
m = a.merge_equivalents(algorithm=algorithm)
m.show_summary(out=StringIO())
j = m.array().adopt_set(a)
assert flex.linear_correlation(j.data(),
a.data()).coefficient() > 1 - 1.e-6
if (a.sigmas() is not None):
assert flex.linear_correlation(
j.sigmas(), a.sigmas()).coefficient() > 1 - 1.e-6
redundancies = flex.size_t()
for i in xrange(fs.indices().size()):
redundancies.append(random.randrange(5) + 1)
space_group = space_group_info.group()
r_indices = flex.miller_index()
r_data = flex.double()
r_sigmas = flex.double()
for i, n in enumerate(redundancies):
h = fs.indices()[i]
h_eq = miller.sym_equiv_indices(space_group, h).indices()
for j in xrange(n):
r_indices.append(h_eq[random.randrange(len(h_eq))].h())
r_data.append(fs.data()[i])
r_sigmas.append(fs.sigmas()[i])
r = miller.array(miller_set=miller.set(crystal_symmetry=fs,
indices=r_indices,
anomalous_flag=fs.anomalous_flag()),
data=r_data,
sigmas=r_sigmas)
assert not r.is_unique_set_under_symmetry()
noise = flex.random_double(size=r.indices().size())
r = r.sort(by_value=noise)
for algorithm in ["gaussian", "shelx"]:
m = r.merge_equivalents(algorithm=algorithm)
m.show_summary(out=StringIO())
j = m.array().adopt_set(fs)
assert j.is_unique_set_under_symmetry()
assert flex.linear_correlation(j.data(),
fs.data()).coefficient() > 1 - 1.e-6
fssr = fs.sigmas() / flex.sqrt(redundancies.as_double())
assert flex.linear_correlation(j.sigmas(),
fssr).coefficient() > 1 - 1.e-6
#
if (anomalous_flag):
f_calc_ave = f_calc.average_bijvoet_mates() # uses merge_equivalents
f_calc_com = f_calc.as_non_anomalous_array().common_set(f_calc_ave)
assert f_calc_com.indices().all_eq(f_calc_ave.indices())
for part in [flex.real, flex.imag]:
assert flex.linear_correlation(part(
f_calc_com.data()), part(
f_calc_ave.data())).coefficient() > 1 - 1.e-6
# test use_internal_variance=False
m = r.merge_equivalents(algorithm="gaussian", use_internal_variance=False)
j = m.array().adopt_set(fs)
fssr = fs.sigmas() / flex.sqrt(redundancies.as_double())
assert flex.linear_correlation(j.sigmas(), fssr).coefficient() > 1 - 1.e-6
def run(args):
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
# The script usage
usage = "dials.stereographic_projection [options] [param.phil] indexed.expt"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message,
)
params, options = parser.parse_args(args=args, show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if not experiments:
parser.print_help()
return
if not params.hkl and params.hkl_limit is None:
sys.exit("Please provide hkl or hkl_limit parameters.")
if params.plot.labels and len(params.plot.labels) != len(experiments):
sys.exit(
"Number of labels (%i) must equal number of experiments (%i)"
% (len(params.plot.labels), len(experiments))
)
if params.hkl is not None and len(params.hkl):
miller_indices = flex.miller_index(params.hkl)
elif params.hkl_limit is not None:
limit = params.hkl_limit
miller_indices = flex.miller_index()
for h in range(-limit, limit + 1):
for k in range(-limit, limit + 1):
for l in range(-limit, limit + 1):
if (h, k, l) == (0, 0, 0):
continue
miller_indices.append((h, k, l))
crystals = experiments.crystals()
symmetry = crystal.symmetry(
unit_cell=crystals[0].get_unit_cell(), space_group=crystals[0].get_space_group()
)
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
if params.eliminate_sys_absent:
d_spacings = d_spacings.eliminate_sys_absent()
if params.expand_to_p1:
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
miller_indices = d_spacings.indices()
# find the greatest common factor (divisor) between miller indices
miller_indices_unique = flex.miller_index()
for hkl in miller_indices:
gcd = gcd_list(hkl)
if gcd > 1:
miller_indices_unique.append(tuple(int(h / gcd) for h in hkl))
elif gcd < 1:
pass
else:
miller_indices_unique.append(hkl)
miller_indices = miller_indices_unique
miller_indices = flex.miller_index(list(set(miller_indices)))
ref_crystal = crystals[0]
U = matrix.sqr(ref_crystal.get_U())
B = matrix.sqr(ref_crystal.get_B())
R = matrix.identity(3)
if params.frame == "laboratory":
reference_poles = reference_poles_perpendicular_to_beam(
experiments[0].beam, experiments[0].goniometer
)
if params.use_starting_angle:
rotation_axis = matrix.col(experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
experiments[0].scan.get_oscillation()[0], deg=True
)
elif params.phi_angle != 0:
rotation_axis = matrix.col(experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
params.phi_angle, deg=True
)
else:
if params.plane_normal is not None:
plane_normal = params.plane_normal
else:
plane_normal = (0, 0, 1)
reference_poles = reference_poles_crystal(
ref_crystal, plane_normal=plane_normal
)
if params.frame == "crystal":
U = matrix.identity(3)
reciprocal_space_points = list(R * U * B) * miller_indices.as_vec3_double()
projections_ref = stereographic_projection(reciprocal_space_points, reference_poles)
projections_all = [projections_ref]
if experiments:
from dials.algorithms.indexing.compare_orientation_matrices import (
difference_rotation_matrix_axis_angle,
)
for expt in experiments[1:]:
cryst = expt.crystal
if params.frame == "crystal":
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
ref_crystal, cryst
)
U = R_ij
elif params.use_starting_angle:
if params.use_starting_angle:
rotation_axis = matrix.col(expt.goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
expt.scan.get_oscillation()[0], deg=True
)
else:
U = matrix.sqr(cryst.get_U())
reciprocal_space_points = (
list(R * U * matrix.sqr(cryst.get_B()))
* miller_indices.as_vec3_double()
)
projections = stereographic_projection(
reciprocal_space_points, reference_poles
)
projections_all.append(projections)
if params.save_coordinates:
with open("projections.txt", "w") as f:
f.write("crystal h k l x y" + os.linesep)
for i_cryst, projections in enumerate(projections_all):
for hkl, proj in zip(miller_indices, projections):
f.write("%i " % (i_cryst + 1))
f.write("%i %i %i " % hkl)
f.write(("%f %f" + os.linesep) % proj)
if params.plot.filename:
epochs = None
if params.plot.colour_map is not None:
if experiments[0].scan is not None:
epochs = [expt.scan.get_epochs()[0] for expt in experiments]
else:
epochs = [i for i, expt in enumerate(experiments)]
plot_projections(
projections_all,
filename=params.plot.filename,
colours=params.plot.colours,
marker_size=params.plot.marker_size,
font_size=params.plot.font_size,
gridsize=params.plot.gridsize,
label_indices=miller_indices if params.plot.label_indices else False,
epochs=epochs,
colour_map=params.plot.colour_map,
)
if params.json.filename:
projections_as_json(
projections_all, filename=params.json.filename, labels=params.plot.labels
)
def run(self, experiments, reflections):
self.logger.log_step_time("SCALE_FRAMES")
if self.params.scaling.algorithm != "mark0": # mark1 implies no scaling/post-refinement
self.logger.log("No scaling was done")
if self.mpi_helper.rank == 0:
self.logger.main_log("No scaling was done")
return experiments, reflections
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
# scale experiments, one at a time. Reject experiments that do not correlate with the reference or fail to scale.
results = []
slopes = []
correlations = []
high_res_experiments = 0
experiments_rejected_because_of_low_signal = 0
experiments_rejected_because_of_low_correlation_with_reference = 0
target_symm = symmetry(
unit_cell=self.params.scaling.unit_cell,
space_group_info=self.params.scaling.space_group)
for experiment in experiments:
exp_reflections = reflections.select(
reflections['exp_id'] == experiment.identifier)
# Build a miller array for the experiment reflections
exp_miller_indices = miller.set(
target_symm, exp_reflections['miller_index_asymmetric'], True)
exp_intensities = miller.array(
exp_miller_indices, exp_reflections['intensity.sum.value'],
flex.double(
flex.sqrt(exp_reflections['intensity.sum.variance'])))
model_intensities = self.params.scaling.i_model
# Extract an array of HKLs from the model to match the experiment HKLs
matching_indices = miller.match_multi_indices(
miller_indices_unique=model_intensities.indices(),
miller_indices=exp_intensities.indices())
# Least squares
if self.params.scaling.mark0.fit_reference_to_experiment: # RB: in cxi-merge we fit reference to experiment, but we should really do it the other way
result = self.fit_reference_to_experiment(
model_intensities, exp_intensities, matching_indices)
else:
result = self.fit_experiment_to_reference(
model_intensities, exp_intensities, matching_indices)
if result.error == scaling_result.err_low_signal:
experiments_rejected_because_of_low_signal += 1
continue
elif result.error == scaling_result.err_low_correlation:
experiments_rejected_because_of_low_correlation_with_reference += 1
continue
slopes.append(result.slope)
correlations.append(result.correlation)
if self.params.output.log_level == 0:
self.logger.log(
"Experiment ID: %s; Slope: %f; Correlation %f" %
(experiment.identifier, result.slope, result.correlation))
# count high resolution experiments
if exp_intensities.d_min() <= self.params.merging.d_min:
high_res_experiments += 1
# apply scale factors
if not self.params.postrefinement.enable:
if self.params.scaling.mark0.fit_reference_to_experiment:
exp_reflections['intensity.sum.value'] /= result.slope
exp_reflections['intensity.sum.variance'] /= (
result.slope**2)
else:
exp_reflections['intensity.sum.value'] *= result.slope
exp_reflections['intensity.sum.variance'] *= (
result.slope**2)
new_experiments.append(experiment)
new_reflections.extend(exp_reflections)
rejected_experiments = len(experiments) - len(new_experiments)
assert rejected_experiments == experiments_rejected_because_of_low_signal + \
experiments_rejected_because_of_low_correlation_with_reference
reflections_removed_because_of_rejected_experiments = reflections.size(
) - new_reflections.size()
self.logger.log("Experiments rejected because of low signal: %d" %
experiments_rejected_because_of_low_signal)
self.logger.log(
"Experiments rejected because of low correlation with reference: %d"
% experiments_rejected_because_of_low_correlation_with_reference)
self.logger.log(
"Reflections rejected because of rejected experiments: %d" %
reflections_removed_because_of_rejected_experiments)
self.logger.log("High resolution experiments: %d" %
high_res_experiments)
if self.params.postrefinement.enable:
self.logger.log(
"Note: scale factors were not applied, because postrefinement is enabled"
)
# MPI-reduce all counts
comm = self.mpi_helper.comm
MPI = self.mpi_helper.MPI
total_experiments_rejected_because_of_low_signal = comm.reduce(
experiments_rejected_because_of_low_signal, MPI.SUM, 0)
total_experiments_rejected_because_of_low_correlation_with_reference = comm.reduce(
experiments_rejected_because_of_low_correlation_with_reference,
MPI.SUM, 0)
total_reflections_removed_because_of_rejected_experiments = comm.reduce(
reflections_removed_because_of_rejected_experiments, MPI.SUM, 0)
total_high_res_experiments = comm.reduce(high_res_experiments, MPI.SUM,
0)
all_slopes = comm.reduce(slopes, MPI.SUM, 0)
all_correlations = comm.reduce(correlations, MPI.SUM, 0)
# rank 0: log data statistics
if self.mpi_helper.rank == 0:
self.logger.main_log(
'Experiments rejected because of low signal: %d' %
total_experiments_rejected_because_of_low_signal)
self.logger.main_log(
'Experiments rejected because of low correlation with reference: %d'
%
total_experiments_rejected_because_of_low_correlation_with_reference
)
self.logger.main_log(
'Reflections rejected because of rejected experiments: %d' %
total_reflections_removed_because_of_rejected_experiments)
self.logger.main_log(
'Experiments with high resolution of %5.2f Angstrom or better: %d'
% (self.params.merging.d_min, total_high_res_experiments))
if len(all_slopes) > 0:
stats_slope = flex.mean_and_variance(flex.double(all_slopes))
self.logger.main_log(
'Average experiment scale factor wrt reference: %f' %
(stats_slope.mean()))
if len(all_correlations) > 0:
stats_correlation = flex.mean_and_variance(
flex.double(all_correlations))
self.logger.main_log(
'Average experiment correlation with reference: %f +/- %f'
%
(stats_correlation.mean(),
stats_correlation.unweighted_sample_standard_deviation()))
if self.params.postrefinement.enable:
self.logger.main_log(
"Note: scale factors were not applied, because postrefinement is enabled"
)
self.logger.log_step_time("SCALE_FRAMES", True)
return new_experiments, new_reflections
def finite_diffs_aniso(p_scale, u_star, centric=False, h=0.0001):
d_star_sq = flex.double(2, 0.25)
f_obs = flex.double(2, 1.0)
centric_array = flex.bool(2, centric)
sigma_f_obs = f_obs / 10.0
sigma_sq = flex.double(2, 1.0)
epsilon = flex.double(2, 1.0)
gamma = flex.double(2, 0.0)
unit_cell = uctbx.unit_cell('20, 30, 40, 90.0, 90.0, 90.0')
mi = flex.miller_index(((1, 2, 3), (1, 2, 3)))
xs = crystal.symmetry((20, 30, 40), "P 2 2 2")
ms = miller.set(xs, mi)
nll_norm = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale,
unit_cell, u_star)
nll_scale = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale + h,
unit_cell, u_star)
u_star[0] += h
nll_u11 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale,
unit_cell, u_star)
u_star[0] -= h
u_star[1] += h
nll_u22 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale,
unit_cell, u_star)
u_star[1] -= h
u_star[2] += h
nll_u33 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale,
unit_cell, u_star)
u_star[2] -= h
u_star[3] += h
nll_u12 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale,
unit_cell, u_star)
u_star[3] -= h
u_star[4] += h
nll_u13 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale,
unit_cell, u_star)
u_star[4] -= h
u_star[5] += h
nll_u23 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale,
unit_cell, u_star)
g = scaling.wilson_single_nll_aniso_gradient(ms.indices()[0], f_obs[0],
sigma_f_obs[0], epsilon[0],
sigma_sq[0], gamma[0],
centric_array[0], p_scale,
unit_cell, u_star)
g2 = scaling.wilson_total_nll_aniso_gradient(ms.indices(), f_obs,
sigma_f_obs, epsilon,
sigma_sq, gamma,
centric_array, p_scale,
unit_cell, u_star)
ds = (nll_norm - nll_scale) / -h
du11 = (nll_norm - nll_u11) / -h
du22 = (nll_norm - nll_u22) / -h
du33 = (nll_norm - nll_u33) / -h
du12 = (nll_norm - nll_u12) / -h
du13 = (nll_norm - nll_u13) / -h
du23 = (nll_norm - nll_u23) / -h
assert approx_equal(ds, g[0]), (ds, g[0])
assert approx_equal(du11, g[1]), (du11, g[1])
assert approx_equal(du22, g[2])
assert approx_equal(du33, g[3])
assert approx_equal(du12, g[4])
assert approx_equal(du13, g[5])
assert approx_equal(du23, g[6])
assert approx_equal(ds, g2[0] / 2.0)
assert approx_equal(du11, g2[1] / 2.0)
assert approx_equal(du22, g2[2] / 2.0)
assert approx_equal(du33, g2[3] / 2.0)
assert approx_equal(du12, g2[4] / 2.0)
assert approx_equal(du13, g2[5] / 2.0)
assert approx_equal(du23, g2[6] / 2.0)
def filtered_arrays_from_experiments_reflections(
experiments,
reflections,
outlier_rejection_after_filter=False,
partiality_threshold=0.99,
):
"""Create a list of filtered arrays from experiments and reflections.
A partiality threshold can be set, and if outlier_rejection_after_filter
is True, and intensity.scale values are not present, then a round of
outlier rejection will take place.
Raises:
ValueError: if no datasets remain after filtering.
"""
miller_arrays = []
ids_to_del = []
for idx, (expt, refl) in enumerate(zip(experiments, reflections)):
crystal_symmetry = crystal.symmetry(
unit_cell=expt.crystal.get_unit_cell(),
space_group=expt.crystal.get_space_group(),
)
# want to use scale intensities if present, else sum + prf (if available)
if "intensity.scale.value" in refl:
intensity_choice = ["scale"]
intensity_to_use = "intensity.scale"
else:
assert "intensity.sum.value" in refl
intensity_to_use = "intensity.sum"
intensity_choice = ["sum"]
if "intensity.prf.value" in refl:
intensity_choice.append("profile")
intensity_to_use = "intensity.prf"
try:
logger.info("Filtering reflections for dataset %s" %
(expt.identifier if expt.identifier else idx))
refl = filter_reflection_table(
refl,
intensity_choice,
min_isigi=-5,
filter_ice_rings=False,
combine_partials=True,
partiality_threshold=partiality_threshold,
)
except ValueError:
logger.info(
"Dataset %s removed as no reflections left after filtering",
idx)
ids_to_del.append(idx)
else:
# If scale was chosen - will return scale or have raised ValueError
# If prf or sum, possible was no prf but want to continue.
try:
refl["intensity"] = refl[intensity_to_use + ".value"]
refl["variance"] = refl[intensity_to_use + ".variance"]
except KeyError: # catch case where prf were removed.
refl["intensity"] = refl["intensity.sum.value"]
refl["variance"] = refl["intensity.sum.variance"]
if outlier_rejection_after_filter and intensity_to_use != "intensity.scale":
refl = reject_outliers(refl, expt, method="simple", zmax=12.0)
refl = refl.select(
~refl.get_flags(refl.flags.outlier_in_scaling))
miller_set = miller.set(crystal_symmetry,
refl["miller_index"],
anomalous_flag=False)
intensities = miller.array(miller_set,
data=refl["intensity"],
sigmas=flex.sqrt(refl["variance"]))
intensities.set_observation_type_xray_intensity()
intensities.set_info(
miller.array_info(source="DIALS", source_type="pickle"))
miller_arrays.append(intensities)
if not miller_arrays:
raise ValueError(
"""No datasets remain after pre-filtering. Please check input data.
The datasets may not contain any full reflections; the command line
option partiality_threshold can be lowered to include partials.""")
for id_ in ids_to_del[::-1]:
del experiments[id_]
del reflections[id_]
return miller_arrays
def run(args):
import libtbx.load_env
from dials.util.options import OptionParser
from dials.util.options import flatten_experiments
# The script usage
usage = "usage: %s [options] [param.phil] experiments.json" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
read_experiments=True,
check_format=False,
epilog=help_message)
params, options = parser.parse_args(show_diff_phil=True)
experiments = flatten_experiments(params.input.experiments)
if not experiments:
parser.print_help()
return
if not params.hkl and params.hkl_limit is None:
from libtbx.utils import Sorry
raise Sorry("Please provide hkl or hkl_limit parameters.")
if params.hkl is not None and len(params.hkl):
miller_indices = flex.miller_index(params.hkl)
elif params.hkl_limit is not None:
limit = params.hkl_limit
miller_indices = flex.miller_index()
for h in range(-limit, limit + 1):
for k in range(-limit, limit + 1):
for l in range(-limit, limit + 1):
if (h, k, l) == (0, 0, 0): continue
miller_indices.append((h, k, l))
crystals = experiments.crystals()
symmetry = crystal.symmetry(unit_cell=crystals[0].get_unit_cell(),
space_group=crystals[0].get_space_group())
miller_set = miller.set(symmetry, miller_indices)
d_spacings = miller_set.d_spacings()
if params.eliminate_sys_absent:
d_spacings = d_spacings.eliminate_sys_absent()
if params.expand_to_p1:
d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1()
d_spacings = d_spacings.generate_bijvoet_mates()
miller_indices = d_spacings.indices()
# find the greatest common factor (divisor) between miller indices
miller_indices_unique = flex.miller_index()
for hkl in miller_indices:
gcd = gcd_list(hkl)
if gcd > 1:
miller_indices_unique.append(tuple(int(h / gcd) for h in hkl))
elif gcd < 1:
pass
else:
miller_indices_unique.append(hkl)
miller_indices = miller_indices_unique
miller_indices = flex.miller_index(list(set(miller_indices)))
ref_crystal = crystals[0]
A = ref_crystal.get_A()
U = ref_crystal.get_U()
B = ref_crystal.get_B()
R = matrix.identity(3)
if params.frame == 'laboratory':
reference_poles = reference_poles_perpendicular_to_beam(
experiments[0].beam, experiments[0].goniometer)
if params.use_starting_angle:
rotation_axis = matrix.col(
experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
experiments[0].scan.get_oscillation()[0], deg=True)
elif params.phi_angle != 0:
rotation_axis = matrix.col(
experiments[0].goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
params.phi_angle, deg=True)
else:
if params.plane_normal is not None:
plane_normal = params.plane_normal
else:
plane_normal = (0, 0, 1)
reference_poles = reference_poles_crystal(ref_crystal,
plane_normal=plane_normal)
if params.frame == 'crystal':
U = matrix.identity(3)
reciprocal_space_points = list(R * U * B) * miller_indices.as_vec3_double()
projections_ref = stereographic_projection(reciprocal_space_points,
reference_poles)
projections_all = [projections_ref]
if experiments:
from dials.algorithms.indexing.compare_orientation_matrices import \
difference_rotation_matrix_axis_angle
for expt in experiments[1:]:
cryst = expt.crystal
if params.frame == 'crystal':
R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle(
ref_crystal, cryst)
U = R_ij
elif params.use_starting_angle:
if params.use_starting_angle:
rotation_axis = matrix.col(
expt.goniometer.get_rotation_axis())
R = rotation_axis.axis_and_angle_as_r3_rotation_matrix(
expt.scan.get_oscillation()[0], deg=True)
else:
U = cryst.get_U()
reciprocal_space_points = list(
R * U * cryst.get_B()) * miller_indices.as_vec3_double()
projections = stereographic_projection(reciprocal_space_points,
reference_poles)
projections_all.append(projections)
if params.save_coordinates:
with open('projections.txt', 'wb') as f:
print >> f, "crystal h k l x y"
for i_cryst, projections in enumerate(projections_all):
for hkl, proj in zip(miller_indices, projections):
print >> f, "%i" % (i_cryst + 1),
print >> f, "%i %i %i" % hkl,
print >> f, "%f %f" % proj
if params.plot.show or params.plot.filename:
plot_projections(projections_all,
filename=params.plot.filename,
show=params.plot.show,
colours=params.plot.colours,
marker_size=params.plot.marker_size,
font_size=params.plot.font_size,
label_indices=params.plot.label_indices)
def add_miller_array(self,
array,
array_type=None,
column_name=None,
column_names=None):
"""
Accepts a miller array, and one of array_type, column_name or column_names.
"""
assert [array_type, column_name, column_names].count(None) == 2
if array_type is not None:
assert array_type in ('calc', 'meas')
elif column_name is not None:
column_names = [column_name]
if array.is_complex_array():
if column_names is None:
column_names = [
self.prefix + 'F_' + array_type,
self.prefix + 'phase_' + array_type
]
else:
assert len(column_names) == 2
if (('_A_' in column_names[0] and '_B_' in column_names[1]) or
('.A_' in column_names[0] and '.B_' in column_names[1])):
data = [
flex.real(array.data()).as_string(),
flex.imag(array.data()).as_string()
]
else:
data = [
flex.abs(array.data()).as_string(),
array.phases(deg=True).data().as_string()
]
elif array.is_hendrickson_lattman_array():
if column_names is None:
column_names = [
self.prefix + 'HL_%s_iso' % abcd for abcd in 'ABCD'
]
else:
assert len(column_names) == 4
data = [d.as_string() for d in array.data().as_abcd()]
else:
if array_type is not None:
if array.is_xray_intensity_array():
obs_ext = 'squared_'
else:
obs_ext = ''
column_names = [self.prefix + 'F_' + obs_ext + array_type]
if array.sigmas() is not None:
column_names.append(self.prefix + 'F_' + obs_ext + 'sigma')
if isinstance(array.data(), flex.std_string):
data = [array.data()]
else:
data = [array.data().as_string()]
if array.anomalous_flag():
if ((array.sigmas() is not None and len(column_names) == 4) or
(array.sigmas() is None and len(column_names) == 2)):
data = []
asu, matches = array.match_bijvoet_mates()
for anomalous_sign in ("+", "-"):
sel = matches.pairs_hemisphere_selection(
anomalous_sign)
sel.extend(
matches.singles_hemisphere_selection(
anomalous_sign))
if (anomalous_sign == "+"):
indices = asu.indices().select(sel)
hemisphere_column_names = column_names[:len(
column_names) // 2]
else:
indices = -asu.indices().select(sel)
hemisphere_column_names = column_names[
len(column_names) // 2:]
hemisphere_data = asu.data().select(sel)
hemisphere_array = miller.array(
miller.set(array.crystal_symmetry(), indices),
hemisphere_data)
if array.sigmas() is not None:
hemisphere_array.set_sigmas(
asu.sigmas().select(sel))
if self.refln_loop is None:
# then this is the first array to be added to the loop,
# hack so we don't have both hemispheres of indices
self.indices = indices
self.add_miller_array(
hemisphere_array,
column_names=hemisphere_column_names)
return
if array.sigmas() is not None and len(column_names) == 2:
data.append(array.sigmas().as_string())
if not (self.indices.size() == array.indices().size()
and self.indices.all_eq(array.indices())):
from cctbx.miller import match_indices
other_indices = array.indices().deep_copy()
match = match_indices(self.indices, other_indices)
if match.singles(0).size():
# array is missing some reflections indices that already appear in the loop
# therefore pad the data with '?' values
other_indices.extend(
self.indices.select(match.single_selection(0)))
for d in data:
d.extend(
flex.std_string(['?'] *
(other_indices.size() - d.size())))
for d in data:
assert d.size() == other_indices.size()
match = match_indices(self.indices, other_indices)
if match.singles(1).size():
# this array contains some reflections that are not already present in the
# cif loop, therefore need to add rows of '?' values
single_indices = other_indices.select(
match.single_selection(1))
self.indices.extend(single_indices)
n_data_columns = len(self.refln_loop) - 3
for hkl in single_indices:
row = list(hkl) + ['?'] * n_data_columns
self.refln_loop.add_row(row)
match = match_indices(self.indices, other_indices)
match = match_indices(self.indices, other_indices)
perm = match.permutation()
data = [d.select(perm) for d in data]
if self.refln_loop is None:
self.refln_loop = miller_indices_as_cif_loop(self.indices,
prefix=self.prefix)
columns = OrderedDict(zip(column_names, data))
for key in columns:
assert key not in self.refln_loop
self.refln_loop.add_columns(columns)
def populate_observations(self): intensities = self.reflections['intensity.' + self.method + '.value'] variances = self.reflections['intensity.' + self.method + '.variance'] space_group = crystal.symmetry(self.xtal.get_unit_cell(), str(self.xtal.get_space_group().info())) miller_set = miller.set(space_group, self.reflections['miller_index']) self.frame['observations'][0] = cctbx.miller.array(miller_set, intensities, flex.sqrt(variances)).set_observation_type_xray_intensity()
def make_cif_block(self, experiments, reflections):
"""Write the data to a cif block"""
# Select reflections
selection = reflections.get_flags(reflections.flags.integrated,
all=True)
reflections = reflections.select(selection)
# Filter out bad variances and other issues, but don't filter on ice rings
# or alter partialities.
# Assumes you want to apply the lp and dqe corrections to sum and prf
# Do we want to combine partials?
reflections = filter_reflection_table(
reflections,
self.params.intensity,
combine_partials=False,
partiality_threshold=0.0,
d_min=self.params.mtz.d_min,
)
# Get the cif block
cif_block = iotbx.cif.model.block()
# Audit trail
dials_version = dials.util.version.dials_version()
cif_block["_audit.revision_id"] = 1
cif_block["_audit.creation_method"] = dials_version
cif_block["_audit.creation_date"] = datetime.date.today().isoformat()
cif_block["_entry.id"] = "DIALS"
# add software loop
mmcif_software_header = (
"_software.pdbx_ordinal",
"_software.citation_id",
"_software.name", # as defined at [1]
"_software.version",
"_software.type",
"_software.classification",
"_software.description",
)
mmcif_citations_header = (
"_citation.id",
"_citation.journal_abbrev",
"_citation.journal_volume",
"_citation.journal_issue",
"_citation.page_first",
"_citation.page_last",
"_citation.year",
"_citation.title",
)
software_loop = iotbx.cif.model.loop(header=mmcif_software_header)
citations_loop = iotbx.cif.model.loop(header=mmcif_citations_header)
software_loop.add_row((
1,
1,
"DIALS",
dials_version,
"package",
"data processing",
"Data processing and integration within the DIALS software package",
))
citations_loop.add_row((
1,
"Acta Cryst. D",
74,
2,
85,
97,
2018,
"DIALS: implementation and evaluation of a new integration package",
))
if "scale" in self.params.intensity:
software_loop.add_row((
2,
2,
"DIALS",
dials_version,
"program",
"data scaling",
"Data scaling and merging within the DIALS software package",
))
citations_loop.add_row((
2,
"Acta Cryst. D",
76,
4,
385,
399,
2020,
"Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling",
))
cif_block.add_loop(software_loop)
cif_block.add_loop(citations_loop)
# Hard coding X-ray
if self.params.mmcif.pdb_version == "v5_next":
cif_block["_pdbx_diffrn_data_section.id"] = "dials"
cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray"
cif_block["_pdbx_diffrn_data_section.type_merged"] = "false"
cif_block["_pdbx_diffrn_data_section.type_scaled"] = str(
"scale" in self.params.intensity).lower()
# FIXME finish metadata addition - detector and source details needed
# http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html
# Add source information;
# _diffrn_source.pdbx_wavelength_list = (list of wavelengths)
# _diffrn_source.source = (general class of source e.g. synchrotron)
# _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04)
wls = []
epochs = []
for exp in experiments:
wls.append(round(exp.beam.get_wavelength(), 5))
epochs.append(exp.scan.get_epochs()[0])
unique_wls = set(wls)
cif_block["_exptl_crystal.id"] = 1 # links to crystal_id
cif_block["_diffrn.id"] = 1 # links to diffrn_id
cif_block["_diffrn.crystal_id"] = 1
cif_block["_diffrn_source.diffrn_id"] = 1
cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join(
str(w) for w in unique_wls)
# Add detector information;
# _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc)
# _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd)
# _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M)
# One date is required, so if multiple just use the first date.
min_epoch = min(epochs)
date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch))
cif_block["_diffrn_detector.diffrn_id"] = 1
cif_block["_diffrn_detector.pdbx_collection_date"] = date_str
# Write reflection data
# Required columns
header = (
"_pdbx_diffrn_unmerged_refln.reflection_id",
"_pdbx_diffrn_unmerged_refln.scan_id",
"_pdbx_diffrn_unmerged_refln.image_id_begin",
"_pdbx_diffrn_unmerged_refln.image_id_end",
"_pdbx_diffrn_unmerged_refln.index_h",
"_pdbx_diffrn_unmerged_refln.index_k",
"_pdbx_diffrn_unmerged_refln.index_l",
)
extra_items = {
"scales": ("_pdbx_diffrn_unmerged_refln.scale_value", "%5.3f"),
"intensity.scale.value": (
"_pdbx_diffrn_unmerged_refln.intensity_meas",
"%8.3f",
),
"intensity.scale.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_sigma",
"%8.3f",
),
"intensity.sum.value": (
"_pdbx_diffrn_unmerged_refln.intensity_sum",
"%8.3f",
),
"intensity.sum.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_sum_sigma",
"%8.3f",
),
"intensity.prf.value": (
"_pdbx_diffrn_unmerged_refln.intensity_prf",
"%8.3f",
),
"intensity.prf.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_prf_sigma",
"%8.3f",
),
"angle":
("_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "%7.4f"),
"partiality": ("_pdbx_diffrn_unmerged_refln.partiality", "%7.4f"),
}
variables_present = []
if "scale" in self.params.intensity:
reflections["scales"] = 1.0 / reflections["inverse_scale_factor"]
reflections["intensity.scale.sigma"] = flex.sqrt(
reflections["intensity.scale.variance"])
variables_present.extend(
["scales", "intensity.scale.value", "intensity.scale.sigma"])
if "sum" in self.params.intensity:
reflections["intensity.sum.sigma"] = flex.sqrt(
reflections["intensity.sum.variance"])
variables_present.extend(
["intensity.sum.value", "intensity.sum.sigma"])
if "profile" in self.params.intensity:
reflections["intensity.prf.sigma"] = flex.sqrt(
reflections["intensity.prf.variance"])
variables_present.extend(
["intensity.prf.value", "intensity.prf.sigma"])
# Should always exist
reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG
variables_present.extend(["angle"])
if "partiality" in reflections:
variables_present.extend(["partiality"])
for name in variables_present:
if name in reflections:
header += (extra_items[name][0], )
self._fmt += " " + extra_items[name][1]
if "scale" in self.params.intensity:
# Write dataset_statistics - first make a miller array
crystal_symmetry = cctbxcrystal.symmetry(
space_group=experiments[0].crystal.get_space_group(),
unit_cell=experiments[0].crystal.get_unit_cell(),
)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=reflections["miller_index"],
anomalous_flag=False,
)
i_obs = miller.array(miller_set,
data=reflections["intensity.scale.value"])
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas(reflections["intensity.scale.sigma"])
i_obs.set_info(
miller.array_info(source="DIALS",
source_type="reflection_tables"))
result = dataset_statistics(
i_obs=i_obs,
crystal_symmetry=crystal_symmetry,
use_internal_variance=False,
eliminate_sys_absent=False,
assert_is_not_unique_set_under_symmetry=False,
)
merged_block = iotbx.cif.model.block()
merged_block["_reflns.pdbx_ordinal"] = 1
merged_block["_reflns.pdbx_diffrn_id"] = 1
merged_block["_reflns.entry_id"] = "DIALS"
merged_data = result.as_cif_block()
merged_block.update(merged_data)
cif_block.update(merged_block)
# Write the crystal information
# if v5, that's all so return
if self.params.mmcif.pdb_version == "v5":
return cif_block
# continue if v5_next
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_unmerged_cell.ordinal",
"_pdbx_diffrn_unmerged_cell.crystal_id",
"_pdbx_diffrn_unmerged_cell.wavelength",
"_pdbx_diffrn_unmerged_cell.cell_length_a",
"_pdbx_diffrn_unmerged_cell.cell_length_b",
"_pdbx_diffrn_unmerged_cell.cell_length_c",
"_pdbx_diffrn_unmerged_cell.cell_angle_alpha",
"_pdbx_diffrn_unmerged_cell.cell_angle_beta",
"_pdbx_diffrn_unmerged_cell.cell_angle_gamma",
"_pdbx_diffrn_unmerged_cell.Bravais_lattice",
))
crystals = experiments.crystals()
crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)}
for i, exp in enumerate(experiments):
crystal = exp.crystal
crystal_id = crystal_to_id[crystal]
wavelength = exp.beam.get_wavelength()
a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters()
latt_type = str(
bravais_types.bravais_lattice(group=crystal.get_space_group()))
cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha,
beta, gamma, latt_type))
cif_block.add_loop(cif_loop)
# Write the scan information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_scan.scan_id",
"_pdbx_diffrn_scan.crystal_id",
"_pdbx_diffrn_scan.image_id_begin",
"_pdbx_diffrn_scan.image_id_end",
"_pdbx_diffrn_scan.scan_angle_begin",
"_pdbx_diffrn_scan.scan_angle_end",
))
expid_to_scan_id = {
exp.identifier: i + 1
for i, exp in enumerate(experiments)
}
for i, exp in enumerate(experiments):
scan = exp.scan
crystal_id = crystal_to_id[exp.crystal]
image_range = scan.get_image_range()
osc_range = scan.get_oscillation_range(deg=True)
cif_loop.add_row((
i + 1,
crystal_id,
image_range[0],
image_range[1],
osc_range[0],
osc_range[1],
))
cif_block.add_loop(cif_loop)
_, _, _, _, z0, z1 = reflections["bbox"].parts()
h, k, l = [
hkl.iround()
for hkl in reflections["miller_index"].as_vec3_double().parts()
]
# make scan id consistent with header as defined above
scan_id = flex.int(reflections.size(), 0)
for id_ in reflections.experiment_identifiers().keys():
expid = reflections.experiment_identifiers()[id_]
sel = reflections["id"] == id_
scan_id.set_selected(sel, expid_to_scan_id[expid])
loop_values = [
flex.size_t_range(1,
len(reflections) + 1),
scan_id,
z0,
z1,
h,
k,
l,
] + [reflections[name] for name in variables_present]
cif_loop = iotbx.cif.model.loop(data=dict(zip(header, loop_values)))
cif_block.add_loop(cif_loop)
return cif_block
def exercise_miller_array_as_cns_hkl():
s = StringIO()
crystal_symmetry = crystal.symmetry()
for anomalous_flag in [False, True]:
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=flex.miller_index([(1,2,3),(-3,5,-7)]),
anomalous_flag=anomalous_flag)
for data in [flex.bool((False,True)),
flex.int((-3,4)),
flex.double((10,13)),
flex.complex_double((10,13)),
flex.hendrickson_lattman(((0,1,2,3),(4,5,6,7)))]:
miller_array = miller_set.array(data=data)
miller_array.export_as_cns_hkl(file_object=s)
if (isinstance(data, flex.double)):
miller_array = miller_set.array(data=data, sigmas=data/10.)
miller_array.export_as_cns_hkl(file_object=s)
assert not show_diff(s.getvalue(), """\
NREFlections=2
ANOMalous=FALSe
DECLare NAME=DATA DOMAin=RECIprocal TYPE=INTEger END
INDEx 1 2 3 DATA= 0
INDEx -3 5 -7 DATA= 1
NREFlections=2
ANOMalous=FALSe
DECLare NAME=DATA DOMAin=RECIprocal TYPE=INTEger END
INDEx 1 2 3 DATA= -3
INDEx -3 5 -7 DATA= 4
NREFlections=2
ANOMalous=FALSe
DECLare NAME=DATA DOMAin=RECIprocal TYPE=REAL END
INDEx 1 2 3 DATA= 10
INDEx -3 5 -7 DATA= 13
NREFlections=2
ANOMalous=FALSe
DECLare NAME=FOBS DOMAin=RECIprocal TYPE=REAL END
DECLare NAME=SIGMA DOMAin=RECIprocal TYPE=REAL END
INDEx 1 2 3 FOBS= 10 SIGMA= 1
INDEx -3 5 -7 FOBS= 13 SIGMA= 1.3
NREFlections=2
ANOMalous=FALSe
DECLare NAME=F DOMAin=RECIprocal TYPE=COMPLEX END
INDEx 1 2 3 F= 10 0
INDEx -3 5 -7 F= 13 0
NREFlections=2
ANOMalous=FALSe
DECLare NAME=PA DOMAin=RECIprocal TYPE=REAL END
DECLare NAME=PB DOMAin=RECIprocal TYPE=REAL END
DECLare NAME=PC DOMAin=RECIprocal TYPE=REAL END
DECLare NAME=PD DOMAin=RECIprocal TYPE=REAL END
GROUp TYPE=HL
OBJEct=PA
OBJEct=PB
OBJEct=PC
OBJEct=PD
END
INDEx 1 2 3 PA= 0 PB= 1 PC= 2 PD= 3
INDEx -3 5 -7 PA= 4 PB= 5 PC= 6 PD= 7
NREFlections=2
ANOMalous=TRUE
DECLare NAME=DATA DOMAin=RECIprocal TYPE=INTEger END
INDEx 1 2 3 DATA= 0
INDEx -3 5 -7 DATA= 1
NREFlections=2
ANOMalous=TRUE
DECLare NAME=DATA DOMAin=RECIprocal TYPE=INTEger END
INDEx 1 2 3 DATA= -3
INDEx -3 5 -7 DATA= 4
NREFlections=2
ANOMalous=TRUE
DECLare NAME=DATA DOMAin=RECIprocal TYPE=REAL END
INDEx 1 2 3 DATA= 10
INDEx -3 5 -7 DATA= 13
NREFlections=2
ANOMalous=TRUE
DECLare NAME=FOBS DOMAin=RECIprocal TYPE=REAL END
DECLare NAME=SIGMA DOMAin=RECIprocal TYPE=REAL END
INDEx 1 2 3 FOBS= 10 SIGMA= 1
INDEx -3 5 -7 FOBS= 13 SIGMA= 1.3
NREFlections=2
ANOMalous=TRUE
DECLare NAME=F DOMAin=RECIprocal TYPE=COMPLEX END
INDEx 1 2 3 F= 10 0
INDEx -3 5 -7 F= 13 0
NREFlections=2
ANOMalous=TRUE
DECLare NAME=PA DOMAin=RECIprocal TYPE=REAL END
DECLare NAME=PB DOMAin=RECIprocal TYPE=REAL END
DECLare NAME=PC DOMAin=RECIprocal TYPE=REAL END
DECLare NAME=PD DOMAin=RECIprocal TYPE=REAL END
GROUp TYPE=HL
OBJEct=PA
OBJEct=PB
OBJEct=PC
OBJEct=PD
END
INDEx 1 2 3 PA= 0 PB= 1 PC= 2 PD= 3
INDEx -3 5 -7 PA= 4 PB= 5 PC= 6 PD= 7
""")
def scaled_data_as_miller_array(
reflection_table_list,
experiments,
best_unit_cell=None,
anomalous_flag=False,
wavelength=None,
):
"""Get a scaled miller array from an experiment and reflection table."""
if len(reflection_table_list) > 1:
joint_table = flex.reflection_table()
for reflection_table in reflection_table_list:
# better to just create many miller arrays and join them?
refl_for_joint_table = flex.reflection_table()
for col in [
"miller_index",
"intensity.scale.value",
"inverse_scale_factor",
"intensity.scale.variance",
]:
refl_for_joint_table[col] = reflection_table[col]
good_refl_sel = ~reflection_table.get_flags(
reflection_table.flags.bad_for_scaling, all=False)
refl_for_joint_table = refl_for_joint_table.select(good_refl_sel)
joint_table.extend(refl_for_joint_table)
else:
reflection_table = reflection_table_list[0]
good_refl_sel = ~reflection_table.get_flags(
reflection_table.flags.bad_for_scaling, all=False)
joint_table = reflection_table.select(good_refl_sel)
# Filter out negative scale factors to avoid merging statistics errors.
# These are not removed from the output data, as it is likely one would
# want to do further analysis e.g. delta cc1/2 and rescaling, to exclude
# certain data and get better scale factors for all reflections.
pos_scales = joint_table["inverse_scale_factor"] > 0
if pos_scales.count(False) > 0:
logger.info(
"""There are %s reflections with non-positive scale factors which
will not be used for calculating merging statistics""",
pos_scales.count(False),
)
joint_table = joint_table.select(pos_scales)
if best_unit_cell is None:
best_unit_cell = determine_best_unit_cell(experiments)
miller_set = miller.set(
crystal_symmetry=crystal.symmetry(
unit_cell=best_unit_cell,
space_group=experiments[0].crystal.get_space_group(),
assert_is_compatible_unit_cell=False,
),
indices=joint_table["miller_index"],
anomalous_flag=anomalous_flag,
)
i_obs = miller.array(
miller_set,
data=joint_table["intensity.scale.value"] /
joint_table["inverse_scale_factor"],
)
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas(
flex.sqrt(joint_table["intensity.scale.variance"]) /
joint_table["inverse_scale_factor"])
if not wavelength:
wavelength = np.mean(
[expt.beam.get_wavelength() for expt in experiments])
i_obs.set_info(
miller.array_info(
source="DIALS",
source_type="reflection_tables",
wavelength=wavelength,
))
return i_obs
def write(self, experiments, reflections):
"""
Write the experiments and reflections to file
"""
# if mmcif filename is auto, then choose scaled.cif or integrated.cif
if self.params.mmcif.hklout in (None, Auto, "auto"):
if ("intensity.scale.value"
in reflections) and ("intensity.scale.variance"
in reflections):
filename = "scaled.cif"
logger.info(
"Data appears to be scaled, setting mmcif.hklout = 'scaled.cif'"
)
else:
filename = "integrated.cif"
logger.info(
"Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'"
)
else:
filename = self.params.mmcif.hklout
# Select reflections
selection = reflections.get_flags(reflections.flags.integrated,
all=True)
reflections = reflections.select(selection)
# Filter out bad variances and other issues, but don't filter on ice rings
# or alter partialities.
# Assumes you want to apply the lp and dqe corrections to sum and prf
# Do we want to combine partials?
reflections = filter_reflection_table(
reflections,
self.params.intensity,
combine_partials=False,
partiality_threshold=0.0,
d_min=self.params.mtz.d_min,
)
# Get the cif block
cif_block = iotbx.cif.model.block()
# Audit trail
dials_version = dials.util.version.dials_version()
cif_block["_audit.creation_method"] = dials_version
cif_block["_audit.creation_date"] = datetime.date.today().isoformat()
cif_block["_computing.data_reduction"] = (
"%s (Winter, G. et al., 2018)" % dials_version)
cif_block[
"_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97."
if "scale" in self.params.intensity:
cif_block[
"_publ.section_references"] += "\nBeilsten-Edmands, J. et al. (2020) Acta Cryst. D76, 385-399."
# Hard coding X-ray
cif_block["_pdbx_diffrn_data_section.id"] = "dials"
cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray"
cif_block["_pdbx_diffrn_data_section.type_merged"] = "false"
cif_block["_pdbx_diffrn_data_section.type_scaled"] = str(
"scale" in self.params.intensity).lower()
# FIXME finish metadata addition - detector and source details needed
# http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html
# Add source information;
# _diffrn_source.pdbx_wavelength_list = (list of wavelengths)
# _diffrn_source.source = (general class of source e.g. synchrotron)
# _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04)
wls = []
epochs = []
for exp in experiments:
wls.append(round(exp.beam.get_wavelength(), 5))
epochs.append(exp.scan.get_epochs()[0])
unique_wls = set(wls)
cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join(
str(w) for w in unique_wls)
# Add detector information;
# _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc)
# _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd)
# _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M)
# One date is required, so if multiple just use the first date.
min_epoch = min(epochs)
date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch))
cif_block["_diffrn_detector.pdbx_collection_date"] = date_str
# Write the crystal information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_unmerged_cell.ordinal",
"_pdbx_diffrn_unmerged_cell.crystal_id",
"_pdbx_diffrn_unmerged_cell.wavelength",
"_pdbx_diffrn_unmerged_cell.cell_length_a",
"_pdbx_diffrn_unmerged_cell.cell_length_b",
"_pdbx_diffrn_unmerged_cell.cell_length_c",
"_pdbx_diffrn_unmerged_cell.cell_angle_alpha",
"_pdbx_diffrn_unmerged_cell.cell_angle_beta",
"_pdbx_diffrn_unmerged_cell.cell_angle_gamma",
"_pdbx_diffrn_unmerged_cell.Bravais_lattice",
))
crystals = experiments.crystals()
crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)}
for i, exp in enumerate(experiments):
crystal = exp.crystal
crystal_id = crystal_to_id[crystal]
wavelength = exp.beam.get_wavelength()
a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters()
latt_type = str(
bravais_types.bravais_lattice(group=crystal.get_space_group()))
cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha,
beta, gamma, latt_type))
cif_block.add_loop(cif_loop)
# Write the scan information
cif_loop = iotbx.cif.model.loop(header=(
"_pdbx_diffrn_scan.scan_id",
"_pdbx_diffrn_scan.crystal_id",
"_pdbx_diffrn_scan.image_id_begin",
"_pdbx_diffrn_scan.image_id_end",
"_pdbx_diffrn_scan.scan_angle_begin",
"_pdbx_diffrn_scan.scan_angle_end",
))
for i, exp in enumerate(experiments):
scan = exp.scan
crystal_id = crystal_to_id[exp.crystal]
image_range = scan.get_image_range()
osc_range = scan.get_oscillation_range(deg=True)
cif_loop.add_row((
i + 1,
crystal_id,
image_range[0],
image_range[1],
osc_range[0],
osc_range[1],
))
cif_block.add_loop(cif_loop)
# Make a dict of unit_cell parameters
unit_cell_parameters = {}
if crystal.num_scan_points > 1:
for i in range(crystal.num_scan_points):
a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point(
i).parameters()
unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma)
else:
unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma)
# Write reflection data
# Required columns
header = (
"_pdbx_diffrn_unmerged_refln.reflection_id",
"_pdbx_diffrn_unmerged_refln.scan_id",
"_pdbx_diffrn_unmerged_refln.image_id_begin",
"_pdbx_diffrn_unmerged_refln.image_id_end",
"_pdbx_diffrn_unmerged_refln.index_h",
"_pdbx_diffrn_unmerged_refln.index_k",
"_pdbx_diffrn_unmerged_refln.index_l",
)
extra_items = {
"scales": ("_pdbx_diffrn_unmerged_refln.scale_value", "%5.3f"),
"intensity.scale.value": (
"_pdbx_diffrn_unmerged_refln.intensity_meas",
"%8.3f",
),
"intensity.scale.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_sigma",
"%8.3f",
),
"intensity.sum.value": (
"_pdbx_diffrn_unmerged_refln.intensity_sum",
"%8.3f",
),
"intensity.sum.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_sum_sigma",
"%8.3f",
),
"intensity.prf.value": (
"_pdbx_diffrn_unmerged_refln.intensity_prf",
"%8.3f",
),
"intensity.prf.sigma": (
"_pdbx_diffrn_unmerged_refln.intensity_prf_sigma",
"%8.3f",
),
"angle":
("_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "%7.4f"),
"partiality": ("_pdbx_diffrn_unmerged_refln.partiality", "%7.4f"),
}
fmt = "%6i %2i %5i %5i %-2i %-2i %-2i"
variables_present = []
if "scale" in self.params.intensity:
reflections["scales"] = 1.0 / reflections["inverse_scale_factor"]
reflections["intensity.scale.sigma"] = flex.sqrt(
reflections["intensity.scale.variance"])
variables_present.extend(
["scales", "intensity.scale.value", "intensity.scale.sigma"])
if "sum" in self.params.intensity:
reflections["intensity.sum.sigma"] = flex.sqrt(
reflections["intensity.sum.variance"])
variables_present.extend(
["intensity.sum.value", "intensity.sum.sigma"])
if "profile" in self.params.intensity:
reflections["intensity.prf.sigma"] = flex.sqrt(
reflections["intensity.prf.variance"])
variables_present.extend(
["intensity.prf.value", "intensity.prf.sigma"])
# Should always exist
reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG
variables_present.extend(["angle"])
if "partiality" in reflections:
variables_present.extend(["partiality"])
for name in variables_present:
if name in reflections:
header += (extra_items[name][0], )
fmt += " " + extra_items[name][1]
loop_format_strings = {"_pdbx_diffrn_unmerged_refln": fmt}
if "scale" in self.params.intensity:
# Write dataset_statistics - first make a miller array
crystal_symmetry = cctbxcrystal.symmetry(
space_group=experiments[0].crystal.get_space_group(),
unit_cell=experiments[0].crystal.get_unit_cell(),
)
miller_set = miller.set(
crystal_symmetry=crystal_symmetry,
indices=reflections["miller_index"],
anomalous_flag=False,
)
i_obs = miller.array(miller_set,
data=reflections["intensity.scale.value"])
i_obs.set_observation_type_xray_intensity()
i_obs.set_sigmas(reflections["intensity.scale.sigma"])
i_obs.set_info(
miller.array_info(source="DIALS",
source_type="reflection_tables"))
result = dataset_statistics(
i_obs=i_obs,
crystal_symmetry=crystal_symmetry,
use_internal_variance=False,
eliminate_sys_absent=False,
)
cif_block.update(result.as_cif_block())
_, _, _, _, z0, z1 = reflections["bbox"].parts()
h, k, l = [
hkl.iround()
for hkl in reflections["miller_index"].as_vec3_double().parts()
]
loop_values = [
flex.size_t_range(1,
len(reflections) + 1),
reflections["id"] + 1,
z0,
z1,
h,
k,
l,
] + [reflections[name] for name in variables_present]
cif_loop = iotbx.cif.model.loop(data=dict(zip(header, loop_values)))
cif_block.add_loop(cif_loop)
# Add the block
self._cif["dials"] = cif_block
# Print to file
if self.params.mmcif.compress and not filename.endswith(
"." + self.params.mmcif.compress):
filename += "." + self.params.mmcif.compress
if self.params.mmcif.compress == "gz":
open_fn = gzip.open
elif self.params.mmcif.compress == "bz2":
open_fn = bz2.open
elif self.params.mmcif.compress == "xz":
open_fn = lzma.open
else:
open_fn = open
with open_fn(filename, "wt") as fh:
self._cif.show(out=fh, loop_format_strings=loop_format_strings)
# Log
logger.info("Wrote reflections to %s" % filename)
def rho_stats(
xray_structure,
d_min,
resolution_factor,
electron_sum_radius,
zero_out_f000):
n_real = []
n_half_plus = []
n_half_minus = []
s2 = d_min * resolution_factor * 2
for l in xray_structure.unit_cell().parameters()[:3]:
nh = ifloor(l / s2)
n_real.append(2*nh+1)
n_half_plus.append(nh)
n_half_minus.append(-nh)
n_real = tuple(n_real)
n_real_product = matrix.col(n_real).product()
crystal_gridding = maptbx.crystal_gridding(
unit_cell=xray_structure.unit_cell(),
space_group_info=xray_structure.space_group_info(),
pre_determined_n_real=n_real)
miller_indices = flex.miller_index()
miller_indices.reserve(n_real_product)
for h in flex.nested_loop(n_half_minus, n_half_plus, open_range=False):
miller_indices.append(h)
assert miller_indices.size() == n_real_product
#
miller_set = miller.set(
crystal_symmetry=xray_structure,
anomalous_flag=True,
indices=miller_indices).sort(by_value="resolution")
assert miller_set.indices()[0] == (0,0,0)
f_calc = miller_set.structure_factors_from_scatterers(
xray_structure=xray_structure,
algorithm="direct",
cos_sin_table=False).f_calc()
if (zero_out_f000):
f_calc.data()[0] = 0j
#
unit_cell_volume = xray_structure.unit_cell().volume()
voxel_volume = unit_cell_volume / n_real_product
number_of_miller_indices = []
rho_max = []
electron_sums_around_atoms = []
densities_along_x = []
for f in [f_calc, f_calc.resolution_filter(d_min=d_min)]:
assert f.indices()[0] == (0,0,0)
number_of_miller_indices.append(f.indices().size())
fft_map = miller.fft_map(
crystal_gridding=crystal_gridding,
fourier_coefficients=f)
assert fft_map.n_real() == n_real
rho = fft_map.real_map_unpadded() / unit_cell_volume
assert approx_equal(voxel_volume*flex.sum(rho), f_calc.data()[0])
if (xray_structure.scatterers().size() == 1):
assert flex.max_index(rho) == 0
rho_max.append(rho[0])
else:
rho_max.append(flex.max(rho))
site_cart = xray_structure.sites_cart()[0]
gias = maptbx.grid_indices_around_sites(
unit_cell=xray_structure.unit_cell(),
fft_n_real=n_real,
fft_m_real=n_real,
sites_cart=flex.vec3_double([site_cart]),
site_radii=flex.double([electron_sum_radius]))
electron_sums_around_atoms.append(
flex.sum(rho.as_1d().select(gias))*voxel_volume)
#
a = xray_structure.unit_cell().parameters()[0]
nx = n_real[0]
nxh = nx//2
x = []
y = []
for ix in xrange(-nxh,nxh+1):
x.append(a*ix/nx)
y.append(rho[(ix%nx,0,0)])
densities_along_x.append((x,y))
#
print \
"%3.1f %4.2f %-12s %5d %5d | %6.3f %6.3f | %6.3f %6.3f | %4.2f %5.1f" % (
d_min,
resolution_factor,
n_real,
number_of_miller_indices[0],
number_of_miller_indices[1],
electron_sums_around_atoms[0],
electron_sums_around_atoms[1],
rho_max[0],
rho_max[1],
f_calc.data()[0].real,
u_as_b(xray_structure.scatterers()[0].u_iso))
#
return densities_along_x
