def direct_space_squaring(start, selection_fixed):
map_gridding = miller.index_span(
miller.set.expand_to_p1(start).indices()).map_grid()
if (selection_fixed is None):
fixed = start
var = start
else:
fixed = start.select(selection_fixed)
var = start.select(~selection_fixed)
rfft = fftpack.real_to_complex_3d([n * 3 // 2 for n in map_gridding])
conjugate_flag = True
structure_factor_map = maptbx.structure_factors.to_map(
space_group=fixed.space_group(),
anomalous_flag=fixed.anomalous_flag(),
miller_indices=fixed.indices(),
structure_factors=fixed.data(),
n_real=rfft.n_real(),
map_grid=flex.grid(rfft.n_complex()),
conjugate_flag=conjugate_flag)
real_map = rfft.backward(structure_factor_map.complex_map())
squared_map = flex.pow2(real_map)
squared_sf_map = rfft.forward(squared_map)
allow_miller_indices_outside_map = False
from_map = maptbx.structure_factors.from_map(
anomalous_flag=var.anomalous_flag(),
miller_indices=var.indices(),
complex_map=squared_sf_map,
conjugate_flag=conjugate_flag,
allow_miller_indices_outside_map=allow_miller_indices_outside_map)
if (selection_fixed is None):
return from_map.data()
result = start.data().deep_copy()
result.set_selected(~selection_fixed, from_map.data())
assert result.select(selection_fixed).all_eq(fixed.data())
return result
def generate_cif(crystal, refiner, filename):
logger.info("Saving CIF information to %s", filename)
from cctbx import miller
import iotbx.cif.model
block = iotbx.cif.model.block()
block["_audit_creation_method"] = dials_version()
block["_audit_creation_date"] = datetime.date.today().isoformat()
# block["_publ_section_references"] = '' # once there is a reference...
for cell, esd, cifname in zip(
crystal.get_unit_cell().parameters(),
crystal.get_cell_parameter_sd(),
[
"length_a",
"length_b",
"length_c",
"angle_alpha",
"angle_beta",
"angle_gamma",
],
):
block["_cell_%s" % cifname] = format_float_with_standard_uncertainty(
cell, esd
)
block["_cell_volume"] = format_float_with_standard_uncertainty(
crystal.get_unit_cell().volume(), crystal.get_cell_volume_sd()
)
used_reflections = refiner.get_matches()
block["_cell_measurement_reflns_used"] = len(used_reflections)
block["_cell_measurement_theta_min"] = (
flex.min(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2
)
block["_cell_measurement_theta_max"] = (
flex.max(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2
)
block["_diffrn_reflns_number"] = len(used_reflections)
miller_span = miller.index_span(used_reflections["miller_index"])
min_h, min_k, min_l = miller_span.min()
max_h, max_k, max_l = miller_span.max()
block["_diffrn_reflns_limit_h_min"] = min_h
block["_diffrn_reflns_limit_h_max"] = max_h
block["_diffrn_reflns_limit_k_min"] = min_k
block["_diffrn_reflns_limit_k_max"] = max_k
block["_diffrn_reflns_limit_l_min"] = min_l
block["_diffrn_reflns_limit_l_max"] = max_l
block["_diffrn_reflns_theta_min"] = (
flex.min(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2
)
block["_diffrn_reflns_theta_max"] = (
flex.max(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2
)
cif = iotbx.cif.model.cif()
cif["two_theta_refine"] = block
with open(filename, "w") as fh:
cif.show(out=fh)
def generate_mmcif(crystal, refiner, filename):
logger.info("Saving mmCIF information to %s", filename)
block = iotbx.cif.model.block()
block["_audit.revision_id"] = 1
block["_audit.creation_method"] = dials_version()
block["_audit.creation_date"] = datetime.date.today().isoformat()
block["_entry.id"] = "two_theta_refine"
# block["_publ.section_references"] = '' # once there is a reference...
block["_cell.entry_id"] = "two_theta_refine"
for cell, esd, cifname in zip(
crystal.get_unit_cell().parameters(),
crystal.get_cell_parameter_sd(),
[
"length_a",
"length_b",
"length_c",
"angle_alpha",
"angle_beta",
"angle_gamma",
],
):
block["_cell.%s" % cifname] = "%.8f" % cell
block["_cell.%s_esd" % cifname] = "%.8f" % esd
block["_cell.volume"] = "%f" % crystal.get_unit_cell().volume()
block["_cell.volume_esd"] = "%f" % crystal.get_cell_volume_sd()
used_reflections = refiner.get_matches()
block["_cell_measurement.entry_id"] = "two_theta_refine"
block["_cell_measurement.reflns_used"] = len(used_reflections)
block["_cell_measurement.theta_min"] = (
flex.min(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_cell_measurement.theta_max"] = (
flex.max(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_exptl_crystal.id"] = 1
block["_diffrn.id"] = "two_theta_refine"
block["_diffrn.crystal_id"] = 1
block["_diffrn_reflns.diffrn_id"] = "two_theta_refine"
block["_diffrn_reflns.number"] = len(used_reflections)
miller_span = miller.index_span(used_reflections["miller_index"])
min_h, min_k, min_l = miller_span.min()
max_h, max_k, max_l = miller_span.max()
block["_diffrn_reflns.limit_h_min"] = min_h
block["_diffrn_reflns.limit_h_max"] = max_h
block["_diffrn_reflns.limit_k_min"] = min_k
block["_diffrn_reflns.limit_k_max"] = max_k
block["_diffrn_reflns.limit_l_min"] = min_l
block["_diffrn_reflns.limit_l_max"] = max_l
block["_diffrn_reflns.theta_min"] = (
flex.min(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
block["_diffrn_reflns.theta_max"] = (
flex.max(used_reflections["2theta_obs.rad"]) * 180 / math.pi / 2)
cif = iotbx.cif.model.cif()
cif["two_theta_refine"] = block
with open(filename, "w") as fh:
cif.show(out=fh)
def exercise_index_span(): miller_indices = flex.miller_index(((1,-2,3), (-3,5,0))) s = miller.index_span(miller_indices) assert s.min() == (-3,-2,0) assert s.max() == (1,5,3) assert s.abs_range() == (4,6,4) assert s.map_grid() == (7,11,7) assert s.is_in_domain((-1,2,1)) assert not s.is_in_domain((0,6,0)) assert tuple(s.pack(miller_indices)) == (131, 28)
def exercise_index_span(): miller_indices = flex.miller_index(((1,-2,3), (-3,5,0))) s = miller.index_span(miller_indices) assert s.min() == (-3,-2,0) assert s.max() == (1,5,3) assert s.abs_range() == (4,6,4) assert s.map_grid() == (7,11,7) assert s.is_in_domain((-1,2,1)) assert not s.is_in_domain((0,6,0)) assert tuple(s.pack(miller_indices)) == (131, 28)
def generate_mmcif(crystal, refiner, file):
logger.info('Saving mmCIF information to %s' % file)
from cctbx import miller
import datetime
import iotbx.cif.model
import math
block = iotbx.cif.model.block()
block["_audit.creation_method"] = dials_version()
block["_audit.creation_date"] = datetime.date.today().isoformat()
# block["_publ.section_references"] = '' # once there is a reference...
for cell, esd, cifname in zip(
crystal.get_unit_cell().parameters(),
crystal.get_cell_parameter_sd(), [
'length_a', 'length_b', 'length_c', 'angle_alpha',
'angle_beta', 'angle_gamma'
]):
block['_cell.%s' % cifname] = "%.8f" % cell
block['_cell.%s_esd' % cifname] = "%.8f" % esd
block['_cell.volume'] = "%f" % crystal.get_unit_cell().volume()
block['_cell.volume_esd'] = "%f" % crystal.get_cell_volume_sd()
used_reflections = refiner.get_matches()
block['_cell_measurement.reflns_used'] = len(used_reflections)
block['_cell_measurement.theta_min'] = flex.min(
used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_cell_measurement.theta_max'] = flex.max(
used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_diffrn_reflns.number'] = len(used_reflections)
miller_span = miller.index_span(used_reflections['miller_index'])
min_h, min_k, min_l = miller_span.min()
max_h, max_k, max_l = miller_span.max()
block['_diffrn_reflns.limit_h_min'] = min_h
block['_diffrn_reflns.limit_h_max'] = max_h
block['_diffrn_reflns.limit_k_min'] = min_k
block['_diffrn_reflns.limit_k_max'] = max_k
block['_diffrn_reflns.limit_l_min'] = min_l
block['_diffrn_reflns.limit_l_max'] = max_l
block['_diffrn_reflns.theta_min'] = flex.min(
used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_diffrn_reflns.theta_max'] = flex.max(
used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
cif = iotbx.cif.model.cif()
cif['two_theta_refine'] = block
with open(file, 'w') as fh:
cif.show(out=fh)
def generate_mmcif(crystal, refiner, file):
logger.info('Saving mmCIF information to %s' % file)
from cctbx import miller
import datetime
import iotbx.cif.model
import math
block = iotbx.cif.model.block()
block["_audit.creation_method"] = dials_version()
block["_audit.creation_date"] = datetime.date.today().isoformat()
# block["_publ.section_references"] = '' # once there is a reference...
for cell, esd, cifname in zip(crystal.get_unit_cell().parameters(),
crystal.get_cell_parameter_sd(),
['length_a', 'length_b', 'length_c', 'angle_alpha', 'angle_beta', 'angle_gamma']):
block['_cell.%s' % cifname] = "%.8f" % cell
block['_cell.%s_esd' % cifname] = "%.8f" % esd
block['_cell.volume'] = "%f" % crystal.get_unit_cell().volume()
block['_cell.volume_esd'] = "%f" % crystal.get_cell_volume_sd()
used_reflections = refiner.get_matches()
block['_cell_measurement.reflns_used'] = len(used_reflections)
block['_cell_measurement.theta_min'] = flex.min(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_cell_measurement.theta_max'] = flex.max(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_diffrn_reflns.number'] = len(used_reflections)
miller_span = miller.index_span(used_reflections['miller_index'])
min_h, min_k, min_l = miller_span.min()
max_h, max_k, max_l = miller_span.max()
block['_diffrn_reflns.limit_h_min'] = min_h
block['_diffrn_reflns.limit_h_max'] = max_h
block['_diffrn_reflns.limit_k_min'] = min_k
block['_diffrn_reflns.limit_k_max'] = max_k
block['_diffrn_reflns.limit_l_min'] = min_l
block['_diffrn_reflns.limit_l_max'] = max_l
block['_diffrn_reflns.theta_min'] = flex.min(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
block['_diffrn_reflns.theta_max'] = flex.max(used_reflections['2theta_obs.rad']) * 180 / math.pi / 2
cif = iotbx.cif.model.cif()
cif['two_theta_refine'] = block
with open(file, 'w') as fh:
cif.show(out=fh)
def direct_space_squaring(start, selection_fixed):
map_gridding = miller.index_span(miller.set.expand_to_p1(start).indices()).map_grid()
if selection_fixed is None:
fixed = start
var = start
else:
fixed = start.select(selection_fixed)
var = start.select(~selection_fixed)
rfft = fftpack.real_to_complex_3d([n * 3 // 2 for n in map_gridding])
conjugate_flag = True
structure_factor_map = maptbx.structure_factors.to_map(
space_group=fixed.space_group(),
anomalous_flag=fixed.anomalous_flag(),
miller_indices=fixed.indices(),
structure_factors=fixed.data(),
n_real=rfft.n_real(),
map_grid=flex.grid(rfft.n_complex()),
conjugate_flag=conjugate_flag,
)
real_map = rfft.backward(structure_factor_map.complex_map())
squared_map = flex.pow2(real_map)
squared_sf_map = rfft.forward(squared_map)
allow_miller_indices_outside_map = False
from_map = maptbx.structure_factors.from_map(
anomalous_flag=var.anomalous_flag(),
miller_indices=var.indices(),
complex_map=squared_sf_map,
conjugate_flag=conjugate_flag,
allow_miller_indices_outside_map=allow_miller_indices_outside_map,
)
if selection_fixed is None:
return from_map.data()
result = start.data().deep_copy()
result.set_selected(~selection_fixed, from_map.data())
assert result.select(selection_fixed).all_eq(fixed.data())
return result
def run(args):
assert args in [[], ["python"], ["c++"]]
#
sgno_list_by_index_list_by_cs = {}
from cctbx import sgtbx
from cctbx import miller
for symbols in sgtbx.space_group_symbol_iterator():
psgi = sgtbx.space_group_info(symbols.universal_hermann_mauguin()) \
.primitive_setting()
p_indices = miller.index_generator(space_group_type=psgi.type(),
anomalous_flag=False,
max_index=[4] * 3).to_array()
# 4 is the smallest value leading to correct results; any larger
# value will work, too, but will make this procedure slower
p1_indices = miller.expand_to_p1_iselection(
space_group=psgi.group(),
anomalous_flag=False,
indices=p_indices,
build_iselection=False).indices
from cctbx.array_family import flex
sort_perm = flex.sort_permutation(
data=miller.index_span(p1_indices).pack(p1_indices))
p1_indices = p1_indices.select(sort_perm)
index_list = tuple(p1_indices)
sgno = psgi.type().number()
sgno_list_by_index_list = sgno_list_by_index_list_by_cs \
.setdefault(symbols.crystal_system(), {})
sgno_list_by_index_list.setdefault(index_list, []).append(sgno)
from scitbx.graph import tardy_tree
cluster_manager = tardy_tree.cluster_manager(n_vertices=231)
for cs, sgno_list_by_index_list in sgno_list_by_index_list_by_cs.items():
for sgno_list in sgno_list_by_index_list.values():
i = sgno_list[0]
for j in sgno_list[1:]:
cluster_manager.connect_vertices(i=i, j=j, optimize=True)
cluster_manager.tidy()
#
# everything below is just to format the results
#
if (args == []):
for cluster in cluster_manager.clusters:
if (len(cluster) == 1): break
print(cluster)
else:
note = ("""\
Output of: cctbx/examples/find_sys_abs_equiv_space_groups.py %s
If you have to edit this table, please send email to: [email protected]
""" % args[0]).splitlines()
#
if (args == ["python"]):
print("space_group_numbers = [")
for line in note:
print(" #", line)
ci = cluster_manager.cluster_indices
cl = cluster_manager.clusters
for sgno in range(231):
cluster = list(cl[ci[sgno]])
cluster.remove(sgno)
if (len(cluster) == 0): s = "None"
else: s = str(tuple(cluster))
if (sgno == 230): comma = ""
else: comma = ","
print(" %s%s" % (s, comma))
print("]")
else:
print("""\
#ifndef CCTBX_SGTBX_SYS_ABS_EQUIV_H
#define CCTBX_SGTBX_SYS_ABS_EQUIV_H
namespace cctbx { namespace sgtbx { namespace sys_abs_equiv {
""")
data = []
ci = cluster_manager.cluster_indices
cl = cluster_manager.clusters
for line in note:
print(" //", line)
for sgno in range(231):
cluster = list(cl[ci[sgno]])
cluster.remove(sgno)
if (len(cluster) == 0):
data.append("0")
else:
cid = "data_%03d" % sgno
data.append(cid)
print(" static const unsigned %s[] = {%d, %s};" %
(cid, len(cluster), ", ".join(
[str(i) for i in cluster])))
print("")
print(" static const unsigned* space_group_numbers[] = {")
print(" ", ",\n ".join(data))
print("""\
};
}}}
#endif // GUARD""")
def show_split_stats(stream, nindexed, symm, params, anoref=None, ref=None, out_prefix="out", start_at=0):
random.seed(params.random_seed)
nstep = nindexed // params.nsplit
out = open(out_prefix + ".dat", "w")
out_byres = open(out_prefix + "_byres.dat", "w")
print >>out_byres, ("%"+str(len(str(nindexed)))+"s")%"nframes",
print >>out_byres, "dmin dmax red cmpl acmpl rsplit rano cc1/2 ccano snr rano/split CCanoref CCref"
#formatstr_for_eachout = out_prefix+"_to%."+str(len(str(nindexed)))+"d_byres.dat"
#indices1, iobs1, sel1 = [], [], []
#indices2, iobs2, sel2 = [], [], []
indices1, indices2 = flex.miller_index(), flex.miller_index()
iobs1, iobs2 = flex.double(), flex.double()
sel1, sel2 = flex.bool(), flex.bool()
print >>out, " nframes red Rsplit Rano CC1/2 CCano snr Rano/Rsplit CCanoref CCref"
chunks = read_stream(stream, start_at)
for i in xrange(params.nsplit):
e = (i+1)*nstep
if i == params.nsplit-1: e = nindexed
read_all = (i == params.nsplit-1)
slc = []
for j, c in enumerate(chunks):
sys.stderr.write("\rprocessed: %d" % (i*nstep+j))
sys.stderr.flush()
slc.append(c)
if not read_all and j >= nstep-1:
break
if nindexed <= i*nstep+j:
break
if params.halve_method == "alternate":
slc1 = slc[0::2]
slc2 = slc[1::2]
elif params.halve_method == "random":
perm = range(len(slc))
random.shuffle(perm)
nhalf = len(slc)//2
slc1 = [slc[r] for r in perm[:nhalf]]
slc2 = [slc[r] for r in perm[nhalf:]]
else:
raise "Not-supported:", params.halve_method
i1, o1, s1 = [], [], []
for x in slc1:
i1.extend(x.indices)
o1.extend(x.iobs)
if params.adu_cutoff is not None:
s1.extend((a <= params.adu_cutoff for a in x.peak))
i2, o2, s2 = [], [], []
for x in slc2:
i2.extend(x.indices)
o2.extend(x.iobs)
if params.adu_cutoff is not None:
s2.extend((a <= params.adu_cutoff for a in x.peak))
# Concatenate and sort
indices1 = indices1.concatenate(flex.miller_index(i1))
iobs1 = iobs1.concatenate(flex.double(o1))
sel1 = sel1.concatenate(flex.bool(s1))
indices2 = indices2.concatenate(flex.miller_index(i2))
iobs2 = iobs2.concatenate(flex.double(o2))
sel2 = sel2.concatenate(flex.bool(s2))
perm1 = flex.sort_permutation(data=miller.index_span(indices1).pack(indices1), reverse=False)
perm2 = flex.sort_permutation(data=miller.index_span(indices2).pack(indices2), reverse=False)
indices1 = indices1.select(perm1)
indices2 = indices2.select(perm2)
iobs1 = iobs1.select(perm1)
iobs2 = iobs2.select(perm2)
if params.adu_cutoff is not None:
sel1 = sel1.select(perm1)
sel2 = sel2.select(perm2)
# Merge
m = merge_obs(indices1.concatenate(indices2), iobs1.concatenate(iobs2), sel1.concatenate(sel2), symm, anomalous_flag=params.anomalous, d_min=params.dmin, d_max=params.dmax)
m1 = merge_obs(indices1, iobs1, sel1, symm, anomalous_flag=params.anomalous, d_min=params.dmin, d_max=params.dmax)
m2 = merge_obs(indices2, iobs2, sel2, symm, anomalous_flag=params.anomalous, d_min=params.dmin, d_max=params.dmax)
a1, a2 = m1.array().common_sets(m2.array())
red = flex.sum(m.redundancies().data()) / m.array().data().size() if m.array().data().size()>0 else 0
rsplit = split_stats.calc_rsplit(a1, a2)
rano = split_stats.calc_rano(a1, a2) if params.anomalous else float("nan")
cc = split_stats.calc_cc(a1, a2)
ccano = split_stats.calc_ccano(a1, a2) if params.anomalous else float("nan")
ccanoref = split_stats.calc_ccano(m.array(), anoref, take_common=True) if params.anomalous and anoref is not None else float("nan")
ccref = split_stats.calc_cc(m.array(), ref, take_common=True) if ref is not None else float("nan")
snr = flex.mean(m.array().data()/m.array().sigmas())
print >>out, "%10d %7.2f %7.4f %7.4f % .4f % .4f %.2f %.4f % .4f % .4f" % (e, red, rsplit, rano, cc, ccano, snr, rano/rsplit, ccanoref, ccref)
out.flush()
byresolution_stats(params, a1, a2, m, anoref, ref, ("%"+str(len(str(nindexed)))+"d")%e, out_byres)
print >>out_byres, "#"
def run(args):
assert args in [[], ["python"], ["c++"]]
#
sgno_list_by_index_list_by_cs = {}
from cctbx import sgtbx
from cctbx import miller
for symbols in sgtbx.space_group_symbol_iterator():
psgi = sgtbx.space_group_info(symbols.universal_hermann_mauguin()) \
.primitive_setting()
p_indices = miller.index_generator(
space_group_type=psgi.type(),
anomalous_flag=False,
max_index=[4]*3).to_array()
# 4 is the smallest value leading to correct results; any larger
# value will work, too, but will make this procedure slower
p1_indices = miller.expand_to_p1_iselection(
space_group=psgi.group(),
anomalous_flag=False,
indices=p_indices,
build_iselection=False).indices
from cctbx.array_family import flex
sort_perm = flex.sort_permutation(
data=miller.index_span(p1_indices).pack(p1_indices))
p1_indices = p1_indices.select(sort_perm)
index_list = tuple(p1_indices)
sgno = psgi.type().number()
sgno_list_by_index_list = sgno_list_by_index_list_by_cs \
.setdefault(symbols.crystal_system(), {})
sgno_list_by_index_list.setdefault(index_list, []).append(sgno)
from scitbx.graph import tardy_tree
cluster_manager = tardy_tree.cluster_manager(n_vertices=231)
for cs,sgno_list_by_index_list in sgno_list_by_index_list_by_cs.items():
for sgno_list in sgno_list_by_index_list.values():
i = sgno_list[0]
for j in sgno_list[1:]:
cluster_manager.connect_vertices(i=i, j=j, optimize=True)
cluster_manager.tidy()
#
# everything below is just to format the results
#
if (args == []):
for cluster in cluster_manager.clusters:
if (len(cluster) == 1): break
print cluster
else:
note = ("""\
Output of: cctbx/examples/find_sys_abs_equiv_space_groups.py %s
If you have to edit this table, please send email to: [email protected]
""" % args[0]).splitlines()
#
if (args == ["python"]):
print "space_group_numbers = ["
for line in note:
print " #", line
ci = cluster_manager.cluster_indices
cl = cluster_manager.clusters
for sgno in xrange(231):
cluster = list(cl[ci[sgno]])
cluster.remove(sgno)
if (len(cluster) == 0): s = "None"
else: s = str(tuple(cluster))
if (sgno == 230): comma = ""
else: comma = ","
print " %s%s" % (s, comma)
print "]"
else:
print """\
#ifndef CCTBX_SGTBX_SYS_ABS_EQUIV_H
#define CCTBX_SGTBX_SYS_ABS_EQUIV_H
namespace cctbx { namespace sgtbx { namespace sys_abs_equiv {
"""
data = []
ci = cluster_manager.cluster_indices
cl = cluster_manager.clusters
for line in note:
print " //", line
for sgno in xrange(231):
cluster = list(cl[ci[sgno]])
cluster.remove(sgno)
if (len(cluster) == 0):
data.append("0")
else:
cid = "data_%03d" % sgno
data.append(cid)
print " static const unsigned %s[] = {%d, %s};" % (
cid, len(cluster), ", ".join([str(i) for i in cluster]))
print ""
print " static const unsigned* space_group_numbers[] = {"
print " ", ",\n ".join(data)
print """\
def show_split_stats(stream,
nindexed,
symm,
params,
anoref=None,
ref=None,
out_prefix="out",
start_at=0):
random.seed(params.random_seed)
nstep = nindexed // params.nsplit
out = open(out_prefix + ".dat", "w")
out_byres = open(out_prefix + "_byres.dat", "w")
print >> out_byres, ("%" + str(len(str(nindexed))) + "s") % "nframes",
print >> out_byres, "dmin dmax red cmpl acmpl rsplit rano cc1/2 ccano snr rano/split CCanoref CCref"
#formatstr_for_eachout = out_prefix+"_to%."+str(len(str(nindexed)))+"d_byres.dat"
#indices1, iobs1, sel1 = [], [], []
#indices2, iobs2, sel2 = [], [], []
indices1, indices2 = flex.miller_index(), flex.miller_index()
iobs1, iobs2 = flex.double(), flex.double()
sel1, sel2 = flex.bool(), flex.bool()
print >> out, " nframes red Rsplit Rano CC1/2 CCano snr Rano/Rsplit CCanoref CCref"
chunks = read_stream(stream, start_at)
for i in xrange(params.nsplit):
e = (i + 1) * nstep
if i == params.nsplit - 1: e = nindexed
read_all = (i == params.nsplit - 1)
slc = []
for j, c in enumerate(chunks):
sys.stderr.write("\rprocessed: %d" % (i * nstep + j))
sys.stderr.flush()
slc.append(c)
if not read_all and j >= nstep - 1:
break
if nindexed <= i * nstep + j:
break
if params.halve_method == "alternate":
slc1 = slc[0::2]
slc2 = slc[1::2]
elif params.halve_method == "random":
perm = range(len(slc))
random.shuffle(perm)
nhalf = len(slc) // 2
slc1 = [slc[r] for r in perm[:nhalf]]
slc2 = [slc[r] for r in perm[nhalf:]]
else:
raise "Not-supported:", params.halve_method
i1, o1, s1 = [], [], []
for x in slc1:
i1.extend(x.indices)
o1.extend(x.iobs)
if params.adu_cutoff is not None:
s1.extend((a <= params.adu_cutoff for a in x.peak))
i2, o2, s2 = [], [], []
for x in slc2:
i2.extend(x.indices)
o2.extend(x.iobs)
if params.adu_cutoff is not None:
s2.extend((a <= params.adu_cutoff for a in x.peak))
# Concatenate and sort
indices1 = indices1.concatenate(flex.miller_index(i1))
iobs1 = iobs1.concatenate(flex.double(o1))
sel1 = sel1.concatenate(flex.bool(s1))
indices2 = indices2.concatenate(flex.miller_index(i2))
iobs2 = iobs2.concatenate(flex.double(o2))
sel2 = sel2.concatenate(flex.bool(s2))
perm1 = flex.sort_permutation(
data=miller.index_span(indices1).pack(indices1), reverse=False)
perm2 = flex.sort_permutation(
data=miller.index_span(indices2).pack(indices2), reverse=False)
indices1 = indices1.select(perm1)
indices2 = indices2.select(perm2)
iobs1 = iobs1.select(perm1)
iobs2 = iobs2.select(perm2)
if params.adu_cutoff is not None:
sel1 = sel1.select(perm1)
sel2 = sel2.select(perm2)
# Merge
m = merge_obs(indices1.concatenate(indices2),
iobs1.concatenate(iobs2),
sel1.concatenate(sel2),
symm,
anomalous_flag=params.anomalous,
d_min=params.dmin,
d_max=params.dmax)
m1 = merge_obs(indices1,
iobs1,
sel1,
symm,
anomalous_flag=params.anomalous,
d_min=params.dmin,
d_max=params.dmax)
m2 = merge_obs(indices2,
iobs2,
sel2,
symm,
anomalous_flag=params.anomalous,
d_min=params.dmin,
d_max=params.dmax)
a1, a2 = m1.array().common_sets(m2.array())
red = flex.sum(m.redundancies().data()) / m.array().data().size(
) if m.array().data().size() > 0 else 0
rsplit = split_stats.calc_rsplit(a1, a2)
rano = split_stats.calc_rano(a1,
a2) if params.anomalous else float("nan")
cc = split_stats.calc_cc(a1, a2)
ccano = split_stats.calc_ccano(
a1, a2) if params.anomalous else float("nan")
ccanoref = split_stats.calc_ccano(
m.array(), anoref, take_common=True
) if params.anomalous and anoref is not None else float("nan")
ccref = split_stats.calc_cc(
m.array(), ref,
take_common=True) if ref is not None else float("nan")
snr = flex.mean(m.array().data() / m.array().sigmas())
print >> out, "%10d %7.2f %7.4f %7.4f % .4f % .4f %.2f %.4f % .4f % .4f" % (
e, red, rsplit, rano, cc, ccano, snr, rano / rsplit, ccanoref,
ccref)
out.flush()
byresolution_stats(params, a1, a2, m, anoref, ref,
("%" + str(len(str(nindexed))) + "d") % e,
out_byres)
print >> out_byres, "#"
def get_unit_cell_errors(stop_after=None):
'''Actually process something...'''
wd = os.getcwd()
all_miller_indices, all_two_thetas_obs, reference_cell, reference_lattice, reference_wavelength = load_sweeps_with_common_indexing()
Chatter.banner('Unit cell sampling')
Debug.banner('Unit cell sampling')
span = miller.index_span(all_miller_indices)
Chatter.write("Found %d reflections in 2theta range %.3f - %.3f deg" % (len(all_miller_indices), min(all_two_thetas_obs), max(all_two_thetas_obs)))
Chatter.write("Miller index range: %s - %s" % (str(span.min()), str(span.max())))
unit_cell_info = { 'reflections':
{ 'count': len(all_miller_indices),
'min_2theta': min(all_two_thetas_obs),
'max_2theta': max(all_two_thetas_obs),
'min_miller': list(span.min()),
'max_miller': list(span.max())
} }
# Exclude 1% of reflections to remove potential outliers
# eg. indexed/integrated high angle noise
two_theta_cutoff = sorted(all_two_thetas_obs)[-int(len(all_two_thetas_obs) * 0.01)-1]
Chatter.write("Excluding outermost 1%% of reflections (2theta >= %.3f)" % two_theta_cutoff)
two_thetas_select = all_two_thetas_obs < two_theta_cutoff
all_two_thetas_obs = all_two_thetas_obs.select(two_thetas_select)
all_miller_indices = all_miller_indices.select(two_thetas_select)
Chatter.write("Kept %d reflections in 2theta range %.3f - %.3f deg" % (len(all_miller_indices), min(all_two_thetas_obs), max(all_two_thetas_obs)))
span = miller.index_span(all_miller_indices)
unit_cell_info['reflections_filtered'] = \
{ 'count': len(all_miller_indices),
'min_2theta': min(all_two_thetas_obs),
'max_2theta': max(all_two_thetas_obs),
'min_miller': list(span.min()),
'max_miller': list(span.max())
}
# prepare MonteCarlo sampling
mc_runs = 50
sample_size = min(len(all_miller_indices) // 2, 100)
unit_cell_info['sampling'] = { 'method': 'montecarlo', 'runs': mc_runs, 'used_per_run': sample_size }
unit_cell_info['reference'] = { 'cell': reference_cell.parameters(), 'cell_volume': reference_cell.volume(),
'lattice': reference_lattice, 'wavelength': reference_wavelength }
Chatter.write("\nRandomly sampling %d x %d reflections for Monte Carlo iterations" % (mc_runs, sample_size))
Debug.write("Refinements start with reference unit cell: %s" % reference_cell)
MC = []
MCconstrained = []
used_index_range = flex.miller_index()
used_two_theta_range_min = 1e300
used_two_theta_range_max = 0
used_reflections = set()
for n in range(mc_runs): # MC sampling
# Select sample_size reflections
sample = flex.size_t(random.sample(range(len(all_miller_indices)), sample_size))
used_reflections = used_reflections.union(set(sample))
miller_indices = all_miller_indices.select(sample)
two_thetas_obs = all_two_thetas_obs.select(sample)
# Record
span = miller.index_span(miller_indices)
used_index_range.append(span.min())
used_index_range.append(span.max())
used_two_theta_range_min = min(used_two_theta_range_min, min(two_thetas_obs))
used_two_theta_range_max = max(used_two_theta_range_max, max(two_thetas_obs))
refined = _refinery(two_thetas_obs, miller_indices, reference_wavelength, reference_cell)
MC.append(refined.unit_cell().parameters() + (refined.unit_cell().volume(),))
Debug.write('Run %d refined to: %s' % (n, str(refined.unit_cell())))
if reference_lattice is not None and reference_lattice is not 'aP':
refined = _refinery(two_thetas_obs, miller_indices, reference_wavelength, reference_cell, reference_lattice[0])
MCconstrained.append(refined.unit_cell().parameters() + (refined.unit_cell().volume(),))
Debug.write('Run %d (constrained %s) refined to: %s' % (n, reference_lattice[0], str(refined.unit_cell())))
if (n % 50) == 0:
sys.stdout.write("\n%5s ." % (str(n) if n > 0 else ''))
else:
sys.stdout.write(".")
sys.stdout.flush()
assert used_two_theta_range_min < used_two_theta_range_max
def stats_summary(l):
mean = sum(l) / len(l)
var = 0
for y in l:
var = var + ((y - mean) ** 2)
popvar = var / (len(l)-1)
popstddev = math.sqrt(popvar)
stderr = popstddev / math.sqrt(len(l))
return { 'mean': mean, 'variance': var, 'population_variance': popvar,
'population_standard_deviation': popstddev, 'standard_error': stderr }
print
Chatter.write("")
Chatter.write("Unit cell estimation based on %d Monte Carlo runs," % len(MC))
span = miller.index_span(used_index_range)
Chatter.write("drawn from miller indices between %s and %s" % (str(span.min()), str(span.max())))
Chatter.write("with associated 2theta angles between %.3f and %.3f deg" % (used_two_theta_range_min, used_two_theta_range_max))
unit_cell_info['sampling'].update({
'used_reflections': len(used_reflections),
'used_max_2theta': used_two_theta_range_max,
'used_min_2theta': used_two_theta_range_min,
'used_max_miller': span.max(),
'used_min_miller': span.min() })
unit_cell_info['solution_unconstrained'] = {}
unit_cell_info['solution_constrained'] = { 'lattice': reference_lattice }
if reference_lattice is None or reference_lattice == 'aP':
Chatter.write("\n Unconstrained estimate:", strip=False)
for dimension, estimate in zip(['a', 'b', 'c', 'alpha', 'beta', 'gamma', 'volume'], zip(*MC)):
est_stats = stats_summary(estimate)
unit_cell_info['solution_unconstrained'][dimension] = est_stats
unit_cell_info['solution_constrained'][dimension] = est_stats
Chatter.write(" %6s =%10.5f (SD: %.5f, SE: %.5f)" % (dimension,
est_stats['mean'],
est_stats['population_standard_deviation'],
est_stats['standard_error']), strip=False)
else:
Chatter.write("\n Unconstrained estimate: | Constrained estimate (%s):" % reference_lattice, strip=False)
for dimension, estimate, constrained in zip(['a', 'b', 'c', 'alpha', 'beta', 'gamma', 'volume'], zip(*MC), zip(*MCconstrained)):
est_stats = stats_summary(estimate)
rest_stats = stats_summary(constrained)
unit_cell_info['solution_unconstrained'][dimension] = est_stats
unit_cell_info['solution_constrained'][dimension] = rest_stats
Chatter.write(" %6s =%10.5f (SD: %.5f, SE: %.5f) | %6s =%10.5f (SD: %.5f, SE: %.5f)" %
(dimension,
est_stats['mean'],
est_stats['population_standard_deviation'],
est_stats['standard_error'],
dimension,
rest_stats['mean'],
rest_stats['population_standard_deviation'],
rest_stats['standard_error']), strip=False)
with open(os.path.join(wd, 'xia2.get_unit_cell_errors.json'), 'w') as fh:
json.dump(unit_cell_info, fh, indent = 2, sort_keys=True)