def byresolution_stats(params, a1, a2, m, anoref, ref, prefix, out):
binner = a1.setup_binner(n_bins=params.nshells)
for i_bin in binner.range_used():
dmax, dmin = binner.bin_d_range(i_bin)
sel1 = a1.select(binner.bin_indices() == i_bin)
sel2 = a2.select(binner.bin_indices() == i_bin)
selm = m.array().resolution_filter_selection(d_max=dmax, d_min=dmin)
m_sel = m.array().select(selm)
red = flex.sum(m.redundancies().select(selm).data()) / m_sel.data(
).size() if m_sel.data().size() > 0 else 0
cmpl = m_sel.completeness(d_max=dmax) * 100.
cmplano = m_sel.anomalous_completeness(
d_max=dmax) * 100. if params.anomalous else float("nan")
rsplit = split_stats.calc_rsplit(sel1, sel2)
rano = split_stats.calc_rano(
sel1, sel2) if params.anomalous else float("nan")
cc = split_stats.calc_cc(sel1, sel2)
ccano = split_stats.calc_ccano(
sel1, sel2) if params.anomalous else float("nan")
ccanoref = split_stats.calc_ccano(
m_sel, anoref, take_common=True
) if params.anomalous and anoref is not None else float("nan")
ccref = split_stats.calc_cc(
m_sel, ref, take_common=True) if ref is not None else float("nan")
snr = flex.mean(m_sel.data() / m_sel.sigmas())
print >> out, "%s %5.2f %5.2f %5d %5.1f %5.1f %7.4f %7.4f % .4f % .4f %.2f %7.4f % .4f % .4f" % (
prefix, dmax, dmin, red, cmpl, cmplano, rsplit, rano, cc, ccano,
snr, rano / rsplit, ccanoref, ccref)
def byresolution_stats(params, a1, a2, m, anoref, ref, prefix, out):
binner = a1.setup_binner(n_bins=params.nshells)
for i_bin in binner.range_used():
dmax, dmin = binner.bin_d_range(i_bin)
sel1 = a1.select(binner.bin_indices() == i_bin)
sel2 = a2.select(binner.bin_indices() == i_bin)
selm = m.array().resolution_filter_selection(d_max=dmax, d_min=dmin)
m_sel = m.array().select(selm)
red = flex.sum(m.redundancies().select(selm).data()) / m_sel.data().size() if m_sel.data().size()>0 else 0
cmpl = m_sel.completeness(d_max=dmax)*100.
cmplano = m_sel.anomalous_completeness(d_max=dmax)*100. if params.anomalous else float("nan")
rsplit = split_stats.calc_rsplit(sel1, sel2)
rano = split_stats.calc_rano(sel1, sel2) if params.anomalous else float("nan")
cc = split_stats.calc_cc(sel1, sel2)
ccano = split_stats.calc_ccano(sel1, sel2) if params.anomalous else float("nan")
ccanoref = split_stats.calc_ccano(m_sel, anoref, take_common=True) if params.anomalous and anoref is not None else float("nan")
ccref = split_stats.calc_cc(m_sel, ref, take_common=True) if ref is not None else float("nan")
snr = flex.mean(m_sel.data()/m_sel.sigmas())
print >>out, "%s %5.2f %5.2f %5d %5.1f %5.1f %7.4f %7.4f % .4f % .4f %.2f %7.4f % .4f % .4f" % (prefix, dmax, dmin, red, cmpl, cmplano, rsplit, rano, cc, ccano, snr, rano/rsplit, ccanoref, ccref)
def show_split_stats(stream, nindexed, symm, params, anoref=None, ref=None, out_prefix="out", start_at=0):
random.seed(params.random_seed)
nstep = nindexed // params.nsplit
out = open(out_prefix + ".dat", "w")
out_byres = open(out_prefix + "_byres.dat", "w")
print >>out_byres, ("%"+str(len(str(nindexed)))+"s")%"nframes",
print >>out_byres, "dmin dmax red cmpl acmpl rsplit rano cc1/2 ccano snr rano/split CCanoref CCref"
#formatstr_for_eachout = out_prefix+"_to%."+str(len(str(nindexed)))+"d_byres.dat"
#indices1, iobs1, sel1 = [], [], []
#indices2, iobs2, sel2 = [], [], []
indices1, indices2 = flex.miller_index(), flex.miller_index()
iobs1, iobs2 = flex.double(), flex.double()
sel1, sel2 = flex.bool(), flex.bool()
print >>out, " nframes red Rsplit Rano CC1/2 CCano snr Rano/Rsplit CCanoref CCref"
chunks = read_stream(stream, start_at)
for i in xrange(params.nsplit):
e = (i+1)*nstep
if i == params.nsplit-1: e = nindexed
read_all = (i == params.nsplit-1)
slc = []
for j, c in enumerate(chunks):
sys.stderr.write("\rprocessed: %d" % (i*nstep+j))
sys.stderr.flush()
slc.append(c)
if not read_all and j >= nstep-1:
break
if nindexed <= i*nstep+j:
break
if params.halve_method == "alternate":
slc1 = slc[0::2]
slc2 = slc[1::2]
elif params.halve_method == "random":
perm = range(len(slc))
random.shuffle(perm)
nhalf = len(slc)//2
slc1 = [slc[r] for r in perm[:nhalf]]
slc2 = [slc[r] for r in perm[nhalf:]]
else:
raise "Not-supported:", params.halve_method
i1, o1, s1 = [], [], []
for x in slc1:
i1.extend(x.indices)
o1.extend(x.iobs)
if params.adu_cutoff is not None:
s1.extend((a <= params.adu_cutoff for a in x.peak))
i2, o2, s2 = [], [], []
for x in slc2:
i2.extend(x.indices)
o2.extend(x.iobs)
if params.adu_cutoff is not None:
s2.extend((a <= params.adu_cutoff for a in x.peak))
# Concatenate and sort
indices1 = indices1.concatenate(flex.miller_index(i1))
iobs1 = iobs1.concatenate(flex.double(o1))
sel1 = sel1.concatenate(flex.bool(s1))
indices2 = indices2.concatenate(flex.miller_index(i2))
iobs2 = iobs2.concatenate(flex.double(o2))
sel2 = sel2.concatenate(flex.bool(s2))
perm1 = flex.sort_permutation(data=miller.index_span(indices1).pack(indices1), reverse=False)
perm2 = flex.sort_permutation(data=miller.index_span(indices2).pack(indices2), reverse=False)
indices1 = indices1.select(perm1)
indices2 = indices2.select(perm2)
iobs1 = iobs1.select(perm1)
iobs2 = iobs2.select(perm2)
if params.adu_cutoff is not None:
sel1 = sel1.select(perm1)
sel2 = sel2.select(perm2)
# Merge
m = merge_obs(indices1.concatenate(indices2), iobs1.concatenate(iobs2), sel1.concatenate(sel2), symm, anomalous_flag=params.anomalous, d_min=params.dmin, d_max=params.dmax)
m1 = merge_obs(indices1, iobs1, sel1, symm, anomalous_flag=params.anomalous, d_min=params.dmin, d_max=params.dmax)
m2 = merge_obs(indices2, iobs2, sel2, symm, anomalous_flag=params.anomalous, d_min=params.dmin, d_max=params.dmax)
a1, a2 = m1.array().common_sets(m2.array())
red = flex.sum(m.redundancies().data()) / m.array().data().size() if m.array().data().size()>0 else 0
rsplit = split_stats.calc_rsplit(a1, a2)
rano = split_stats.calc_rano(a1, a2) if params.anomalous else float("nan")
cc = split_stats.calc_cc(a1, a2)
ccano = split_stats.calc_ccano(a1, a2) if params.anomalous else float("nan")
ccanoref = split_stats.calc_ccano(m.array(), anoref, take_common=True) if params.anomalous and anoref is not None else float("nan")
ccref = split_stats.calc_cc(m.array(), ref, take_common=True) if ref is not None else float("nan")
snr = flex.mean(m.array().data()/m.array().sigmas())
print >>out, "%10d %7.2f %7.4f %7.4f % .4f % .4f %.2f %.4f % .4f % .4f" % (e, red, rsplit, rano, cc, ccano, snr, rano/rsplit, ccanoref, ccref)
out.flush()
byresolution_stats(params, a1, a2, m, anoref, ref, ("%"+str(len(str(nindexed)))+"d")%e, out_byres)
print >>out_byres, "#"
def show_split_stats(stream,
nindexed,
symm,
params,
anoref=None,
ref=None,
out_prefix="out",
start_at=0):
random.seed(params.random_seed)
nstep = nindexed // params.nsplit
out = open(out_prefix + ".dat", "w")
out_byres = open(out_prefix + "_byres.dat", "w")
print >> out_byres, ("%" + str(len(str(nindexed))) + "s") % "nframes",
print >> out_byres, "dmin dmax red cmpl acmpl rsplit rano cc1/2 ccano snr rano/split CCanoref CCref"
#formatstr_for_eachout = out_prefix+"_to%."+str(len(str(nindexed)))+"d_byres.dat"
#indices1, iobs1, sel1 = [], [], []
#indices2, iobs2, sel2 = [], [], []
indices1, indices2 = flex.miller_index(), flex.miller_index()
iobs1, iobs2 = flex.double(), flex.double()
sel1, sel2 = flex.bool(), flex.bool()
print >> out, " nframes red Rsplit Rano CC1/2 CCano snr Rano/Rsplit CCanoref CCref"
chunks = read_stream(stream, start_at)
for i in xrange(params.nsplit):
e = (i + 1) * nstep
if i == params.nsplit - 1: e = nindexed
read_all = (i == params.nsplit - 1)
slc = []
for j, c in enumerate(chunks):
sys.stderr.write("\rprocessed: %d" % (i * nstep + j))
sys.stderr.flush()
slc.append(c)
if not read_all and j >= nstep - 1:
break
if nindexed <= i * nstep + j:
break
if params.halve_method == "alternate":
slc1 = slc[0::2]
slc2 = slc[1::2]
elif params.halve_method == "random":
perm = range(len(slc))
random.shuffle(perm)
nhalf = len(slc) // 2
slc1 = [slc[r] for r in perm[:nhalf]]
slc2 = [slc[r] for r in perm[nhalf:]]
else:
raise "Not-supported:", params.halve_method
i1, o1, s1 = [], [], []
for x in slc1:
i1.extend(x.indices)
o1.extend(x.iobs)
if params.adu_cutoff is not None:
s1.extend((a <= params.adu_cutoff for a in x.peak))
i2, o2, s2 = [], [], []
for x in slc2:
i2.extend(x.indices)
o2.extend(x.iobs)
if params.adu_cutoff is not None:
s2.extend((a <= params.adu_cutoff for a in x.peak))
# Concatenate and sort
indices1 = indices1.concatenate(flex.miller_index(i1))
iobs1 = iobs1.concatenate(flex.double(o1))
sel1 = sel1.concatenate(flex.bool(s1))
indices2 = indices2.concatenate(flex.miller_index(i2))
iobs2 = iobs2.concatenate(flex.double(o2))
sel2 = sel2.concatenate(flex.bool(s2))
perm1 = flex.sort_permutation(
data=miller.index_span(indices1).pack(indices1), reverse=False)
perm2 = flex.sort_permutation(
data=miller.index_span(indices2).pack(indices2), reverse=False)
indices1 = indices1.select(perm1)
indices2 = indices2.select(perm2)
iobs1 = iobs1.select(perm1)
iobs2 = iobs2.select(perm2)
if params.adu_cutoff is not None:
sel1 = sel1.select(perm1)
sel2 = sel2.select(perm2)
# Merge
m = merge_obs(indices1.concatenate(indices2),
iobs1.concatenate(iobs2),
sel1.concatenate(sel2),
symm,
anomalous_flag=params.anomalous,
d_min=params.dmin,
d_max=params.dmax)
m1 = merge_obs(indices1,
iobs1,
sel1,
symm,
anomalous_flag=params.anomalous,
d_min=params.dmin,
d_max=params.dmax)
m2 = merge_obs(indices2,
iobs2,
sel2,
symm,
anomalous_flag=params.anomalous,
d_min=params.dmin,
d_max=params.dmax)
a1, a2 = m1.array().common_sets(m2.array())
red = flex.sum(m.redundancies().data()) / m.array().data().size(
) if m.array().data().size() > 0 else 0
rsplit = split_stats.calc_rsplit(a1, a2)
rano = split_stats.calc_rano(a1,
a2) if params.anomalous else float("nan")
cc = split_stats.calc_cc(a1, a2)
ccano = split_stats.calc_ccano(
a1, a2) if params.anomalous else float("nan")
ccanoref = split_stats.calc_ccano(
m.array(), anoref, take_common=True
) if params.anomalous and anoref is not None else float("nan")
ccref = split_stats.calc_cc(
m.array(), ref,
take_common=True) if ref is not None else float("nan")
snr = flex.mean(m.array().data() / m.array().sigmas())
print >> out, "%10d %7.2f %7.4f %7.4f % .4f % .4f %.2f %.4f % .4f % .4f" % (
e, red, rsplit, rano, cc, ccano, snr, rano / rsplit, ccanoref,
ccref)
out.flush()
byresolution_stats(params, a1, a2, m, anoref, ref,
("%" + str(len(str(nindexed))) + "d") % e,
out_byres)
print >> out_byres, "#"