def add_multi(xds_file, workmtz, dmin=None, dmax=None, force_anomalous=False):
from yamtbx.dataproc.xds import xds_ascii
print "Adding multiplicity for each reflection"
xac = xds_ascii.XDS_ASCII(xds_file)
iobs = xac.i_obs(anomalous_flag=True if force_anomalous else None)
iobs = iobs.resolution_filter(
d_min=float(dmin) if dmin is not None else None,
d_max=float(dmax) if dmax is not None else None)
iobs = iobs.select(iobs.sigmas() > 0).map_to_asu()
merge = iobs.merge_equivalents(use_internal_variance=False)
array_merged = merge.array()
reject_sel = (array_merged.data() < -3 * array_merged.sigmas())
#print " rejecting %d reflections (<<I>/sd(<I>)> < -3)" % reject_sel.count(True)
iobs = iobs.delete_indices(other=array_merged.select(reject_sel))
# merge again after rejection
merge = iobs.merge_equivalents(use_internal_variance=False)
mtz_object = iotbx.mtz.object(workmtz)
crystals = mtz_object.crystals()
crystals[-1].datasets()[-1].add_miller_array(
miller_array=merge.redundancies(), column_root_label="MULTIPLICITY")
mtz_object.write(file_name=workmtz)
def run(files, params):
print "filename",
for cut in params.cut_ios: print "cut_ios_%.2f" % cut,
print
for f in files:
is_xac = xds_ascii.is_xds_ascii(f)
i_obs = None
if is_xac:
xac = xds_ascii.XDS_ASCII(f, read_data=True, i_only=True)
xac.remove_rejected()
i_obs = xac.i_obs().resolution_filter(d_min=params.d_min, d_max=params.d_max)
if params.fix_variance_model:
ao, bo = xac.variance_model
an, bn = params.variance_model
i_obs = i_obs.customized_copy(sigmas = flex.sqrt(flex.abs(an * (i_obs.sigmas()**2/ao + (bn-bo)*flex.pow2(i_obs.data())))))
else:
ihkl = integrate_hkl_as_flex.reader(f, read_columns=("IOBS","SIGMA"))
i_obs = ihkl.i_obs().resolution_filter(d_min=params.d_min, d_max=params.d_max)
if params.fix_variance_model:
a, b = params.variance_model
i_obs = i_obs.customized_copy(sigmas = flex.sqrt(a * (i_obs.sigmas()**2 + b*flex.pow2(i_obs.data()))))
cutoffs = eval_resolution(i_obs, params.n_bins, params.cut_ios)
print "%s %s" % (f, " ".join(map(lambda x: "%.2f"%x, cutoffs)))
def eval_cc_with_original_file(f, merged_iobs):
print "reading", f
xac = xds_ascii.XDS_ASCII(f)
iobs = xac.i_obs(
anomalous_flag=merged_iobs.anomalous_flag()).merge_equivalents(
use_internal_variance=False).array()
n_all = iobs.size()
m, i = merged_iobs.common_sets(iobs, assert_is_similar_symmetry=False)
n_common = m.size()
corr = flex.linear_correlation(m.data(), i.data())
cc = corr.coefficient() if corr.is_well_defined() else float("nan")
ret1 = (n_all, n_common, cc)
ret2 = []
for frame in xrange(min(xac.iframe), max(xac.iframe) + 1):
sel = xac.iframe == frame
iobs = xac.i_obs(
anomalous_flag=merged_iobs.anomalous_flag()).select(sel)
iobs = iobs.merge_equivalents(use_internal_variance=False).array()
n_all = iobs.size()
m, i = merged_iobs.common_sets(iobs, assert_is_similar_symmetry=False)
n_common = m.size()
corr = flex.linear_correlation(m.data(), i.data())
cc = corr.coefficient() if corr.is_well_defined() else float("nan")
ret2.append([frame, n_all, n_common, cc])
return ret1, ret2
def run(xds_ascii_in):
reader = xds_ascii.XDS_ASCII(xds_ascii_in, sys.stdout)
#reader.remove_rejected()
data = [[] for i in xrange(10)]
for iobs, zd in zip(reader.iobs, reader.zd):
dec = math.modf(zd)[0]
if dec < 0:
dec += 1
idx = int(math.ceil((dec-0.05)/0.1))
if idx > 9:
idx = 0
data[idx].append(iobs)
iobs_overall = sum(map(lambda x:sum(x), data)) / sum(map(lambda x:len(x), data))
print " INTERVAL NUMBER INTENSITY FACTOR"
for i, iobs in enumerate(data):
iobs_interval = sum(iobs)/len(iobs)
print " % .2f % .2f %9d %14.3f %11.3f"%(-0.05+i*.1, -0.05+i*.1+.1, len(iobs), iobs_interval, iobs_overall/iobs_interval)
print
print " % .2f % .2f %9d %14.3f %11.3f"%(-0.05, 0.95, reader.iobs.size(), iobs_overall, 1.)
def run_blend0R(wdir, xds_ascii_files, logout="blend0.log"):
ofs_cell = open(os.path.join(wdir, "forR_macropar.dat"), "w")
ofs_lst = open(os.path.join(wdir, "xds_lookup_table.txt"), "w")
ofs_files = open(os.path.join(wdir, "NEW_list_of_files.dat"), "w") # Just to avoid error
for i, xac in enumerate(xds_ascii_files):
try:
symm = xds_ascii.XDS_ASCII(xac, read_data=False).symm
except:
symm = any_reflection_file(xac).as_miller_arrays()[0].crystal_symmetry()
cell = " ".join(map(lambda x: "%7.3f"%x, symm.unit_cell().parameters()))
sgnum = int(symm.space_group().type().number())
ofs_lst.write("dataset_%.3d %s\n" % (i, xac))
ofs_cell.write("%4d %s 0 0 0 \"%s\"\n" % (i+1, cell, sg_to_cent[sgnum]))
ofs_files.write("%s\n"%xac)
ofs_lst.close()
ofs_cell.close()
ofs_files.close()
shutil.copyfile(os.path.join(wdir, "forR_macropar.dat"),
os.path.join(wdir, "forR_macropar.dat.bak"))
pathblend = "modules/yamtbx/blend/R/blend0.R"
useNC = os.getenv("BLEND_USE_NCDIST","no")
useSF = os.getenv("BLEND_USE_SFDIST","no")
if useNC != "no":
pathblend = "modules/yamtbx/blend/R/blend0NC.R"
if useSF != "no":
pathblend = "modules/yamtbx/blend/R/blend0SF.R"
util.call("Rscript", os.path.join(os.environ["PHENIX"], pathblend),
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "w"))
open(os.path.join(wdir, "hctojson.R"), "w").write("""\
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
labels<-hc$labels
merge<-data.frame(hc$merge)
for (i in (1:nrow(merge))) {
if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
}
eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
return(JSON)
}
load("BLEND0.RData",.GlobalEnv)
JSON<-HCtoJSON(npar.hc_ward)
cat(JSON, file="dendro.json")
""")
util.call("Rscript", "hctojson.R",
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "a"))
def run(hklin):
xscaled = xds_ascii.XDS_ASCII(hklin) # Must be XSCALE output
merged_iobs = xscaled.i_obs().merge_equivalents(
use_internal_variance=False).array()
binner = merged_iobs.setup_binner(n_bins=100)
isets = set(xscaled.iset)
for_plot = {}
cut_ios = (2, 1, 0.5, 0)
print "iset file",
for cut in cut_ios:
print "cut_ios_%.2f" % cut,
print
for iset in isets:
sel = (xscaled.iset == iset)
data_i = xscaled.i_obs().select(sel).merge_equivalents(
use_internal_variance=False).array()
cutoffs = eval_resolution(data_i, 100, cut_ios)
print "%3d %s %s" % (iset, xscaled.input_files[iset][0], " ".join(
map(lambda x: "%.2f" % x, cutoffs)))
name = "data%.2d" % iset
for i_bin in binner.range_used():
dmax, dmin = binner.bin_d_range(i_bin)
Isel = data_i.resolution_filter(d_max=dmax, d_min=dmin)
for_plot.setdefault(
name,
[]).append(flex.mean(Isel.data()) if Isel.size() > 0 else 0)
import matplotlib
matplotlib.use('Agg') # Allow to work without X
import pylab
import math
from matplotlib.ticker import FuncFormatter
s2_formatter = lambda x, pos: "inf" if x == 0 else "%.2f" % (1. / math.
sqrt(x))
fig, ax1 = pylab.plt.subplots()
plot_x = [binner.bin_d_range(i)[1]**(-2) for i in binner.range_used()]
plots = {}
for name, vals in for_plot.items():
name = name[-30:].lstrip("_")
plots[name] = pylab.plot(plot_x, vals, label=name)
pylab.legend()
pylab.xlabel('resolution (d^-3)')
pylab.ylabel('<I>')
pylab.setp(pylab.gca().get_legend().get_texts(), fontsize="small")
plot_title = ""
pylab.title(plot_title)
pylab.gca().xaxis.set_major_formatter(FuncFormatter(s2_formatter))
pylab.savefig("test.pdf")
pylab.show()
def load_xds_data_only_indices(xac_files, d_min=None):
miller_sets = {}
for f in xac_files:
if xds_ascii.is_xds_ascii(f):
print "Loading", f
ms = xds_ascii.XDS_ASCII(f, i_only=True).as_miller_set()
miller_sets[f] = ms.resolution_filter(d_min=d_min)
elif integrate_hkl_as_flex.is_integrate_hkl(f):
print "Sorry, skipping", f
else:
print "Skipping unrecognized:", f
return miller_sets
def run_blend0R(wdir, xds_ascii_files, logout="blend0.log"):
ofs_cell = open(os.path.join(wdir, "forR_macropar.dat"), "w")
ofs_lst = open(os.path.join(wdir, "xds_lookup_table.txt"), "w")
ofs_files = open(os.path.join(wdir, "NEW_list_of_files.dat"),
"w") # Just to avoid error
for i, xac in enumerate(xds_ascii_files):
symm = xds_ascii.XDS_ASCII(xac, read_data=False).symm
cell = " ".join(
map(lambda x: "%7.3f" % x,
symm.unit_cell().parameters()))
ofs_lst.write("dataset_%.3d %s\n" % (i, xac))
ofs_cell.write("%4d %s 0 0 0\n" % (i + 1, cell))
ofs_files.write("%s\n" % xac)
ofs_lst.close()
ofs_cell.close()
ofs_files.close()
shutil.copyfile(os.path.join(wdir, "forR_macropar.dat"),
os.path.join(wdir, "forR_macropar.dat.bak"))
util.call("Rscript",
os.path.join(os.environ["CCP4"], "share/blend/R/blend0.R"),
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "w"))
open(os.path.join(wdir, "hctojson.R"), "w").write("""\
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
labels<-hc$labels
merge<-data.frame(hc$merge)
for (i in (1:nrow(merge))) {
if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
}
eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
return(JSON)
}
load("BLEND0.RData",.GlobalEnv)
JSON<-HCtoJSON(npar.hc_ward)
cat(JSON, file="dendro.json")
""")
util.call("Rscript",
"hctojson.R",
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "a"))
def run(hklin, output_dir=None, eval_internal=True):
if output_dir is None: output_dir = os.getcwd()
merged = xds_ascii.XDS_ASCII(hklin)
merged_iobs = merged.i_obs().merge_equivalents(
use_internal_variance=False).array()
fwidth = max(map(lambda x: len(x[0]), merged.input_files.values()))
formatf = "%" + str(fwidth) + "s"
out_files = open(os.path.join(output_dir, "cc_files.dat"), "w")
out_frames = open(os.path.join(output_dir, "cc_frames.dat"), "w")
print >> out_files, "file name n.all n.common cc"
print >> out_frames, "file name frame n.all n.common cc"
cutforname1 = len(
os.path.commonprefix(map(lambda x: x[0], merged.input_files.values())))
cutforname2 = len(
os.path.commonprefix(
map(lambda x: x[0][::-1], merged.input_files.values())))
formatn = "%" + str(fwidth - cutforname1 - cutforname2) + "s"
if eval_internal:
results = map(lambda x: eval_cc_internal(merged_iobs, merged, x),
sorted(set(merged.iset)))
else:
results = map(
lambda x: eval_cc_with_original_file(x, merged_iobs),
map(
lambda x: x[0] if os.path.isabs(x[0]) else os.path.join(
os.path.dirname(hklin), x[0]),
merged.input_files.values()))
ret = collections.OrderedDict()
for (f, wavelen, cell), (ret1, ret2) in zip(merged.input_files.values(),
results):
name = f[cutforname1 + 1:-cutforname2]
n_all, n_common, cc = ret1
print >> out_files, formatf % f, formatn % name, "%5d %5d %.4f" % (
n_all, n_common, cc)
ret[f] = []
for frame, n_all, n_common, cc in ret2:
print >> out_frames, formatf % f, formatn % name, "%6d %5d %5d %.4f" % (
frame, n_all, n_common, cc)
ret[f].append([frame, n_all, n_common, cc])
return ret
def run_for_symm(self, hklin=None, xdsin=None, logout=None, tolerance=None, d_min=None, choose_laue=None, xdsin_to_p1=False, setting="c2"):
assert (hklin, xdsin).count(None) == 1
assert setting.lower().startswith(("c2", "cell", "symm"))
tmpfile = None
out = open(logout, "w") if logout else null_out()
# Merge in P1 before pointless for quick evaluation
if xdsin and xdsin_to_p1:
out.write("Merging in P1 symmetry before Pointless..\n")
xac = xds_ascii.XDS_ASCII(xdsin)
xac.remove_rejected()
i_obs = xac.i_obs(anomalous_flag=True)
i_obs.show_summary(out, " ")
i_obs = i_obs.apply_change_of_basis(change_of_basis="to_niggli_cell", out=out)[0]
i_obs = i_obs.customized_copy(space_group_info=sgtbx.space_group_info("P1"))
i_obs = i_obs.merge_equivalents(use_internal_variance=False).array()
tmpfile = util.get_temp_filename(prefix="", suffix=os.path.basename(xdsin) + "_p1.mtz")
i_obs.as_mtz_dataset(column_root_label="I").mtz_object().write(tmpfile)
xdsin, hklin = None, tmpfile
out.write("\n")
filein, wdir = self.input_str(hklin, xdsin)
# Need centre keyword?
stdin = """\
SETTING %s
%s
""" % (setting, filein)
if tolerance is not None:
stdin += "tolerance %f\n" % tolerance
if d_min is not None:
stdin += "resolution high %f\n" % d_min
if choose_laue is not None:
stdin += "choose lauegroup %s\n" % choose_laue
exitcode, output, err = util.call(pointless_comm,
stdin=stdin, wdir=wdir)
parsed = parse_pointless_output_for_symm(output)
out.write(output)
if tmpfile and os.path.isfile(tmpfile):
os.remove(tmpfile)
return parsed
def run(params, target_files):
assert params.normalization in ("no", "E")
ofs = open(params.dat_out, "w")
xac_files = util.read_path_list(params.lstin)
targets = read_target_files(target_files, params.d_min, params.d_max,
params.normalization, ofs)
cellcon = CellConstraints(targets.values()[0].space_group())
#for i, t in enumerate(targets): ofs.write("# target%.3d = %s\n" % (i,t))
ofs.write("# normalization = %s\n" % params.normalization)
ofs.write("# d_min, d_max = %s, %s\n" % (params.d_min, params.d_max))
ofs.write("file %s " % cellcon.get_label_for_free_params())
ofs.write(" ".join(
map(lambda x: "cc.%.3d nref.%.3d" % (x, x), xrange(len(targets)))))
ofs.write("\n")
for xac_file in xac_files:
print "reading", xac_file
xac = xds_ascii.XDS_ASCII(xac_file)
xac.remove_rejected()
iobs = xac.i_obs(anomalous_flag=False).merge_equivalents(
use_internal_variance=False).array()
ofs.write("%s %s" %
(xac_file, cellcon.format_free_params(iobs.unit_cell())))
fail_flag = False
if params.normalization == "E":
try:
normaliser = kernel_normalisation(iobs, auto_kernel=True)
iobs = iobs.customized_copy(
data=iobs.data() / normaliser.normalizer_for_miller_array,
sigmas=iobs.sigmas() /
normaliser.normalizer_for_miller_array)
except:
fail_flag = True
for i, ta in enumerate(targets.values()):
if fail_flag:
ofs.write(" % .4f %4d" % cc_num)
else:
cc_num = calc_cc(iobs, ta)
ofs.write(" % .4f %4d" % cc_num)
ofs.write("\n")
def load_xds_data_only_indices(xac_files, d_min=None):
miller_sets = {}
for f in xac_files:
try:
indices = []
miller_arrays = any_reflection_file(f).as_miller_arrays()
symm = miller_arrays[0].crystal_symmetry()
ms = miller.set(crystal_symmetry=symm, indices=miller_arrays[0].indices(),anomalous_flag=False)
miller_sets[f] = ms.resolution_filter(d_min=d_min)
except:
if xds_ascii.is_xds_ascii(f):
print "Loading", f
ms = xds_ascii.XDS_ASCII(f, i_only=True).as_miller_set()
miller_sets[f] = ms.resolution_filter(d_min=d_min)
elif integrate_hkl_as_flex.is_integrate_hkl(f):
print "Sorry, skipping", f
else:
print "Skipping unrecognized:", f
return miller_sets
def find_unconnected_xds_files(xds_files, min_ios=3, d_min=None):
import networkx as nx
from yamtbx.dataproc.xds import xds_ascii
from yamtbx.dataproc.xds import integrate_hkl_as_flex
if len(xds_files) < 2:
return []
G = nx.Graph()
arrays = []
for f in xds_files:
if xds_ascii.is_xds_ascii(f):
a = xds_ascii.XDS_ASCII(
f, i_only=True).i_obs().resolution_filter(d_min=d_min)
elif integrate_hkl_as_flex.is_integrate_hkl(f):
a = integrate_hkl_as_flex.reader(
f, ["IOBS", "SIGMA"]).i_obs().resolution_filter(d_min=d_min)
else:
raise "Never reaches here"
a = a.select(a.sigmas() > 0)
a = a.merge_equivalents(use_internal_variance=False).array()
arrays.append(a.select(a.data() / a.sigmas() >= min_ios))
for i in xrange(len(arrays) - 1):
for j in xrange(i + 1, len(arrays)):
matchs = arrays[i].match_indices(other=arrays[j],
assert_is_similar_symmetry=False)
if matchs.pairs().size() >= 10:
G.add_edge(i, j)
print "edge", i, j
ccomps = map(lambda x: x, nx.connected_components(G))
print "DEBUG:: Connected components=", ccomps
keep_idxes = ccomps[0]
remove_idxes = filter(lambda x: x not in keep_idxes,
xrange(len(xds_files)))
return remove_idxes
def run(tabin, cc_min):
lst = read_input(tabin, cc_min)
inp_ofs = open("XSCALE.INP", "w")
topdir = os.path.dirname(
os.path.commonprefix(map(lambda x: os.path.abspath(x), lst.keys())))
print "Cut frames with CC < %.4f" % cc_min
for f in lst:
useframes = lst[f]
xac = xds_ascii.XDS_ASCII(f)
hklout = os.path.join("files", os.path.relpath(f, topdir))
if not os.path.exists(os.path.dirname(hklout)):
os.makedirs(os.path.dirname(hklout))
if useframes is None or set(useframes).issuperset(
set(range(min(xac.iframe), max(xac.iframe)))):
print "Just copying to %s" % hklout
shutil.copyfile(f, hklout)
else:
sel = xac.iframe == useframes[0]
for x in useframes[1:]:
sel |= xac.iframe == x
# XXX we may need to check I/sigma and throw < -3..
if sum(sel) < 10:
print "Skipping %s with only %6d/%6d" % (hklout, sum(sel),
len(sel))
continue
print "Saving %s %6d/%6d" % (hklout, sum(sel), len(sel))
xac.write_selected(sel, hklout)
inp_ofs.write("INPUT_FILE= %s\n" % hklout)
def get_data_from_xac(params, xac):
if xac.endswith(".pkl"):
tmp = pickle.load(open(xac))
else:
tmp = xds_ascii.XDS_ASCII(xac)
sel_remove = flex.bool(tmp.iobs.size(), False)
if params.min_peak is not None:
sel = tmp.peak < params.min_peak
sel_remove |= sel
elif params.min_peak_percentile is not None:
q = numpy.percentile(tmp.peak, params.min_peak_percentile)
print "percentile %.2f %s" % (q, xac)
sel = tmp.peak < q
sel_remove |= sel
if params.skip_rejected: sel_remove |= (tmp.sigma_iobs <= 0)
if params.dmin is not None:
sel_remove |= ~tmp.as_miller_set().resolution_filter_selection(d_min=params.dmin)
if params.correct_peak:
sel_remove |= (tmp.peak < 1) # remove PEAK==0
# Remove selected
print "DEBUG:: removing %d reflections" % sel_remove.count(True) #sum(sel_remove)#
tmp.remove_selection(sel_remove)
if not params.skip_rejected: tmp.sigma_iobs = flex.abs(tmp.sigma_iobs)
# Correct I,sigI if needed
if params.correct_peak:
tmp.iobs *= tmp.peak * .01
tmp.sigma_iobs *= tmp.peak * .01
if params.cancel_rlp:
tmp.iobs /= tmp.rlp
tmp.sigma_iobs /= tmp.rlp
if params.polarization.correct:
# Only works with single-panel detector!!
# Assumes detector fast = (1,0,0), slow = (0,1,0)
sin_sq_2theta = tmp.symm.unit_cell().sin_sq_two_theta(tmp.indices, tmp.wavelength)
cos_sq_2theta = 1. - sin_sq_2theta
sin_theta = tmp.wavelength / tmp.symm.unit_cell().d(tmp.indices) / 2.
Eppi = numpy.cross(params.polarization.plane_normal, params.polarization.incident_beam_direction)
Eppi /= numpy.linalg.norm(Eppi)
S = flex.vec3_double(tmp.xd - tmp.orgx, tmp.yd - tmp.orgy,
flex.double(tmp.xd.size(), tmp.distance/tmp.qx))
S /= S.norms()
zp = S.dot(Eppi.tolist()) * 2. * sin_theta
cosrho = zp / flex.sqrt(sin_sq_2theta)
P0 = 0.5 * (1. + cos_sq_2theta)
PP = (params.polarization.fraction - 0.5) * (2.*cosrho**2 - 1.) * sin_sq_2theta
P = P0 - PP
# Apply correction
tmp.iobs /= P
tmp.sigma_iobs /= P
if 0: # debug
for x, y, p in zip(tmp.xd, tmp.yd, P):
print "pdebug:: %.2f %.2f %.4e" % (x, y, p)
return tmp
def from_xds_ascii(xacin, ref_xs):
xac = xds_ascii.XDS_ASCII(xacin, read_data=False)
return get_angles(xac.incident_axis, numpy.array(xac.a_axis),
numpy.array(xac.b_axis), numpy.array(xac.c_axis),
xac.symm, ref_xs, xacin)
def xds_sequence(img_in, topdir, data_root_dir, params):
relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
print workdir
frame_num = None
if not os.path.exists(img_in):
if "<>" in img_in:
img_in, num = img_in.split("<>")
frame_num = int(num)
if not os.path.exists(img_in):
print "Error: Not found!!", img_in
return
workdir += "_%.6d" % frame_num
assert img_in.endswith(".h5")
else:
for ext in (".bz2", ".gz", ".xz"):
if os.path.exists(img_in+ext):
img_in += ext
break
if params.tmpdir is not None:
tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
else:
tmpdir = workdir
if not os.path.exists(tmpdir): os.makedirs(tmpdir)
xparm = os.path.join(tmpdir, "XPARM.XDS")
xdsinp = os.path.join(tmpdir, "XDS.INP")
integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
decilog = open(os.path.join(tmpdir, "decision.log"), "w")
try:
print >>decilog, "Paramters:"
libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")
print >>decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")
# Prepare XDS.INP
if params.sfx.cheetah_mpccd:
xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
else:
if frame_num is not None: # Eiger
from yamtbx.dataproc import eiger
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
else:
ext = os.path.splitext(img_in)[1]
img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
os.symlink(img_in, img_in_work)
xdsinp_str = xds_inp.generate_xds_inp(img_files=[img_in_work],
inp_dir=tmpdir,
reverse_phi=True, anomalous=params.anomalous,
spot_range=None, minimum=False,
crystal_symmetry=None,
integrate_nimages=None,
osc_range=params.osc_range,
orgx=params.orgx, orgy=params.orgy,
rotation_axis=params.rotation_axis,
distance=params.distance)
ofs = open(xdsinp, "w")
ofs.write(xdsinp_str)
ofs.close()
# Start processing
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF"),
("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
("MINPK", "%.2f"%params.minpk),
("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
("STRONG_PIXEL", "%.2f"%params.strong_pixel),
("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d"%params.minimum_number_of_pixels_in_a_spot),
("SEPMIN", "%.2f"%params.sepmin),
("CLUSTER_RADIUS", "%.2f"%params.cluster_radius),
("INDEX_ERROR", "%.4f"%params.index_error),
("INDEX_MAGNITUDE", "%d"%params.index_magnitude),
("INDEX_QUALITY", "%.2f"%params.index_quality),
("REFINE(IDXREF)", " ".join(params.refine_idxref)),
("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
("VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS", "%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels))
])
if len(params.extra_inp_str) > 0:
ofs = open(xdsinp, "a")
ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
ofs.close()
run_xds(wdir=tmpdir, show_progress=False)
if params.tryhard:
try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)
# If Cell hint exists, try to use it..
if params.sgnum > 0:
flag_try_cell_hint = False
if not os.path.isfile(xparm):
flag_try_cell_hint = True
else:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None: flag_try_cell_hint = True
if flag_try_cell_hint:
print >>decilog, " Worth trying to use prior cell for indexing."
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("UNIT_CELL_CONSTANTS",
" ".join(map(lambda x: "%.3f"%x, params.cell))),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
])
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
])
if not os.path.isfile(xparm):
raise ProcFailed("Indexing failed")
if params.checkcell.check and params.sgnum > 0:
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed("Incompatible cell. Indexing failed.")
if not cosets.combined_cb_ops()[0].is_identity_op():
print >>decilog, "Re-idxref to match reference symmetry."
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))
])
run_xds(wdir=tmpdir, show_progress=False)
modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min)),
])
# To Integration
modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),])
run_xds(wdir=tmpdir, show_progress=False)
if not os.path.isfile(integrate_hkl):
raise ProcFailed("Integration failed.")
# Determine unit cell in CORRECT
if params.refine_correct:
tmp = [("REFINE(CORRECT)", "ALL"),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, params.cell)))]
else:
# XXX What if CELL is refined in INTEGRATE?
xsxds = XPARM(xparm).crystal_symmetry()
cosets = check_cell(params, xsxds)
if cosets.double_cosets is None:
raise ProcFailed(" Incompatible cell. Failed before CORRECT.")
xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
tmp = [("REFINE(CORRECT)", ""),
("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))]
# PEAK-corrected INTEGRATE.HKL
ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
inhkl.write_peak_corrected(ihk_part)
os.rename(ihk, ihk_full)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
("DATA_RANGE", "1 20000"),
("CORRECTIONS", ""),
("NBATCH", "1"),
("SPACE_GROUP_NUMBER", "%d"%params.sgnum)] + tmp)
xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")
# CORRECT for full
os.symlink(ihk_full, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_full)
os.rename(os.path.join(tmpdir, "CORRECT.LP"),
os.path.join(tmpdir, "CORRECT_full.LP"))
os.remove(ihk)
# CORRECT for part
os.symlink(ihk_part, ihk)
run_xds(wdir=tmpdir, comm="xds", show_progress=False)
if os.path.isfile(xac):
os.rename(xac, xac_part)
os.rename(os.path.join(tmpdir, "CORRECT.LP"),
os.path.join(tmpdir, "CORRECT_part.LP"))
os.remove(ihk)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
print >>decilog, "Pickling %s" % os.path.basename(f)
x = xds_ascii.XDS_ASCII(f, log_out=decilog)
if params.light_pickle: x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster
pickle.dump(x, open(f+".pkl", "w"), -1)
if params.pickle_hkl:
for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
print >>decilog, "Pickling %s" % os.path.basename(f)
inhkl = integrate_hkl_as_flex.reader(f, read_columns=["IOBS","SIGMA","XCAL","YCAL","RLP","PEAK","MAXC"])
pickle.dump(inhkl, open(f+".pkl", "w"), -1)
except ProcFailed, e:
print >>decilog, "Processing failed: %s" % e.message
def calc_stats(xac_file,
stat_choice,
n_residues=None,
ref_v6cell=None,
min_peak=None,
min_peak_percentile=None,
correct_peak=None):
# Open XDS_ASCII
if xac_file.endswith(".pkl"): xac = pickle.load(open(xac_file))
else: xac = xds_ascii.XDS_ASCII(xac_file)
sel_remove = flex.bool(xac.iobs.size(), False)
if min_peak is not None:
sel = xac.peak < min_peak
sel_remove |= sel
elif min_peak_percentile is not None:
q = numpy.percentile(xac.peak, min_peak_percentile)
print "percentile %.2f %s" % (q, xac)
sel = xac.peak < q
sel_remove |= sel
if correct_peak: sel_remove |= (xac.peak < 1) # remove PEAK==0
xac.remove_selection(sel_remove)
if params.correct_peak:
xac.iobs *= xac.peak * .01
xac.sigma_iobs *= xac.peak * .01
iobs = xac.i_obs(anomalous_flag=False)
iobs = iobs.select(iobs.sigmas() > 0).merge_equivalents(
use_internal_variance=False).array()
stats = dict(filename=xac_file, cell=iobs.unit_cell().parameters())
if iobs.size() == 0:
return stats
if "ioversigma" in stat_choice or "resnatsnr1" in stat_choice:
binner = iobs.setup_binner(auto_binning=True)
if "ioversigma" in stat_choice:
stats["ioversigma"] = flex.mean(iobs.data() / iobs.sigmas())
if "resnatsnr1" in stat_choice:
res = float("nan")
for i_bin in binner.range_used():
sel = binner.selection(i_bin)
tmp = iobs.select(sel)
if tmp.size() == 0: continue
sn = flex.mean(tmp.data() / tmp.sigmas())
if sn <= 1:
res = binner.bin_d_range(i_bin)[1]
break
stats["resnatsnr1"] = res
if "abdist" in stat_choice:
from cctbx.uctbx.determine_unit_cell import NCDist
G6a, G6b = ref_v6cell, v6cell(iobs.unit_cell().niggli_cell())
abdist = NCDist(G6a, G6b)
stats["abdist"] = abdist
if "wilsonb" in stat_choice:
iso_scale_and_b = ml_iso_absolute_scaling(iobs, n_residues, 0)
stats["wilsonb"] = iso_scale_and_b.b_wilson
print stats
return stats
def run(files):
assert len(files) == 2
hkl1 = xds_ascii.XDS_ASCII(files[0], sys.stdout)
hkl2 = xds_ascii.XDS_ASCII(files[1], sys.stdout)
hkl1_points = numpy.column_stack((hkl1.xd, hkl1.yd, hkl1.zd))
tree1 = spatial.cKDTree(hkl1_points)
n_ovl, n_nonovl = 0, 0
novl_indices, novl_i, novl_sigma = flex.miller_index(), flex.double(
), flex.double()
for i in xrange(len(hkl2.indices)):
x, y, z = hkl2.xd[i], hkl2.yd[i], hkl2.zd[i]
#if z > 180:
# continue
dists, idxs = tree1.query((x, y, z), k=3, p=1)
overlaps = []
for dist, idx in zip(dists, idxs):
idx = int(idx)
xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx]
if abs(z - zo) < 2.5 and (xo - x)**2 + (
yo - y)**2 < 15**2: # FIXME MAGIC NUMBER!
overlaps.append((dist, idx))
if len(overlaps) == 0:
novl_indices.append(hkl2.indices[i])
novl_i.append(hkl2.iobs[i])
novl_sigma.append(hkl2.sigma_iobs[i])
n_nonovl += 1
else:
print hkl2.indices[i], x, y, z
for dist, idx in overlaps:
xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx]
print hkl1.indices[idx], xo, yo, zo
print dist, idx
print
print
n_ref = len(hkl2.indices)
print "%.2f%% overlap!" % (100. * (n_ref - n_nonovl) / n_ref)
novl_array = miller.array(miller_set=miller.set(crystal_symmetry=hkl2.symm,
indices=novl_indices),
data=novl_i,
sigmas=novl_sigma)
stats = dataset_statistics(novl_array,
anomalous=False,
sigma_filtering="xds")
stats.show(out=sys.stdout)
novl_array = novl_array.customized_copy(anomalous_flag=False).map_to_asu()
novl_array = novl_array.eliminate_sys_absent()
novl_array = novl_array.select(novl_array.sigmas() >= 0)
filtr = filter_intensities_by_sigma(novl_array, "xds")
hklout = os.path.splitext(os.path.basename(files[1]))[0] + "_novl.mtz"
filtr.array_merged.set_observation_type_xray_intensity().as_mtz_dataset(
column_root_label="IMEAN").mtz_object().write(hklout)
def run(hklin):
xscaled = xds_ascii.XDS_ASCII(hklin) # Must be XSCALE output
merged_iobs = xscaled.i_obs().merge_equivalents(
use_internal_variance=False).array()
binner = merged_iobs.setup_binner(n_bins=100)
isets = set(xscaled.iset)
for_plot = {}
cut_ios = (2, 1, 0.5, 0)
print "iset file",
for cut in cut_ios:
print "cut_ios_%.2f" % cut,
print
for iset in isets:
sel = (xscaled.iset == iset)
data_i = xscaled.i_obs().select(sel).merge_equivalents(
use_internal_variance=False).array()
cutoffs = eval_resolution(data_i, 100, cut_ios)
print "%3d %s %s" % (iset, xscaled.input_files[iset][0], " ".join(
map(lambda x: "%.2f" % x, cutoffs)))
for i_bin in binner.range_used():
dmax, dmin = binner.bin_d_range(i_bin)
Isel = data_i.resolution_filter(d_max=dmax, d_min=dmin)
for_plot.setdefault(
iset,
[]).append(flex.mean(Isel.data()) if Isel.size() > 0 else 0)
import matplotlib
matplotlib.use('Agg') # Allow to work without X
import pylab
import math
from matplotlib.ticker import FuncFormatter
s2_formatter = lambda x, pos: "inf" if x == 0 else "%.2f" % (1. / numpy.
sqrt(x))
exp_formatter = lambda x, pos: "%.1e" % x
plot_x = [binner.bin_d_range(i)[1]**(-2) for i in binner.range_used()]
from matplotlib.backends.backend_pdf import PdfPages
pp = PdfPages("test.pdf")
keys = sorted(for_plot)
for names in (keys[i:i + 100] for i in xrange(0, len(keys), 100)):
ncols = 5
nrows = int(math.ceil(len(names) / float(ncols)))
fig, axes = pylab.plt.subplots(ncols=ncols,
nrows=nrows,
figsize=(5 * ncols, 5 * nrows),
sharex=False,
sharey=False)
axes = axes.flatten()
for name, ax in zip(names, axes):
ax.plot(
plot_x,
for_plot[name],
linewidth=1,
)
ax.axhline(y=0, color="red", linestyle="-")
ax.set_xlabel('(d^-2)')
ax.set_ylabel('<I>')
ax.xaxis.set_major_formatter(FuncFormatter(s2_formatter))
ax.yaxis.set_major_formatter(FuncFormatter(exp_formatter))
ax.set_title("data%.4d" % name)
ax.grid(True)
pylab.plt.tight_layout()
plot_title = ""
pylab.title(plot_title)
pp.savefig()
#pylab.savefig("test_%.3d.png"%i)
#pylab.show()
pp.close()