def run_cycles(self, xds_files):
self.removed_files = []
self.removed_reason = {}
print >> self.out, "********************* START FUNCTION ***********************"
# Remove unconnected files here.
remove_idxes = find_unconnected_xds_files(xds_files, min_ios=3, d_min=self.d_min)
print >> self.out, "DEBUG:: Need to remove unconnected %d files" % len(remove_idxes)
for i in sorted(remove_idxes):
print >> self.out, " %.3d %s" % (i + 1, xds_files[i])
self.removed_files.append(xds_files[i])
self.removed_reason[xds_files[i]] = "no_common_refls"
keep_idxes = filter(lambda x: x not in remove_idxes, xrange(len(xds_files)))
self.run_cycle(map(lambda i: xds_files[i], keep_idxes))
# Final cycle; average cells and merge again.
used_files = set(xds_files).difference(set(self.removed_files))
sg, cell = self.average_cells(used_files)
print >> self.out, "Final scaling with averaged cell."
print >> self.out, "Averaged cell= %s" % (cell)
# call mtzutils to change cell!
mtz_finalcell = "pointless_finalcell.mtz"
util.call(
cmd="mtzutils hklin %s hklout %s" % (os.path.relpath(self.mtzin, self.workdir), mtz_finalcell),
wdir=self.workdir,
expects_in=[os.path.relpath(self.mtzin, self.workdir)],
expects_out=[mtz_finalcell],
stdin="cell %s\nend\n" % cell,
stdout=open(os.path.join(self.workdir, "mtzutils.log"), "w"),
)
inp_str = ""
for i, f in enumerate(used_files):
brange = self.batch_info[f]
inp_str += "RUN %3d BATCH %4d to %4d\n" % (i + 1, brange[0], brange[1])
aimless.run_aimless(
mtzin=mtz_finalcell,
wdir=self.workdir,
anomalous=self.anomalous_flag,
d_min=self.d_min,
prefix="aimless",
add_stdin=inp_str,
)
ctruncate_arg = "-hklin aimless.mtz -hklout ctruncate.mtz -colin '/*/*/[IMEAN,SIGIMEAN]'"
if self.anomalous_flag:
ctruncate_arg += " -colano '/*/*/[I(+),SIGI(+),I(-),SIGI(-)]'"
util.call(
cmd="ctruncate %s" % ctruncate_arg,
wdir=self.workdir,
expects_in=["aimless.mtz"],
expects_out=["ctruncate.mtz"],
stdout=open(os.path.join(self.workdir, "ctruncate.log"), "w"),
)
return self.removed_files, self.removed_reason
def run_blend(wdir, xds_ascii_files, logout="blend_a.log"):
ofs_lst = open(os.path.join(wdir, "files.lst"), "w")
map(lambda x: ofs_lst.write(os.path.relpath(x, wdir)+"\n"), xds_ascii_files)
ofs_lst.close()
util.call(blend_comm, "-aDO files.lst", stdin="tolerance 10\n",
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "w"))
def run_blend(wdir, xds_ascii_files, logout="blend_a.log"):
ofs_lst = open(os.path.join(wdir, "files.lst"), "w")
map(lambda x: ofs_lst.write(os.path.relpath(x, wdir)+"\n"), xds_ascii_files)
ofs_lst.close()
util.call(blend_comm, "-aDO files.lst", stdin="tolerance 10\n",
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "w"))
def run(search_dir, pf_logfiles=None):
for img_template, min_frame, max_frame in dataset.find_data_sets(search_dir,skip_symlinks=False):
print "Dataset found:"
print " NAME_TEMPLATE_OF_DATA_FRAMES= %s" % img_template
print " DATA_RNAGE= %d %d" % (min_frame, max_frame)
pdir, ppath = os.path.split(img_template)
if not os.path.exists(os.path.join(pdir, "xds")):
os.mkdir(os.path.join(pdir, "xds"))
else:
assert os.path.isdir(os.path.join(pdir, "xds"))
prefix = ppath[:ppath.index("?")]
if not prefix.endswith("_"):
prefix += "_"
if pf_logfiles is None:
wdir = os.path.join(pdir, "xds", "xds_%s%d-%d" % (prefix, min_frame, max_frame))
if os.path.isfile(os.path.join(wdir, "XDS.INP")):
print " Already exist:", os.path.join(wdir, "XDS.INP")
continue
os.mkdir(wdir)
cmd = 'generate_XDS.INP "%s"' % os.path.relpath(img_template, wdir)
print " launching", cmd
call(cmd=cmd, wdir=wdir)
else:
for minf, maxf in split_runs_based_on_pf_logfiles(pf_logfiles):
if maxf < min_frame or max_frame < minf:
continue
if minf < min_frame:
minf = min_frame
if maxf > max_frame:
maxf = max_frame
wdir = os.path.join(pdir, "xds", "xds_%s%d-%d" % (prefix, minf, maxf))
if os.path.isfile(os.path.join(wdir, "XDS.INP")):
print "Already exist:", os.path.isfile(os.path.join(wdir, "XDS.INP"))
continue
os.mkdir(wdir)
cmd = 'generate_XDS.INP "%s"' % os.path.relpath(img_template, wdir)
print " launching", cmd
call(cmd=cmd, wdir=wdir)
# XXX dirty hacks..
inp = open(os.path.join(wdir, "XDS.INP")).readlines()
ofs = open(os.path.join(wdir, "XDS.INP"), "w")
for l in inp:
if l.startswith("DATA_RANGE="):
l = "DATA_RANGE= %d %d\n" % (minf, maxf)
if l.startswith("SPOT_RANGE="):
l = "SPOT_RANGE= %d %d\n" % (minf, (maxf-minf)/2+minf)
ofs.write(l)
ofs.close()
print
def run_blend0R(wdir, xds_ascii_files, logout="blend0.log"):
ofs_cell = open(os.path.join(wdir, "forR_macropar.dat"), "w")
ofs_lst = open(os.path.join(wdir, "xds_lookup_table.txt"), "w")
ofs_files = open(os.path.join(wdir, "NEW_list_of_files.dat"), "w") # Just to avoid error
for i, xac in enumerate(xds_ascii_files):
try:
symm = xds_ascii.XDS_ASCII(xac, read_data=False).symm
except:
symm = any_reflection_file(xac).as_miller_arrays()[0].crystal_symmetry()
cell = " ".join(map(lambda x: "%7.3f"%x, symm.unit_cell().parameters()))
sgnum = int(symm.space_group().type().number())
ofs_lst.write("dataset_%.3d %s\n" % (i, xac))
ofs_cell.write("%4d %s 0 0 0 \"%s\"\n" % (i+1, cell, sg_to_cent[sgnum]))
ofs_files.write("%s\n"%xac)
ofs_lst.close()
ofs_cell.close()
ofs_files.close()
shutil.copyfile(os.path.join(wdir, "forR_macropar.dat"),
os.path.join(wdir, "forR_macropar.dat.bak"))
pathblend = "modules/yamtbx/blend/R/blend0.R"
useNC = os.getenv("BLEND_USE_NCDIST","no")
useSF = os.getenv("BLEND_USE_SFDIST","no")
if useNC != "no":
pathblend = "modules/yamtbx/blend/R/blend0NC.R"
if useSF != "no":
pathblend = "modules/yamtbx/blend/R/blend0SF.R"
util.call("Rscript", os.path.join(os.environ["PHENIX"], pathblend),
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "w"))
open(os.path.join(wdir, "hctojson.R"), "w").write("""\
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
labels<-hc$labels
merge<-data.frame(hc$merge)
for (i in (1:nrow(merge))) {
if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
}
eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
return(JSON)
}
load("BLEND0.RData",.GlobalEnv)
JSON<-HCtoJSON(npar.hc_ward)
cat(JSON, file="dendro.json")
""")
util.call("Rscript", "hctojson.R",
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "a"))
def run_xscale(xscale_inp):
ftable = {}
outfile = None
count = 0
wdir = os.path.dirname(os.path.abspath(xscale_inp))
os.rename(xscale_inp, xscale_inp+".org")
ofs = open(xscale_inp, "w")
# Check line length and make symlink if needed
for l in open(xscale_inp+".org"):
ll = l[:l.index("!")] if "!" in l else l
if "OUTPUT_FILE=" in ll:
outfile = ll[ll.index("=")+1:].strip()
if "INPUT_FILE=" in ll and len(l) > 132: # one line is limited to 131 characters!
filename = ll[ll.index("=")+1:].strip()
if "*" in filename: filename = filename[filename.index("*")+1:].strip()
assert " " not in filename
lnkf = "lnk%.6d.hkl" % count
assert not os.path.isfile(os.path.join(wdir, lnkf))
filename_abs = os.path.normpath(os.path.join(wdir, filename)) if not os.path.isabs(filename) else filename
os.symlink(filename_abs, os.path.join(wdir, lnkf))
print "xscale: %s -> %s" % (lnkf, filename)
count += 1
ftable[lnkf] = filename
l = l.replace(filename, lnkf)
ofs.write(l)
ofs.close()
assert outfile is not None
if len(ftable) == 0:
os.rename(xscale_inp+".org", xscale_inp)
# Run xscale
util.call(xscale_comm, wdir=wdir)
# Replace file names if needed
if len(ftable) > 0:
for f in "XSCALE.LP", outfile:
f = os.path.join(wdir, f)
if not os.path.isfile(f): continue
os.rename(f, f+".org")
ofs = open(f, "w")
for l in open(f+".org"):
for lfn in ftable: l = l.replace(lfn, ftable[lfn])
ofs.write(l)
ofs.close()
for lfn in ftable:
os.remove(os.path.join(wdir, lfn))
os.rename(xscale_inp+".org", xscale_inp)
def work(rootdir, xdsinp, orgxy):
workdir = os.path.join(rootdir, "bs_x%+.1f_y%+.1f" % orgxy)
inpdir = os.path.normpath(os.path.dirname(xdsinp))
print workdir
os.makedirs(workdir)
shutil.copyfile(os.path.join(inpdir, "SPOT.XDS"), os.path.join(workdir, "SPOT.XDS"))
shutil.copyfile(xdsinp, os.path.join(workdir, "XDS.INP"))
modify_xdsinp(os.path.join(workdir, "XDS.INP"), inp_params=[("JOB", "IDXREF"),
("ORGX", orgxy[0]),
("ORGY", orgxy[1]),
])
call("xds", wdir=workdir)
def run_blend0R(wdir, xds_ascii_files, logout="blend0.log"):
ofs_cell = open(os.path.join(wdir, "forR_macropar.dat"), "w")
ofs_lst = open(os.path.join(wdir, "xds_lookup_table.txt"), "w")
ofs_files = open(os.path.join(wdir, "NEW_list_of_files.dat"),
"w") # Just to avoid error
for i, xac in enumerate(xds_ascii_files):
symm = xds_ascii.XDS_ASCII(xac, read_data=False).symm
cell = " ".join(
map(lambda x: "%7.3f" % x,
symm.unit_cell().parameters()))
ofs_lst.write("dataset_%.3d %s\n" % (i, xac))
ofs_cell.write("%4d %s 0 0 0\n" % (i + 1, cell))
ofs_files.write("%s\n" % xac)
ofs_lst.close()
ofs_cell.close()
ofs_files.close()
shutil.copyfile(os.path.join(wdir, "forR_macropar.dat"),
os.path.join(wdir, "forR_macropar.dat.bak"))
util.call("Rscript",
os.path.join(os.environ["CCP4"], "share/blend/R/blend0.R"),
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "w"))
open(os.path.join(wdir, "hctojson.R"), "w").write("""\
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
labels<-hc$labels
merge<-data.frame(hc$merge)
for (i in (1:nrow(merge))) {
if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
}
eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
return(JSON)
}
load("BLEND0.RData",.GlobalEnv)
JSON<-HCtoJSON(npar.hc_ward)
cat(JSON, file="dendro.json")
""")
util.call("Rscript",
"hctojson.R",
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "a"))
def run_xtriage_in_module_if_possible(args, wdir):
try:
from mmtbx.scaling import xtriage
cwd_org = os.getcwd()
try:
os.chdir(wdir)
xtriage.run(args, command_name="mmtbx.xtriage")
except:
print traceback.format_exc()
finally:
os.chdir(cwd_org)
except ImportError:
call("phenix.xtriage", arg=" ".join(args),
stdin=None, stdout=sys.stdout, wdir=wdir)
def tst_h5toxds():
print "Testing H5ToXds.."
rcode, out, err = util.call("H5ToXds")
ignore_msg = "(You can ignore this if you don't process hdf5 files which usually mean Eiger data)"
if rcode == 127: # 127 is "command not found".
print " Not installed. NG %s" % ignore_msg
return False
import numpy
from yamtbx.dataproc.eiger import make_dummy_h5_for_test
from yamtbx.dataproc import cbf
tmpdir = util.get_temp_local_dir("h5test")
data = numpy.random.randint(0, 65535,
size=100).astype(numpy.uint32).reshape(
(1, 10, 10))
master_h5 = make_dummy_h5_for_test(tmpdir, data)
rcode, out, err = util.call("H5ToXds",
"%s 1 1.cbf" % os.path.basename(master_h5),
wdir=tmpdir)
cbfout = os.path.join(tmpdir, "1.cbf")
if not os.path.isfile(cbfout):
print " H5ToXds exists, but not works. Probably Dectris original H5ToXds? Test it with real h5 file. %s" % ignore_msg
if out.strip():
print " -- stdout:"
print out
if err.strip():
print " -- stderr:"
print err
shutil.rmtree(tmpdir)
return False
data_read, _, _ = cbf.load_minicbf_as_numpy(cbfout)
shutil.rmtree(tmpdir)
if numpy.all(data_read.flatten() == data.flatten()):
print " OK"
return True
else:
print " H5ToXds exists, but not correctly works. Probably Dectris original H5ToXds? Test it with real h5 file. %s" % ignore_msg
if out.strip():
print " -- stdout:"
print out
if err.strip():
print " -- stderr:"
print err
return False
def work(rootdir, xdsinp, orgxy):
workdir = os.path.join(rootdir, "bs_x%+.1f_y%+.1f" % orgxy)
inpdir = os.path.normpath(os.path.dirname(xdsinp))
print workdir
os.makedirs(workdir)
shutil.copyfile(os.path.join(inpdir, "SPOT.XDS"),
os.path.join(workdir, "SPOT.XDS"))
shutil.copyfile(xdsinp, os.path.join(workdir, "XDS.INP"))
modify_xdsinp(os.path.join(workdir, "XDS.INP"),
inp_params=[
("JOB", "IDXREF"),
("ORGX", orgxy[0]),
("ORGY", orgxy[1]),
])
call("xds", wdir=workdir)
def tst_R():
print "Testing R.."
rcode, out, err = util.call("Rscript", '-e "print(cos(pi))"')
if rcode != 0 or out.strip() != '[1] -1':
print " Rscript is not avaiable. NG"
return False
rcode, out, err = util.call("Rscript", '-e "library(rjson)"')
if rcode != 0:
print " rjson is not installed. NG"
return False
print " OK"
return True
def tst_R():
print "Testing R.."
rcode, out, err = util.call("Rscript", '-e "print(cos(pi))"')
if rcode != 0 or out.strip() != '[1] -1':
print " Rscript is not avaiable. NG"
return False
rcode, out, err = util.call("Rscript", '-e "library(rjson)"')
if rcode != 0:
print " rjson is not installed. NG"
return False
print " OK"
return True
def run_aimless(prefix, wdir):
script_body = """\
pref=%s
mtzin=../${pref}_forscale.mtz
scaled=${pref}_aimless.mtz
truncated=${pref}_aimless_truncate.mtz
#lowres=$2
#highres=$3
aimless \
HKLIN $mtzin \
HKLOUT $scaled \
SCALES ${pref}.scala \
ROGUES ${pref}_scala_rogues.log \
NORMPLOT ${pref}_scala_normplot.xmgr \
ANOMPLOT ${pref}_scala_anomplot.xmgr \
PLOT ${pref}_scala_surface_plot.plt \
CORRELPLOT ${pref}_scala_correlplot.xmgr \
ROGUEPLOT ${pref}_scala_rogueplot.xmgr \
<<eoi
!resolution low $lowres high $highres
!exclude batch ** to **
bins 20
!run 1 batch 1 to 750
!scales rotation spacing 5.000000 bfactor on tails
!cycles 5
""" % (prefix)
if anomalous:
script_body += """\
anomalous on
eoi
ctruncate -hklin $scaled -hklout $truncated -colin "/*/*/[IMEAN,SIGIMEAN]" -colano "/*/*/[I(+),SIGI(+),I(-),SIGI(-)]"
"""
else:
script_body += """\
anomalous off
eoi
ctruncate -hklin $scaled -hklout $truncated -colin "/*/*/[IMEAN,SIGIMEAN]"
"""
open(os.path.join(wdir, "run_aimless.sh"),"w").write(script_body)
call("sh run_aimless.sh",
stdout=open(os.path.join(wdir, "aimless_truncate.log"), "w"),
wdir=wdir)
def run_aimless(prefix, wdir):
script_body = """\
pref=%s
mtzin=../${pref}_forscale.mtz
scaled=${pref}_aimless.mtz
truncated=${pref}_aimless_truncate.mtz
#lowres=$2
#highres=$3
aimless \
HKLIN $mtzin \
HKLOUT $scaled \
SCALES ${pref}.scala \
ROGUES ${pref}_scala_rogues.log \
NORMPLOT ${pref}_scala_normplot.xmgr \
ANOMPLOT ${pref}_scala_anomplot.xmgr \
PLOT ${pref}_scala_surface_plot.plt \
CORRELPLOT ${pref}_scala_correlplot.xmgr \
ROGUEPLOT ${pref}_scala_rogueplot.xmgr \
<<eoi
!resolution low $lowres high $highres
!exclude batch ** to **
bins 20
!run 1 batch 1 to 750
!scales rotation spacing 5.000000 bfactor on tails
!cycles 5
""" % (prefix)
if anomalous:
script_body += """\
anomalous on
eoi
ctruncate -hklin $scaled -hklout $truncated -colin "/*/*/[IMEAN,SIGIMEAN]" -colano "/*/*/[I(+),SIGI(+),I(-),SIGI(-)]"
"""
else:
script_body += """\
anomalous off
eoi
ctruncate -hklin $scaled -hklout $truncated -colin "/*/*/[IMEAN,SIGIMEAN]"
"""
open(os.path.join(wdir, "run_aimless.sh"), "w").write(script_body)
call("sh run_aimless.sh",
stdout=open(os.path.join(wdir, "aimless_truncate.log"), "w"),
wdir=wdir)
def run(params, xac_files):
if len(xac_files) == 0:
print "No XDS_ASCII.HKL files provided."
return
xscale_inp_head = "!MINIMUM_I/SIGMA= 3\n\n"
infos = {}
d_max, d_min = 0, 100
cells = []
for xds_ascii in xac_files:
info = get_xac_info(xds_ascii)
infos[xds_ascii] = info
resrng = map(float, info["resol_range"].split())
d_max = max(d_max, resrng[0])
d_min = min(d_min, resrng[1])
cells.append(map(float, info["cell"].split()))
if params.d_min is not None:
d_min = max(params.d_min, d_min)
if params.cell == "average":
cell_sum = reduce(lambda x,y: map(lambda a: a[0]+a[1], zip(x,y)), cells)
cell_mean = map(lambda x: x/float(len(cells)), cell_sum)
if params.sgnum is not None: sgnum = str(params.sgnum)
else: sgnum = infos[xac_files[0]]["spgr_num"]
xscale_inp_head += " SPACE_GROUP_NUMBER= %s\n" % sgnum
xscale_inp_head += " UNIT_CELL_CONSTANTS= %s\n" % " ".join(map(lambda x: "%.3f"%x, cell_mean))
#if anomalous_flag is not None:
# xscale_inp_head += " FRIEDEL'S_LAW= %s\n" % ("FALSE" if anomalous_flag else "TRUE")
xscale_inp_head += make_shells(d_max, d_min, params.nbins) + "\n"
xscale_inp_head += " OUTPUT_FILE= %s\n\n" % params.output
xscale_inp = os.path.join(params.workdir, "XSCALE.INP")
inp_out = open(xscale_inp, "w")
inp_out.write(xscale_inp_head)
for xds_ascii in xac_files:
inp_out.write(" INPUT_FILE= %s\n" % os.path.relpath(xds_ascii, params.workdir))
inp_out.write(" ! INCLUDE_RESOLUTION_RANGE= %s\n\n" % infos[xds_ascii]["resol_range"])
inp_out.close()
util.call(xscale_comm, wdir=params.workdir, stdout=sys.stdout)
def btnUnmount_click(self, ev):
def df(d):
ret, out, err = util.call("df", '"%s"'%d) # XXX Linux specific?
out_lines = filter(lambda x: x, map(lambda x: x.strip(), out.splitlines()))
if len(out_lines) < 2: return None
sp = out_lines[1].split()
if not sp: return None
return sp[0]
ddir = self.txtDestDir.GetValue()
if not ddir or not os.path.isabs(ddir) or not os.path.isdir(ddir):
wx.MessageDialog(None, "Invalid directory.", "Error", style=wx.OK).ShowModal()
return
mounted_dev = df(ddir)
if not mounted_dev:
wx.MessageDialog(None, "Invalid directory.", "Error", style=wx.OK).ShowModal()
return
if wx.MessageDialog(None, "Unmounting %s. Are you sure?" % mounted_dev,
"Confirmation", style=wx.OK|wx.CANCEL).ShowModal() != wx.ID_OK:
return
ret, out, err = util.call("udisks", "--unmount %s" % mounted_dev) # XXX Linux specific?
if out:
wx.MessageDialog(None, out, "Error", style=wx.OK).ShowModal()
elif mounted_dev == df(ddir):
wx.MessageDialog(None, "Unmount failed.", "Error", style=wx.OK).ShowModal()
else:
wx.MessageDialog(None, "Successfully unmounted.", "OK", style=wx.OK).ShowModal()
def run_aimless(mtzin, wdir, anomalous=False, d_min=None, prefix=None, add_stdin=None):
if prefix is None:
prefix = os.path.splitext(os.path.basename(mtzin))[0] + "_aimless"
args = """\
HKLIN %(mtzin)s \
HKLOUT %(prefix)s.mtz \
SCALES %(prefix)s.scala \
ROGUES %(prefix)s_rogues.log \
NORMPLOT %(prefix)s_normplot.xmgr \
ANOMPLOT %(prefix)s_anomplot.xmgr \
PLOT %(prefix)s_surface_plot.plt \
CORRELPLOT %(prefix)s_correlplot.xmgr \
ROGUEPLOT %(prefix)s_rogueplot.xmgr \
""" % dict(mtzin=mtzin, prefix=prefix)
stdin = ""
if d_min is not None:
stdin += "resolution high %.3f\n" % d_min
if anomalous:
stdin += "anomalous on\n"
if add_stdin is not None:
stdin += add_stdin
exitcode, output, err = util.call(aimless_comm, args,
stdin=stdin, wdir=wdir)
open(os.path.join(wdir, prefix+".log"), "w").write(output)
def df(d):
ret, out, err = util.call("df", '"%s"'%d) # XXX Linux specific?
out_lines = filter(lambda x: x, map(lambda x: x.strip(), out.splitlines()))
if len(out_lines) < 2: return None
sp = out_lines[1].split()
if not sp: return None
return sp[0]
def run_xtriage_in_module_if_possible(args, wdir):
try:
from mmtbx.scaling import xtriage
cwd_org = os.getcwd()
try:
os.chdir(wdir)
xtriage.run(args, command_name="mmtbx.xtriage")
except:
print traceback.format_exc()
finally:
os.chdir(cwd_org)
except ImportError:
call("phenix.xtriage",
arg=" ".join(args),
stdin=None,
stdout=sys.stdout,
wdir=wdir)
def run_blend0R(wdir, xds_ascii_files, logout="blend0.log"):
ofs_cell = open(os.path.join(wdir, "forR_macropar.dat"), "w")
ofs_lst = open(os.path.join(wdir, "xds_lookup_table.txt"), "w")
ofs_files = open(os.path.join(wdir, "NEW_list_of_files.dat"), "w") # Just to avoid error
for i, xac in enumerate(xds_ascii_files):
symm = xds_ascii.XDS_ASCII(xac, read_data=False).symm
cell = " ".join(map(lambda x: "%7.3f"%x, symm.unit_cell().parameters()))
ofs_lst.write("dataset_%.3d %s\n" % (i, xac))
ofs_cell.write("%4d %s 0 0 0\n" % (i+1, cell))
ofs_files.write("%s\n"%xac)
ofs_lst.close()
ofs_cell.close()
ofs_files.close()
shutil.copyfile(os.path.join(wdir, "forR_macropar.dat"),
os.path.join(wdir, "forR_macropar.dat.bak"))
util.call("Rscript", os.path.join(os.environ["CCP4"], "share/blend/R/blend0.R"),
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "w"))
open(os.path.join(wdir, "hctojson.R"), "w").write("""\
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
labels<-hc$labels
merge<-data.frame(hc$merge)
for (i in (1:nrow(merge))) {
if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
}
eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
return(JSON)
}
load("BLEND0.RData",.GlobalEnv)
JSON<-HCtoJSON(npar.hc_ward)
cat(JSON, file="dendro.json")
""")
util.call("Rscript", "hctojson.R",
wdir=wdir,
stdout=open(os.path.join(wdir, logout), "a"))
def calc_cchalf_by_removing(wdir,
inp_head,
inpfiles,
with_sigma=False,
stat_bin="total",
nproc=1,
nproc_each=None,
batchjobs=None):
assert not with_sigma # Not supported now
assert stat_bin in ("total", "outer")
if not os.path.exists(wdir): os.makedirs(wdir)
datout = open(os.path.join(wdir, "cchalf.dat"), "w")
datout.write("idx exfile cc1/2(%s) Nuniq\n" % stat_bin)
cchalf_list = [] # (i_ex, CC1/2, Nuniq)
# Prep runs
tmpdirs = map(
lambda x: _calc_cchalf_by_removing_worker_1(wdir, inp_head, inpfiles,
x, nproc_each),
xrange(len(inpfiles)))
# Run XSCALE
if batchjobs is not None:
jobs = []
for tmpdir in tmpdirs:
job = batchjob.Job(tmpdir, "xscale.sh", nproc=nproc_each)
job.write_script(xscale_comm)
batchjobs.submit(job)
jobs.append(job)
batchjobs.wait_all(jobs)
else:
easy_mp.pool_map(fixed_func=lambda x: util.call(xscale_comm, wdir=x),
args=tmpdirs,
processes=nproc)
# Finish runs
cchalf_list = map(
lambda x: _calc_cchalf_by_removing_worker_2(wdir, x[1], x[0], stat_bin
), enumerate(tmpdirs))
for iex, cchalf_exi, nuniq in cchalf_list:
datout.write("%3d %s %.4f %d\n" %
(iex, inpfiles[iex], cchalf_exi, nuniq))
cchalf_list.sort(key=lambda x: -x[1])
print
print "# Sorted table"
for idx, cch, nuniq in cchalf_list:
print "%3d %-.4f %4d %s" % (idx, cch, nuniq, inpfiles[idx])
# Remove unuseful (failed) data
cchalf_list = filter(lambda x: x[1] == x[1], cchalf_list)
return cchalf_list
def tst_adxv():
print "Testing Adxv.."
rcode, out, err = util.call("adxv", '-help')
if rcode != 0:
print " Adxv is not avaiable. NG"
return False
print " OK"
return True
def tst_adxv():
print "Testing Adxv.."
rcode, out, err = util.call("adxv", '-help')
if rcode != 0:
print " Adxv is not avaiable. NG"
return False
print " OK"
return True
def run(params):
xdsinp = "XDS.INP"
workdir = os.getcwd()
params.wfac1.sort()
params.minpk.sort()
digit_wfac1 = str(get_digit(params.wfac1))
digit_minpk = str(get_digit(params.minpk))
backup_needed = files.generated_by_CORRECT + ("XDS.INP", )
bk_prefix = make_backup(backup_needed)
try:
for wfac1 in params.wfac1:
for minpk in params.minpk:
work_name = ("CORRECT_wfac1_%." + digit_wfac1 + "f_minpk_%." +
digit_minpk + "f") % (wfac1, minpk)
modify_xdsinp(xdsinp,
inp_params=[
("JOB", "CORRECT"),
("WFAC1", wfac1),
("MINPK", minpk),
])
call("xds")
# make .mtz file
xds2mtz.xds2mtz(os.path.abspath("XDS_ASCII.HKL"),
dir_name=os.path.join(workdir,
"ccp4_" + work_name),
run_ctruncate=True,
run_xtriage=True)
if params.mtz_free is not None:
copy_testflag(mtzfree=params.mtz_free,
mtzin=os.path.join(workdir,
"ccp4_" + work_name,
"XDS_ASCII.mtz"))
make_backup(backup_needed, work_name + "_")
finally:
# 6. Revert XDS.INP, etc.
revert_files(backup_needed, bk_prefix)
def run(hklin):
arrays = filter(lambda x:x.is_xray_intensity_array(),
iotbx.file_reader.any_file(hklin).file_server.miller_arrays)
# TODO Copy non-intensity arrays to new mtz!
if len(arrays) == 0:
print "No intensity array"
return
print "Intensity arrays:"
for ar in arrays:
print "", ar.info().label_string()
print
colin, colano = "", ""
if len(arrays) == 1:
coltmp = arrays[0].info().label_string()
if arrays[0].anomalous_flag():
colano = "/*/*/[%s]" % coltmp.replace(",merged","")
else:
colin = "/*/*/[%s]" % coltmp
elif len(arrays) == 2 and sum(map(lambda x: x.anomalous_flag(), arrays)) == 1:
tmpano = filter(lambda x: x.anomalous_flag(), arrays)[0]
tmpnoano = filter(lambda x: not x.anomalous_flag(), arrays)[0]
colano = "/*/*/[%s]" % tmpano.info().label_string().replace(",merged","")
colin = "/*/*/[%s]" % tmpnoano.info().label_string()
else:
print "What should I do.. exiting."
return
hklout = os.path.splitext(os.path.basename(hklin))[0] + "_ctruncate.mtz"
cmd = "ctruncate -hklin %s -hklout %s " % (hklin, hklout)
if colin != "":
cmd += "-colin '%s' " % colin
if colano != "":
cmd += "-colano '%s' " % colano
call(cmd=cmd,
stdout=open("ctruncate_%s.log" % os.path.splitext(os.path.basename(hklin))[0], "w")
)
def tst_dials():
print "Testing dials (package).."
rcode, out, err = util.call("dials.version")
if rcode != 0:
print " Not installed. NG"
return False
print "\n".join(map(lambda x: " " + x, out.splitlines()))
print " OK"
return True
def tst_xdsstat():
print "Testing XDSSTAT.."
rcode, out, err = util.call("xdsstat")
if rcode != 2:
print " Not installed. NG"
return False
if "XDSSTAT version" not in err:
print " Seems not working. NG"
return False
print " OK"
return True
def run_copy(self, hklout, wdir, hklin=None, xdsin=None, logout=None, tolerance=None):
assert (hklin, xdsin).count(None) == 1
stdin, junk = self.input_str(hklin, xdsin)
if tolerance is not None:
stdin += "tolerance %f\n" % tolerance
exitcode, output, err = util.call(pointless_comm, "-copy hklout %s" % hklout,
stdin=stdin, wdir=wdir)
if logout is not None:
open(logout, "w").write(output)
return parse_pointless_output_for_runs(output)
def tst_xds():
print "Testing XDS.."
rcode, out, err = util.call("xds_par")
if rcode != 0:
print " Not installed. NG"
return False
if "license expired" in out:
print " license expired. Get latest version. NG"
return False
print " OK"
return True
def tst_xdsstat():
print "Testing XDSSTAT.."
rcode, out, err = util.call("xdsstat")
if rcode != 2:
print " Not installed. NG"
return False
if "XDSSTAT version" not in err:
print " Seems not working. NG"
return False
print " OK"
return True
def run(hklin):
arrays = filter(lambda x: x.is_xray_intensity_array(), iotbx.file_reader.any_file(hklin).file_server.miller_arrays)
# TODO Copy non-intensity arrays to new mtz!
if len(arrays) == 0:
print "No intensity array"
return
print "Intensity arrays:"
for ar in arrays:
print "", ar.info().label_string()
print
colin, colano = "", ""
if len(arrays) == 1:
coltmp = arrays[0].info().label_string()
if arrays[0].anomalous_flag():
colano = "/*/*/[%s]" % coltmp.replace(",merged", "")
else:
colin = "/*/*/[%s]" % coltmp
elif len(arrays) == 2 and sum(map(lambda x: x.anomalous_flag(), arrays)) == 1:
tmpano = filter(lambda x: x.anomalous_flag(), arrays)[0]
tmpnoano = filter(lambda x: not x.anomalous_flag(), arrays)[0]
colano = "/*/*/[%s]" % tmpano.info().label_string().replace(",merged", "")
colin = "/*/*/[%s]" % tmpnoano.info().label_string()
else:
print "What should I do.. exiting."
return
hklout = os.path.splitext(os.path.basename(hklin))[0] + "_ctruncate.mtz"
cmd = "ctruncate -hklin %s -hklout %s " % (hklin, hklout)
if colin != "":
cmd += "-colin '%s' " % colin
if colano != "":
cmd += "-colano '%s' " % colano
call(cmd=cmd, stdout=open("ctruncate_%s.log" % os.path.splitext(os.path.basename(hklin))[0], "w"))
def tst_xds():
print "Testing XDS.."
rcode, out, err = util.call("xds_par")
if rcode != 0:
print " Not installed. NG"
return False
if "license expired" in out:
print " license expired. Get latest version. NG"
return False
print " OK"
return True
def run(params):
xdsinp = "XDS.INP"
kwds = dict(get_xdsinp_keyword(xdsinp))
orgx_org, orgy_org = map(float, (kwds["ORGX"], kwds["ORGY"]))
dx, dy = params.dx, params.dy
if params.unit == "mm":
assert "QX" in kwds
assert "QY" in kwds
dx /= float(kwds["QX"])
dy /= float(kwds["QY"])
backup_needed = files.generated_by_IDXREF + ("XDS.INP",)
bk_prefix = make_backup(backup_needed)
try:
results = []
for i in xrange(-params.nx, params.nx+1):
for j in xrange(-params.ny, params.ny+1):
work_name = "bs_x%+.2d_y%+.2d" % (i, j)
orgx = orgx_org + i * dx
orgy = orgy_org + j * dy
print "Trying", orgx, orgy
modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
("ORGX", orgx),
("ORGY", orgy),
])
call("xds")
make_backup(backup_needed, work_name+"_")
results.append([work_name, orgx, orgy])
for ret in results:
print ret,
analyze_result(ret[0]+"_IDXREF.LP")
finally:
revert_files(backup_needed, bk_prefix)
def run_copy(self, hklout, wdir, hklin=None, xdsin=None, logout=None, tolerance=None):
assert (hklin, xdsin).count(None) == 1
stdin, junk = self.input_str(hklin, xdsin)
if tolerance is not None:
stdin += "tolerance %f\n" % tolerance
exitcode, output, err = util.call(pointless_comm, "-copy hklout %s" % hklout,
stdin=stdin, wdir=wdir)
if logout is not None:
open(logout, "w").write(output)
return parse_pointless_output_for_runs(output)
def run(params):
xdsinp = "XDS.INP"
workdir = os.getcwd()
params.wfac1.sort()
params.minpk.sort()
digit_wfac1 = str(get_digit(params.wfac1))
digit_minpk = str(get_digit(params.minpk))
backup_needed = files.generated_by_CORRECT + ("XDS.INP",)
bk_prefix = make_backup(backup_needed)
try:
for wfac1 in params.wfac1:
for minpk in params.minpk:
work_name = ("CORRECT_wfac1_%." + digit_wfac1 + "f_minpk_%." + digit_minpk + "f") % (wfac1, minpk)
modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"), ("WFAC1", wfac1), ("MINPK", minpk)])
call("xds")
# make .mtz file
xds2mtz.xds2mtz(
os.path.abspath("XDS_ASCII.HKL"),
dir_name=os.path.join(workdir, "ccp4_" + work_name),
run_ctruncate=True,
run_xtriage=True,
)
if params.mtz_free is not None:
copy_testflag(
mtzfree=params.mtz_free, mtzin=os.path.join(workdir, "ccp4_" + work_name, "XDS_ASCII.mtz")
)
make_backup(backup_needed, work_name + "_")
finally:
# 6. Revert XDS.INP, etc.
revert_files(backup_needed, bk_prefix)
def run_for_symm(self, hklin=None, xdsin=None, logout=None, tolerance=None, d_min=None, choose_laue=None, xdsin_to_p1=False, setting="c2"):
assert (hklin, xdsin).count(None) == 1
assert setting.lower().startswith(("c2", "cell", "symm"))
tmpfile = None
out = open(logout, "w") if logout else null_out()
# Merge in P1 before pointless for quick evaluation
if xdsin and xdsin_to_p1:
out.write("Merging in P1 symmetry before Pointless..\n")
xac = xds_ascii.XDS_ASCII(xdsin)
xac.remove_rejected()
i_obs = xac.i_obs(anomalous_flag=True)
i_obs.show_summary(out, " ")
i_obs = i_obs.apply_change_of_basis(change_of_basis="to_niggli_cell", out=out)[0]
i_obs = i_obs.customized_copy(space_group_info=sgtbx.space_group_info("P1"))
i_obs = i_obs.merge_equivalents(use_internal_variance=False).array()
tmpfile = util.get_temp_filename(prefix="", suffix=os.path.basename(xdsin) + "_p1.mtz")
i_obs.as_mtz_dataset(column_root_label="I").mtz_object().write(tmpfile)
xdsin, hklin = None, tmpfile
out.write("\n")
filein, wdir = self.input_str(hklin, xdsin)
# Need centre keyword?
stdin = """\
SETTING %s
%s
""" % (setting, filein)
if tolerance is not None:
stdin += "tolerance %f\n" % tolerance
if d_min is not None:
stdin += "resolution high %f\n" % d_min
if choose_laue is not None:
stdin += "choose lauegroup %s\n" % choose_laue
exitcode, output, err = util.call(pointless_comm,
stdin=stdin, wdir=wdir)
parsed = parse_pointless_output_for_symm(output)
out.write(output)
if tmpfile and os.path.isfile(tmpfile):
os.remove(tmpfile)
return parsed
def tst_xds():
print "Testing XDS.."
tmpdir = util.get_temp_local_dir("xdstest")
rcode, out, err = util.call("xds_par", wdir=tmpdir)
if tmpdir:
shutil.rmtree(tmpdir) # just in case; xds shouldn't make any files
if rcode != 0:
print " Not installed. NG"
return False
if "license expired" in out:
print " license expired. Get latest version. NG"
return False
print " OK"
return True
def check_xds_version():
tmpdir = util.get_temp_local_dir("xdstest")
rcode, out, err = util.call("xds", wdir=tmpdir)
if tmpdir:
shutil.rmtree(tmpdir) # just in case; xds shouldn't make any files
# Expected outout:
# ***** XDS ***** (VERSION Mar 15, 2019 BUILT=20191211) 6-Jan-2020
# Author: Wolfgang Kabsch
# Copy licensed until 30-Sep-2020 to
r = re.search("VERSION (.*[0-9]) *BUILT=(.*)\)", out)
if r:
ver, built = r.groups()
return ver, built
return None, None
def calc_cchalf_by_removing(wdir, inp_head, inpfiles, with_sigma=False, stat_bin="total", nproc=1, nproc_each=None, batchjobs=None):
assert not with_sigma # Not supported now
assert stat_bin in ("total", "outer")
if not os.path.exists(wdir): os.makedirs(wdir)
datout = open(os.path.join(wdir, "cchalf.dat"), "w")
datout.write("idx exfile cc1/2(%s) Nuniq\n" % stat_bin)
cchalf_list = [] # (i_ex, CC1/2, Nuniq)
# Prep runs
tmpdirs = map(lambda x: _calc_cchalf_by_removing_worker_1(wdir, inp_head, inpfiles, x, nproc_each),
xrange(len(inpfiles)))
# Run XSCALE
if batchjobs is not None:
jobs = []
for tmpdir in tmpdirs:
job = batchjob.Job(tmpdir, "xscale.sh", nproc=nproc_each)
job.write_script(xscale_comm)
batchjobs.submit(job)
jobs.append(job)
batchjobs.wait_all(jobs)
else:
easy_mp.pool_map(fixed_func=lambda x: util.call(xscale_comm, wdir=x),
args=tmpdirs,
processes=nproc)
# Finish runs
cchalf_list = map(lambda x: _calc_cchalf_by_removing_worker_2(wdir, x[1], x[0], stat_bin), enumerate(tmpdirs))
for iex, cchalf_exi, nuniq in cchalf_list:
datout.write("%3d %s %.4f %d\n" % (iex, inpfiles[iex], cchalf_exi, nuniq))
cchalf_list.sort(key=lambda x: -x[1])
print
print "# Sorted table"
for idx, cch, nuniq in cchalf_list:
print "%3d %-.4f %4d %s" % (idx, cch, nuniq, inpfiles[idx])
# Remove unuseful (failed) data
cchalf_list = filter(lambda x: x[1]==x[1], cchalf_list)
return cchalf_list
def unique(mtzin, mtzout, sg, wdir, logout):
##
# unique -> cad -> mtzutil (EXCLUDE FUNI SIGFUNI)
#
m = mtzutil.MtzFile(os.path.join(wdir, mtzin))
cell = m.get_cell_str()
sg_org = m.get_spacegroup()[1]
resol = min(m.get_resolution())
call(
cmd="unique",
arg="hklout unique.mtz",
wdir=wdir,
stdin=
"CELL %s\nSYMMETRY '%s'\nLABOUT F=FUNI SIGF=SIGFUNI\nRESOLUTION %f" %
(cell, sg_org, resol),
expects_in=[],
expects_out=["unique.mtz"],
stdout=logout,
)
call(
cmd="cad",
arg="hklin1 %s hklin2 %s hklout %s" %
(mtzin, "unique.mtz", "unique_cad.mtz"),
wdir=wdir,
stdin="labin file 1 all\nlabin file 2 all\nend\n",
expects_in=[mtzin, "unique.mtz"],
expects_out=["unique_cad.mtz"],
stdout=logout,
)
call(
cmd="mtzutils",
arg="hklin %s hklout %s" % ("unique_cad.mtz", mtzout),
wdir=wdir,
stdin="EXCLUDE FUNI SIGFUNI\nSYMMETRY %s\nRUN\n" % sg,
expects_in=["unique_cad.mtz"],
expects_out=[mtzout],
stdout=logout,
)
os.remove(os.path.join(wdir, "unique.mtz"))
os.remove(os.path.join(wdir, "unique_cad.mtz"))
def run_for_symm(self, hklin=None, xdsin=None, logout=None, tolerance=None, d_min=None):
assert (hklin, xdsin).count(None) == 1
filein, wdir = self.input_str(hklin, xdsin)
# Need centre keyword?
stdin = """\
SETTING SYMMETRY-BASED
%s
""" % filein
if tolerance is not None:
stdin += "tolerance %f\n" % tolerance
if d_min is not None:
stdin += "resolution high %f\n" % d_min
exitcode, output, err = util.call(pointless_comm,
stdin=stdin, wdir=wdir)
parsed = parse_pointless_output_for_symm(output)
if logout is not None:
open(logout, "w").write(output)
return parsed
def unique(mtzin, mtzout, wdir, logout):
##
# unique -> cad -> mtzutil (EXCLUDE FUNI SIGFUNI)
#
m = mtzutil.MtzFile(os.path.join(wdir, mtzin))
cell = m.get_cell_str()
sg = m.get_spacegroup()[1]
resol = min(m.get_resolution())
call(
cmd="unique",
arg="hklout unique.mtz",
wdir=wdir,
stdin="CELL %s\nSYMMETRY '%s'\nLABOUT F=FUNI SIGF=SIGFUNI\nRESOLUTION %f" % (cell, sg, resol),
expects_in=[],
expects_out=["unique.mtz"],
stdout=logout,
)
call(
cmd="cad",
arg="hklin1 %s hklin2 %s hklout %s" % (mtzin, "unique.mtz", "unique_cad.mtz"),
wdir=wdir,
stdin="labin file 1 all\nlabin file 2 all\nend\n",
expects_in=[mtzin, "unique.mtz"],
expects_out=["unique_cad.mtz"],
stdout=logout,
)
call(
cmd="mtzutils",
arg="hklin %s hklout %s" % ("unique_cad.mtz", mtzout),
wdir=wdir,
stdin="EXCLUDE FUNI SIGFUNI\nRUN\n",
expects_in=["unique_cad.mtz"],
expects_out=[mtzout],
stdout=logout,
)
os.remove(os.path.join(wdir, "unique.mtz"))
os.remove(os.path.join(wdir, "unique_cad.mtz"))
def run(search_dir, pf_logfiles=None):
for img_template, min_frame, max_frame in dataset.find_data_sets(
search_dir, skip_symlinks=False):
print "Dataset found:"
print " NAME_TEMPLATE_OF_DATA_FRAMES= %s" % img_template
print " DATA_RNAGE= %d %d" % (min_frame, max_frame)
pdir, ppath = os.path.split(img_template)
if not os.path.exists(os.path.join(pdir, "xds")):
os.mkdir(os.path.join(pdir, "xds"))
else:
assert os.path.isdir(os.path.join(pdir, "xds"))
prefix = ppath[:ppath.index("?")]
if not prefix.endswith("_"):
prefix += "_"
if pf_logfiles is None:
wdir = os.path.join(pdir, "xds",
"xds_%s%d-%d" % (prefix, min_frame, max_frame))
if os.path.isfile(os.path.join(wdir, "XDS.INP")):
print " Already exist:", os.path.join(wdir, "XDS.INP")
continue
os.mkdir(wdir)
cmd = 'generate_XDS.INP "%s"' % os.path.relpath(img_template, wdir)
print " launching", cmd
call(cmd=cmd, wdir=wdir)
else:
for minf, maxf in split_runs_based_on_pf_logfiles(pf_logfiles):
if maxf < min_frame or max_frame < minf:
continue
if minf < min_frame:
minf = min_frame
if maxf > max_frame:
maxf = max_frame
wdir = os.path.join(pdir, "xds",
"xds_%s%d-%d" % (prefix, minf, maxf))
if os.path.isfile(os.path.join(wdir, "XDS.INP")):
print "Already exist:", os.path.isfile(
os.path.join(wdir, "XDS.INP"))
continue
os.mkdir(wdir)
cmd = 'generate_XDS.INP "%s"' % os.path.relpath(
img_template, wdir)
print " launching", cmd
call(cmd=cmd, wdir=wdir)
# XXX dirty hacks..
inp = open(os.path.join(wdir, "XDS.INP")).readlines()
ofs = open(os.path.join(wdir, "XDS.INP"), "w")
for l in inp:
if l.startswith("DATA_RANGE="):
l = "DATA_RANGE= %d %d\n" % (minf, maxf)
if l.startswith("SPOT_RANGE="):
l = "SPOT_RANGE= %d %d\n" % (minf,
(maxf - minf) / 2 + minf)
ofs.write(l)
ofs.close()
print
def do_clustering(self, nproc=1, b_scale=False, use_normalized=False, html_maker=None):
self.clusters = {}
prefix = os.path.join(self.wdir, "cctable")
assert (b_scale, use_normalized).count(True) <= 1
if len(self.arrays) < 2:
print "WARNING: less than two data! can't do cc-based clustering"
self.clusters[1] = [float("nan"), [0]]
return
# Absolute scaling using Wilson-B factor
if b_scale:
from mmtbx.scaling.matthews import p_vm_calculator
from mmtbx.scaling.absolute_scaling import ml_iso_absolute_scaling
ofs_wilson = open("%s_wilson_scales.dat"%prefix, "w")
n_residues = p_vm_calculator(self.arrays.values()[0], 1, 0).best_guess
ofs_wilson.write("# guessed n_residues= %d\n" % n_residues)
ofs_wilson.write("file wilsonB\n")
for f in self.arrays:
arr = self.arrays[f]
iso_scale_and_b = ml_iso_absolute_scaling(arr, n_residues, 0)
wilson_b = iso_scale_and_b.b_wilson
ofs_wilson.write("%s %.3f\n" % (f, wilson_b))
if wilson_b > 0: # Ignoring data with B<0? is a bad idea.. but how..?
tmp = flex.exp(-2. * wilson_b * arr.unit_cell().d_star_sq(arr.indices())/4.)
self.arrays[f] = arr.customized_copy(data=arr.data()*tmp,
sigmas=arr.sigmas()*tmp)
ofs_wilson.close()
elif use_normalized:
from mmtbx.scaling.absolute_scaling import kernel_normalisation
for f in self.arrays:
arr = self.arrays[f]
normaliser = kernel_normalisation(arr, auto_kernel=True)
self.arrays[f] = arr.customized_copy(data=arr.data()/normaliser.normalizer_for_miller_array,
sigmas=arr.sigmas()/normaliser.normalizer_for_miller_array)
# Prep
args = []
for i in xrange(len(self.arrays)-1):
for j in xrange(i+1, len(self.arrays)):
args.append((i,j))
# Calc all CC
worker = lambda x: calc_cc(self.arrays.values()[x[0]], self.arrays.values()[x[1]])
results = easy_mp.pool_map(fixed_func=worker,
args=args,
processes=nproc)
# Check NaN and decide which data to remove
idx_bad = {}
nans = []
cc_data_for_html = []
for (i,j), (cc,nref) in zip(args, results):
cc_data_for_html.append((i,j,cc,nref))
if cc==cc: continue
idx_bad[i] = idx_bad.get(i, 0) + 1
idx_bad[j] = idx_bad.get(j, 0) + 1
nans.append([i,j])
if html_maker is not None:
html_maker.add_cc_clustering_details(cc_data_for_html)
idx_bad = idx_bad.items()
idx_bad.sort(key=lambda x:x[1])
remove_idxes = set()
for idx, badcount in reversed(idx_bad):
if len(filter(lambda x: idx in x, nans)) == 0: continue
remove_idxes.add(idx)
nans = filter(lambda x: idx not in x, nans)
if len(nans) == 0: break
use_idxes = filter(lambda x: x not in remove_idxes, xrange(len(self.arrays)))
# Make table: original index (in file list) -> new index (in matrix)
count = 0
org2now = collections.OrderedDict()
for i in xrange(len(self.arrays)):
if i in remove_idxes: continue
org2now[i] = count
count += 1
if len(remove_idxes) > 0:
open("%s_notused.lst"%prefix, "w").write("\n".join(map(lambda x: self.arrays.keys()[x], remove_idxes)))
# Make matrix
mat = numpy.zeros(shape=(len(use_idxes), len(use_idxes)))
for (i,j), (cc,nref) in zip(args, results):
if i in remove_idxes or j in remove_idxes: continue
mat[org2now[j], org2now[i]] = cc
open("%s.matrix"%prefix, "w").write(" ".join(map(lambda x:"%.4f"%x, mat.flatten())))
ofs = open("%s.dat"%prefix, "w")
ofs.write(" i j cc nref\n")
for (i,j), (cc,nref) in zip(args, results):
ofs.write("%4d %4d %.4f %4d\n" % (i,j,cc,nref))
open("%s_ana.R"%prefix, "w").write("""\
treeToList2 <- function(htree)
{ # stolen from $CCP4/share/blend/R/blend0.R
groups <- list()
itree <- dim(htree$merge)[1]
for (i in 1:itree)
{
il <- htree$merge[i,1]
ir <- htree$merge[i,2]
if (il < 0) lab1 <- htree$labels[-il]
if (ir < 0) lab2 <- htree$labels[-ir]
if (il > 0) lab1 <- groups[[il]]
if (ir > 0) lab2 <- groups[[ir]]
lab <- c(lab1,lab2)
lab <- as.integer(lab)
groups <- c(groups,list(lab))
}
return(groups)
}
cc<-scan("%(prefix)s.matrix")
md<-matrix(1-cc, ncol=%(ncol)d, byrow=TRUE)
hc <- hclust(as.dist(md),method="ward")
pdf("tree.pdf")
plot(hc)
dev.off()
png("tree.png",height=1000,width=1000)
plot(hc)
dev.off()
hc$labels <- c(%(hclabels)s)
groups <- treeToList2(hc)
cat("ClNumber Nds Clheight IDs\\n",file="./CLUSTERS.txt")
for (i in 1:length(groups))
{
sorted_groups <- sort(groups[[i]])
linea <- sprintf("%%04d %%4d %%7.3f %%s\\n",
i,length(groups[[i]]),hc$height[i], paste(sorted_groups,collapse=" "))
cat(linea, file="./CLUSTERS.txt", append=TRUE)
}
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
labels<-hc$labels
merge<-data.frame(hc$merge)
for (i in (1:nrow(merge))) {
if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
}
eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
return(JSON)
}
JSON<-HCtoJSON(hc)
cat(JSON, file="dendro.json")
q(save="yes")
""" % dict(prefix=os.path.basename(prefix),
ncol=len(self.arrays),
hclabels=",".join(map(lambda x: "%d"%(x+1), org2now.keys()))))
call(cmd="Rscript", arg="%s_ana.R" % os.path.basename(prefix),
wdir=self.wdir)
output = open(os.path.join(self.wdir, "CLUSTERS.txt")).readlines()
for l in output[1:]:
sp = l.split()
clid, clheight, ids = sp[0], sp[2], sp[3:]
self.clusters[int(clid)] = [float(clheight), map(int,ids)]
def copy_testflag(mtzfree, mtzin):
call(cmd="copy_free_R_flag.py -r %s %s" % (mtzfree, mtzin), wdir=os.path.dirname(mtzin))
def run_xdsstat(wdir):
comm = "xdsstat"
out = open(os.path.join(wdir, "XDSSTAT.LP"), "w")
sys.stdout.write("\r\x1b[K Running XDSSTAT")
util.call(comm, wdir=wdir, stdin="\n", stdout=out)
def run(params):
xdsinp = "XDS.INP"
workdir = os.getcwd()
# concerning DELPHI=
if len(params.delphi) == 0:
params.delphi = [get_delphi_defined(xdsinp)]
params.delphi.sort()
digit_delphi = str(get_digit(params.delphi))
# concerning REFINE(INTEGRATE)=
if params.all_refine_combinations:
refine_params = make_all_refine_combinations()
else:
refine_params = get_refine_defined(xdsinp)
backup_needed = files.generated_by_INTEGRATE + ("XDS.INP",)
if params.run_correct:
backup_needed += files.generated_by_CORRECT
bk_prefix = None
try:
for rp in refine_params:
rp_str = "refine_none" if len(rp) == 0 else "refine_" + "+".join(rp)
for delphi in params.delphi:
delphi_str = ("delphi_%."+digit_delphi+"f") % delphi
bk_prefix = make_backup(backup_needed) # Backup existing XDS.INP and others
work_name = "INTEGRATE_%s_%s" % (rp_str, delphi_str)
inp_params = [("JOB","INTEGRATE"),
("DELPHI", delphi),
("REFINE(INTEGRATE)", " ".join(rp))
]
if params.run_correct:
inp_params[0] = ("JOB","INTEGRATE CORRECT")
modify_xdsinp(xdsinp, inp_params=inp_params)
call("xds_par", stdout=sys.stdout)
# Move files into a new directory.
os.mkdir(work_name)
for f in backup_needed:
shutil.move(f, work_name)
if params.run_correct:
# make .mtz file
try:
call("xdsstat", stdin="\n", stdout=open(os.path.join(work_name, "XDSSTAT.LP"),"w"), wdir=work_name)
xds2mtz.xds2mtz(os.path.abspath(os.path.join(work_name, "XDS_ASCII.HKL")),
dir_name=os.path.join(workdir, work_name, "ccp4"),
run_ctruncate=False, run_xtriage=True
)
if params.mtz_free is not None:
copy_testflag(mtzfree=params.mtz_free,
mtzin=os.path.join(workdir, work_name, "ccp4", "XDS_ASCII.mtz"))
except:
print traceback.format_exc()
print "Ignoring xds2mtz error.."
print
revert_files(backup_needed, bk_prefix) # Revert XDS.INP and others
finally:
# 6. Revert XDS.INP, etc.
revert_files(backup_needed, bk_prefix)
def run_dials_sequence(filename_template, prefix, nr_range, wdir, known_xs,
overrides, scan_varying, nproc):
log_out = open(os.path.join(wdir, "dials_sequence.log"), "w")
pointless_log = os.path.join(wdir, "pointless.log")
# Prepare
img_files = find_existing_files_in_template(
filename_template,
nr_range[0],
nr_range[1],
datadir=os.path.dirname(prefix),
check_compressed=True)
if len(img_files) == 0:
mylog.error("No files found for %s %s" % (filename_template, nr_range))
return
nproc_str = "nproc=%d" % nproc
log_out.write("Importing %s range=%s\n" % (img_files, nr_range))
log_out.write(" Overrides: %s\n" % overrides)
log_out.flush()
override_str = "" # TODO support other stuff.. (wavelength, distance, osc_range, rotation_axis,..)
if "orgx" in overrides and "orgy" in overrides:
override_str += "slow_fast_beam_centre=%.2f,%.2f " % (
overrides["orgy"], overrides["orgx"])
if len(img_files) == 1 and img_files[0].endswith(".h5"):
util.call('dials.import "%s" %s image_range=%d,%d' %
(img_files[0], override_str, nr_range[0], nr_range[1]),
wdir=wdir,
stdout=log_out,
expects_out=[os.path.join(wdir, "datablock.json")])
else:
util.call('dials.import %s template="%s" image_range=%d,%d' %
(override_str, filename_template.replace(
"?", "#"), nr_range[0], nr_range[1]),
wdir=wdir,
stdout=log_out,
expects_out=[os.path.join(wdir, "datablock.json")])
util.call(
"dials.find_spots datablock.json filter.d_max=30 %s" %
nproc_str, # global_threshold=200
wdir=wdir,
stdout=log_out,
expects_out=[os.path.join(wdir, "strong.pickle")])
util.call("dials.export strong.pickle format=xds xds.directory=.",
wdir=wdir,
stdout=log_out)
index_ok = False
for index_meth in ("fft3d", "fft1d", "real_space_grid_search"):
for index_assi in ("local", "simple"):
if index_ok: break
cmd = "dials.index datablock.json strong.pickle verbosity=3 "
cmd += "indexing.method=%s index_assignment.method=%s " % (
index_meth, index_assi)
if known_xs is not None: # not in (known.space_group, known.unit_cell):
cmd += "unit_cell=%s space_group=%d " % (
",".join(
map(lambda x: "%.3f" % x,
known_xs.unit_cell().parameters())),
known_xs.space_group().type().number())
elif index_meth == "real_space_grid_search":
continue
log_out.write(
"Trying indexing.method=%s index_assignment.method=%s\n" %
(index_meth, index_assi))
log_out.flush()
util.call(cmd, wdir=wdir, stdout=log_out)
if os.path.isfile(os.path.join(wdir, "experiments.json")):
index_ok = True
else:
for f in ("dials.index.log", "dials.index.debug.log"):
util.rotate_file(os.path.join(wdir, f))
if not index_ok:
return
files_for_integration = "experiments.json indexed.pickle"
if scan_varying:
util.call(
"dials.refine experiments.json indexed.pickle scan_varying=true",
wdir=wdir,
stdout=log_out)
if os.path.isfile(os.path.join(wdir, "refined.pickle")):
files_for_integration = "refined_experiments.json refined.pickle"
else:
log_out.write("dials.refine failed. using intedexed results.\n")
util.call("dials.integrate %s min_spots.per_degree=10 %s" %
(files_for_integration, nproc_str),
wdir=wdir,
stdout=log_out)
util.call(
"dials.export integrated.pickle integrated_experiments.json mtz.hklout=integrated.mtz",
wdir=wdir,
stdout=log_out)
util.call("pointless integrated.mtz hklout pointless.mtz",
wdir=wdir,
stdin="SETTING SYMMETRY-BASED\ntolerance 10\n",
stdout=open(pointless_log, "w"))
util.call(
"dials.export integrated_experiments.json integrated.pickle format=xds_ascii xds_ascii.hklout=DIALS.HKL",
wdir=wdir,
stdout=log_out)
util.call("aimless hklin pointless.mtz hklout aimless.mtz",
wdir=wdir,
stdin="output UNMERGED\n",
stdout=open(os.path.join(wdir, "aimless.log"), "w"))
#job_str += "touch dials_job_finished\n"
ret = calc_merging_stats(os.path.join(wdir, "aimless_unmerged.mtz"))
ret["symm"] = get_most_possible_symmetry(wdir)
pickle.dump(ret, open(os.path.join(wdir, "kamo_dials.pkl"), "w"), -1)
def do_clustering(self,
nproc=1,
b_scale=False,
use_normalized=False,
html_maker=None):
self.clusters = {}
prefix = os.path.join(self.wdir, "cctable")
assert (b_scale, use_normalized).count(True) <= 1
if len(self.arrays) < 2:
print "WARNING: less than two data! can't do cc-based clustering"
self.clusters[1] = [float("nan"), [0]]
return
# Absolute scaling using Wilson-B factor
if b_scale:
from mmtbx.scaling.matthews import p_vm_calculator
from mmtbx.scaling.absolute_scaling import ml_iso_absolute_scaling
ofs_wilson = open("%s_wilson_scales.dat" % prefix, "w")
n_residues = p_vm_calculator(self.arrays.values()[0], 1,
0).best_guess
ofs_wilson.write("# guessed n_residues= %d\n" % n_residues)
ofs_wilson.write("file wilsonB\n")
for f in self.arrays:
arr = self.arrays[f]
iso_scale_and_b = ml_iso_absolute_scaling(arr, n_residues, 0)
wilson_b = iso_scale_and_b.b_wilson
ofs_wilson.write("%s %.3f\n" % (f, wilson_b))
if wilson_b > 0: # Ignoring data with B<0? is a bad idea.. but how..?
tmp = flex.exp(-2. * wilson_b *
arr.unit_cell().d_star_sq(arr.indices()) /
4.)
self.arrays[f] = arr.customized_copy(data=arr.data() * tmp,
sigmas=arr.sigmas() *
tmp)
ofs_wilson.close()
elif use_normalized:
from mmtbx.scaling.absolute_scaling import kernel_normalisation
for f in self.arrays:
arr = self.arrays[f]
normaliser = kernel_normalisation(arr, auto_kernel=True)
self.arrays[f] = arr.customized_copy(
data=arr.data() / normaliser.normalizer_for_miller_array,
sigmas=arr.sigmas() /
normaliser.normalizer_for_miller_array)
# Prep
args = []
for i in xrange(len(self.arrays) - 1):
for j in xrange(i + 1, len(self.arrays)):
args.append((i, j))
# Calc all CC
if self.use_sfdist:
worker = lambda x: calc_sfdist(self.arrays.values()[x[0]],
self.arrays.values()[x[1]])
else:
worker = lambda x: calc_cc(self.arrays.values()[x[0]],
self.arrays.values()[x[1]])
results = easy_mp.pool_map(fixed_func=worker,
args=args,
processes=nproc)
# Check NaN and decide which data to remove
idx_bad = {}
nans = []
cc_data_for_html = []
for (i, j), (cc, nref) in zip(args, results):
cc_data_for_html.append((i, j, cc, nref))
if cc == cc: continue
idx_bad[i] = idx_bad.get(i, 0) + 1
idx_bad[j] = idx_bad.get(j, 0) + 1
nans.append([i, j])
if html_maker is not None:
html_maker.add_cc_clustering_details(cc_data_for_html)
idx_bad = idx_bad.items()
idx_bad.sort(key=lambda x: x[1])
remove_idxes = set()
for idx, badcount in reversed(idx_bad):
if len(filter(lambda x: idx in x, nans)) == 0: continue
remove_idxes.add(idx)
nans = filter(lambda x: idx not in x, nans)
if len(nans) == 0: break
use_idxes = filter(lambda x: x not in remove_idxes,
xrange(len(self.arrays)))
# Make table: original index (in file list) -> new index (in matrix)
count = 0
org2now = collections.OrderedDict()
for i in xrange(len(self.arrays)):
if i in remove_idxes: continue
org2now[i] = count
count += 1
if len(remove_idxes) > 0:
open("%s_notused.lst" % prefix, "w").write("\n".join(
map(lambda x: self.arrays.keys()[x], remove_idxes)))
# Make matrix
mat = numpy.zeros(shape=(len(use_idxes), len(use_idxes)))
for (i, j), (cc, nref) in zip(args, results):
if i in remove_idxes or j in remove_idxes: continue
mat[org2now[j], org2now[i]] = cc
open("%s.matrix" % prefix,
"w").write(" ".join(map(lambda x: "%.4f" % x, mat.flatten())))
ofs = open("%s.dat" % prefix, "w")
ofs.write(" i j cc nref\n")
for (i, j), (cc, nref) in zip(args, results):
ofs.write("%4d %4d %.4f %4d\n" % (i, j, cc, nref))
open("%s_ana.R" % prefix, "w").write("""\
treeToList2 <- function(htree)
{ # stolen from $CCP4/share/blend/R/blend0.R
groups <- list()
itree <- dim(htree$merge)[1]
for (i in 1:itree)
{
il <- htree$merge[i,1]
ir <- htree$merge[i,2]
if (il < 0) lab1 <- htree$labels[-il]
if (ir < 0) lab2 <- htree$labels[-ir]
if (il > 0) lab1 <- groups[[il]]
if (ir > 0) lab2 <- groups[[ir]]
lab <- c(lab1,lab2)
lab <- as.integer(lab)
groups <- c(groups,list(lab))
}
return(groups)
}
cc<-scan("%(prefix)s.matrix")
md<-matrix(1-cc, ncol=%(ncol)d, byrow=TRUE)
hc <- hclust(as.dist(md),method="ward")
pdf("tree.pdf")
plot(hc)
dev.off()
png("tree.png",height=1000,width=1000)
plot(hc)
dev.off()
hc$labels <- c(%(hclabels)s)
groups <- treeToList2(hc)
cat("ClNumber Nds Clheight IDs\\n",file="./CLUSTERS.txt")
for (i in 1:length(groups))
{
sorted_groups <- sort(groups[[i]])
linea <- sprintf("%%04d %%4d %%7.3f %%s\\n",
i,length(groups[[i]]),hc$height[i], paste(sorted_groups,collapse=" "))
cat(linea, file="./CLUSTERS.txt", append=TRUE)
}
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
labels<-hc$labels
merge<-data.frame(hc$merge)
for (i in (1:nrow(merge))) {
if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
}
eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
return(JSON)
}
JSON<-HCtoJSON(hc)
cat(JSON, file="dendro.json")
q(save="yes")
""" % dict(prefix=os.path.basename(prefix),
ncol=len(self.arrays),
hclabels=",".join(map(lambda x: "%d" % (x + 1), org2now.keys()))))
call(cmd="Rscript",
arg="%s_ana.R" % os.path.basename(prefix),
wdir=self.wdir)
output = open(os.path.join(self.wdir, "CLUSTERS.txt")).readlines()
for l in output[1:]:
sp = l.split()
clid, clheight, ids = sp[0], sp[2], sp[3:]
self.clusters[int(clid)] = [float(clheight), map(int, ids)]
def copy_testflag(mtzfree, mtzin, mtzout):
call(cmd="copy_free_R_flag.py -r %s -o %s %s" % (mtzfree, mtzout, mtzin))
def run_xscale(xscale_inp, cbf_to_dat=False, use_tmpdir_if_available=False):
ftable = {}
outfile = None
count = 0
inpdir = os.path.dirname(os.path.abspath(xscale_inp))
wdir = inpdir # may be overridden
tmpdir = None
if use_tmpdir_if_available:
tmpdir = util.get_temp_local_dir("xscale", min_gb=1) # TODO guess required tempdir size
print tmpdir
if tmpdir is None:
print "Can't get temp dir with sufficient size."
if tmpdir is not None:
shutil.copy2(xscale_inp, tmpdir)
xscale_inp = os.path.join(tmpdir, os.path.basename(xscale_inp))
wdir = tmpdir
os.rename(xscale_inp, xscale_inp+".org")
ofs = open(xscale_inp, "w")
# Check line length and make symlink if needed
for l in open(xscale_inp+".org"):
ll = l[:l.index("!")] if "!" in l else l
if "OUTPUT_FILE=" in ll:
outfile = ll[ll.index("=")+1:].strip() # TODO what if the file is not in current directory?
if "INPUT_FILE=" in ll: # and len(l) > 132: # one line is limited to 131 characters!
filename = ll[ll.index("=")+1:].strip()
if "*" in filename: filename = filename[filename.index("*")+1:].strip()
assert " " not in filename
lnkf = "lnk%.6d.hkl" % count
assert not os.path.isfile(os.path.join(wdir, lnkf))
filename_abs = os.path.normpath(os.path.join(inpdir, filename)) if not os.path.isabs(filename) else filename
os.symlink(filename_abs, os.path.join(wdir, lnkf))
print "xscale: %s -> %s" % (lnkf, filename)
count += 1
ftable[lnkf] = filename
l = l.replace(filename, lnkf)
ofs.write(l)
ofs.close()
assert outfile is not None
if len(ftable) == 0:
os.rename(xscale_inp+".org", xscale_inp)
# Run xscale
util.call(xscale_comm, wdir=wdir)
# Replace file names if needed
if len(ftable) > 0:
for i, f in enumerate(("XSCALE.LP", outfile)):
f = os.path.join(wdir, f)
if not os.path.isfile(f): continue
os.rename(f, f+".org")
ofs = open(f, "w")
if i == 0:
for l in open(f+".org"):
if ".hkl" in l:
for lfn in ftable: l = l.replace(lfn, ftable[lfn])
ofs.write(l)
else:
ifs = open(f+".org")
while True:
l = ifs.readline()
if ".hkl" in l:
for lfn in ftable: l = l.replace(lfn, ftable[lfn])
ofs.write(l)
if l.startswith("!END_OF_HEADER"): break
shutil.copyfileobj(ifs, ofs)
ofs.close()
os.remove(f+".org")
for lfn in ftable:
os.remove(os.path.join(wdir, lfn))
os.rename(xscale_inp+".org", xscale_inp)
if cbf_to_dat:
xscale_lp = os.path.join(wdir, "XSCALE.LP")
cbfouts = glob.glob(os.path.join(wdir, "*.cbf"))
if len(cbfouts) > 0:
xscalelp.cbf_to_dat(xscale_lp)
for f in cbfouts: os.remove(f)
# Move to original directory
if tmpdir is not None:
for f in glob.glob(os.path.join(tmpdir, "*")):
shutil.copy2(f, inpdir)
shutil.rmtree(tmpdir)
def calc_delta_cchalf(prev_lp, tmpdir, with_sigma=False, precalc_cchalf_all=None):
"""
Obsolete function. Maybe useful when starting with the last XSCALE.LP...
"""
assert not with_sigma # Not supported now
if not os.path.exists(tmpdir): os.makedirs(tmpdir)
newinp = os.path.join(tmpdir, "XSCALE.INP")
newlp = os.path.join(tmpdir, "XSCALE.LP")
rel_org = os.path.relpath(os.path.dirname(prev_lp), tmpdir)
if precalc_cchalf_all is None:
# read CC1/2(all) from cwd
orgtable = xscalelp.read_stats_table(prev_lp)
assert orgtable["dmin"][-1] is None # None for total
cchalf_all = orgtable["cc_half"][-1]
else:
cchalf_all = precalc_cchalf_all
datout = open(os.path.join(tmpdir, "delta_cchalf.dat"), "w")
datout.write("# CC1/2(all)= %.4f\n" % cchalf_all)
datout.write("idx exfile cc1/2 delta_cc1/2\n")
# Read inp and extract input files.
# XXX What if reference file is included???
orgkwds = xscalelp.read_control_cards(prev_lp)
inpfiles = map(lambda x:x[1],
filter(lambda y: y[0]=="INPUT_FILE", orgkwds))
# XXX Need to take care of xscale specific inp manner - order matters!!
delta_cchalf = []
for iex in xrange(len(inpfiles)):
print "Doing", iex
files = inpfiles[:iex] + inpfiles[iex+1:]
ofs = open(newinp, "w")
for k, v in orgkwds:
if k not in ("INPUT_FILE", "INCLUDE_RESOLUTION_RANGE"):
ofs.write("%s= %s\n" % (k,v))
for f in files:
if not os.path.isabs(f): f = os.path.join(rel_org, f)
ofs.write("INPUT_FILE= %s\n" % f)
ofs.close()
util.call(xscale_comm, wdir=tmpdir)
table = xscalelp.read_stats_table(newlp)
assert table["dmin"][-1] is None # None for total
cchalf_exi = table["cc_half"][-1]
delta_cchalf.append((iex, cchalf_exi - cchalf_all))
os.rename(newinp, newinp+".ex%.3d"%iex)
os.rename(newlp, newlp+".ex%.3d"%iex)
datout.write("%3d %s %.4f %.4f\n" % (iex, inpfiles[iex], cchalf_exi, cchalf_exi-cchalf_all))
delta_cchalf.sort(key=lambda x: -x[1])
print
print "# Sorted table"
for idx, dch in delta_cchalf:
print "%3d %-.4f %s" % (idx, dch, inpfiles[idx])
return delta_cchalf, cchalf_all
def run(hklin, pdbin, wdir, anisotropy_correction=False):
arrays = iotbx.file_reader.any_file(hklin).file_server.miller_arrays
i_arrays = filter(
lambda x: x.is_xray_intensity_array() and x.anomalous_flag(), arrays)
f_arrays = filter(
lambda x: x.is_xray_amplitude_array() and x.anomalous_flag(), arrays)
if not i_arrays and not f_arrays:
print "No anomalous observation data"
return
if os.path.exists(wdir):
print "%s already exists. quiting." % wdir
return
os.mkdir(wdir)
xs = crystal_symmetry_from_any.extract_from(pdbin)
sh_out = open(os.path.join(wdir, "run_anode.sh"), "w")
sh_out.write("#!/bin/sh\n\n")
sh_out.write("shelxc anode <<+ > shelxc.log 2>&1\n")
sh_out.write("cell %s\n" % format_unit_cell(xs.unit_cell()))
sh_out.write("spag %s\n" % str(xs.space_group_info()).replace(" ", ""))
if i_arrays:
obs_array = i_arrays[0]
infile = "%s.hkl" % os.path.splitext(os.path.basename(hklin))[0]
in_opt = "%s" % infile
print "Using intensity array:", obs_array.info().label_string()
else:
obs_array = f_arrays[0]
infile = "%s_f.hkl" % os.path.splitext(os.path.basename(hklin))[0]
in_opt = "-f %s" % infile
print "No intensity arrays. Using amplitude arrays instead:", obs_array.info(
).label_string()
sh_out.write("! data from %s : %s\n" %
(os.path.abspath(hklin), obs_array.info().label_string()))
obs_array.crystal_symmetry().show_summary(sh_out, prefix="! ")
check_symm(obs_array.crystal_symmetry(), xs)
n_org = obs_array.size()
obs_array = obs_array.eliminate_sys_absent()
n_sys_abs = n_org - obs_array.size()
if n_sys_abs > 0:
print " %d systematic absences removed." % n_sys_abs
if anisotropy_correction:
print "Correcting anisotropy.."
n_residues = p_vm_calculator(obs_array, 1, 0).best_guess
abss = ml_aniso_absolute_scaling(obs_array, n_residues=n_residues)
abss.show()
tmp = -2. if i_arrays else -1.
b_cart = map(lambda x: x * tmp, abss.b_cart)
obs_array = obs_array.apply_debye_waller_factors(b_cart=b_cart)
sh_out.write("sad %s\n" % in_opt)
iotbx.shelx.hklf.miller_array_export_as_shelx_hklf(
obs_array,
open(os.path.join(wdir, infile), "w"),
normalise_if_format_overflow=True)
sh_out.write("+\n\n")
sh_out.write('ln -s "%s" anode.pdb\n\n' % os.path.relpath(pdbin, wdir))
sh_out.write("anode anode\n")
sh_out.close()
call(cmd="sh", arg="./run_anode.sh", wdir=wdir)
pha_file = os.path.join(wdir, "anode.pha")
if os.path.isfile(pha_file):
pha2mtz(pha_file, xs, os.path.join(wdir, "anode.pha.mtz"))
print "Done. See %s/" % wdir
fa_file = os.path.join(wdir, "anode_fa.hkl")
if os.path.isfile(fa_file):
r = iotbx.shelx.hklf.reader(open(fa_file))
fa_array = r.as_miller_arrays(crystal_symmetry=xs)[0]
print "\nData stats:"
print " # Cmpl.o = Anomalous completeness in original data"
print " # Cmpl.c = Anomalous completeness in shelxc result (rejections)"
print " # SigAno = <d''/sigma> in shelxc result"
print " d_max d_min Cmpl.o Cmpl.c SigAno"
binner = obs_array.setup_binner(n_bins=12)
for i_bin in binner.range_used():
d_max_bin, d_min_bin = binner.bin_d_range(i_bin)
obs_sel = obs_array.resolution_filter(d_max_bin, d_min_bin)
obs_sel_ano = obs_sel.anomalous_differences()
fa_sel = fa_array.resolution_filter(d_max_bin, d_min_bin)
cmplset = obs_sel_ano.complete_set(
d_max=d_max_bin, d_min=d_min_bin).select_acentric()
n_acentric = cmplset.size()
sigano = flex.mean(
fa_sel.data() /
fa_sel.sigmas()) if fa_sel.size() else float("nan")
print " %5.2f %5.2f %6.2f %6.2f %6.2f" % (
d_max_bin, d_min_bin, 100. * obs_sel_ano.size() / n_acentric,
100. * fa_sel.size() / n_acentric, sigano)
lsa_file = os.path.join(wdir, "anode.lsa")
if os.path.isfile(lsa_file):
print ""
flag = False
for l in open(lsa_file):
if "Strongest unique anomalous peaks" in l:
flag = True
elif "Reflections written to" in l:
flag = False
if flag:
print l.rstrip()
if os.path.isfile(("anode_fa.res")):
x = iotbx.shelx.cctbx_xray_structure_from(file=open("anode_fa.res"))
open("anode_fa.pdb", "w").write(x.as_pdb_file())
def xds2mtz_normal(refl, mtzout, sg, wavelen, logout, use_ctruncate=False, dmin=None, dmax=None):
wdir = os.path.dirname(mtzout)
if not os.path.exists(os.path.join(wdir, "original")):
os.symlink(refl, os.path.join(wdir, "original"))
##
# prepare XDSCONV.INP and run
#
# for I
print "generating MTZ for IMEAN,SIGIMEAN"
ofs = open(os.path.join(wdir, "XDSCONV.INP"), "w")
ofs.write("OUTPUT_FILE=tmp.hkl CCP4_I\n")
ofs.write("INPUT_FILE=original\n")
ofs.write("GENERATE_FRACTION_OF_TEST_REFLECTIONS=0.0\n")
ofs.write("WILSON_STATISTICS= TRUE\n")
if None not in (dmin, dmax):
ofs.write("INCLUDE_RESOLUTION_RANGE= %s %s\n" % (dmax, dmin))
ofs.close()
call(cmd="xdsconv",
wdir=wdir,
expects_in=["original"],
expects_out=["F2MTZ.INP", "tmp.hkl"],
stdout=logout
)
call(cmd="f2mtz",
arg="hklout CCP4_I.mtz",
stdin=open(os.path.join(wdir, "F2MTZ.INP")).read(),
wdir=wdir,
expects_in=["tmp.hkl"],
expects_out=["CCP4_I.mtz"],
stdout=logout
)
# for F
print "generating MTZ for FP,SIGFP"
ctruncate_ok = False
if use_ctruncate:
call(cmd="ctruncate -hklin CCP4_I.mtz -hklout ctruncate.mtz -colin '/*/*/[IMEAN,SIGIMEAN]'",
wdir=wdir,
expects_in=["CCP4_I.mtz"],
stdout=open(os.path.join(wdir, "ctruncate.log"), "w")
)
ctruncate_ok = os.path.isfile(os.path.join(wdir, "ctruncate.mtz"))
if ctruncate_ok:
call(cmd="cad",
arg="hklin1 ctruncate.mtz hklout CCP4_FI.mtz",
stdin="""\
labin file 1 all
xname file 1 ALL=XDS
dname file 1 ALL=XDS
dwavelength file 1 XDS XDS %s
symmetry %s
end
"""%(wavelen,sg),
wdir=wdir,
expects_in=["ctruncate.mtz"],
expects_out=["CCP4_FI.mtz"],
stdout=logout
)
else:
print "Ctruncate failed. Use xdsconv instead."
if not ctruncate_ok:
ofs = open(os.path.join(wdir, "XDSCONV.INP"), "w")
ofs.write("OUTPUT_FILE=tmp.hkl CCP4_F\n")
ofs.write("INPUT_FILE=original\n")
ofs.write("GENERATE_FRACTION_OF_TEST_REFLECTIONS=0.0\n")
ofs.write("WILSON_STATISTICS= TRUE\n")
if None not in (dmin, dmax):
ofs.write("INCLUDE_RESOLUTION_RANGE= %s %s\n" % (dmax, dmin))
ofs.close()
call(cmd="xdsconv",
wdir=wdir,
expects_in=["original"],
expects_out=["F2MTZ.INP", "tmp.hkl"],
stdout=logout
)
call(cmd="f2mtz",
arg="hklout CCP4_F.mtz",
stdin=open(os.path.join(wdir, "F2MTZ.INP")).read(),
wdir=wdir,
expects_in=["tmp.hkl"],
expects_out=["CCP4_F.mtz"],
stdout=logout
)
##
# CAD all mtz files
print "concatenating MTZ files"
call(cmd="cad",
arg="hklin1 CCP4_I.mtz hklin2 CCP4_F.mtz hklout CCP4_FI.mtz",
stdin="""\
labin file 1 all
labin file 2 all
xname file 1 ALL=XDS
xname file 2 ALL=XDS
dname file 1 ALL=XDS
dname file 2 ALL=XDS
dwavelength file 1 XDS XDS %s
symmetry %s
end
"""%(wavelen,sg),
wdir=wdir,
expects_in=["CCP4_I.mtz", "CCP4_F.mtz"],
expects_out=["CCP4_FI.mtz"],
stdout=logout
)
##
# Generate all unique reflections
print "Genrating all unique reflections"
unique(mtzin="CCP4_FI.mtz", mtzout=os.path.basename(mtzout), wdir=wdir, logout=logout)
# remove files
os.remove(os.path.join(wdir, "CCP4_I.mtz"))
os.remove(os.path.join(wdir, "CCP4_FI.mtz"))
os.remove(os.path.join(wdir, "tmp.hkl"))
os.remove(os.path.join(wdir, "XDSCONV.INP"))
os.remove(os.path.join(wdir, "XDSCONV.LP"))
os.remove(os.path.join(wdir, "F2MTZ.INP"))
os.remove(os.path.join(wdir, "original"))
if ctruncate_ok:
os.remove(os.path.join(wdir, "ctruncate.mtz"))
else:
os.remove(os.path.join(wdir, "CCP4_F.mtz"))
def copy_testflag(mtzfree, mtzin):
call(cmd="copy_free_R_flag.py -r %s %s" % (mtzfree, mtzin),
wdir=os.path.dirname(mtzin))
def find_spots(img_file, params):
"""
Use XDS to locate spots.
If params.xds.do_defpix is true, DEFPIX will be run. For DEFPIX, dummy XPARM.XDS is needed. Thanks to DEFPIX, we can mask beam stop shadow and remove areas outside the resolution range.
If false, we need to set TRUSTED_REGION to exclude regions outside resolution range, but it is independent of beam center. Maybe we should remove spots outside the resolution range after XDS run?
"""
# Test if ramdisk available
if os.path.isdir("/dev/shm"):
work_dir = tempfile.mkdtemp(
prefix="shika_x_" +
os.path.splitext(os.path.basename(img_file))[0],
dir="/dev/shm")
else:
work_dir = os.path.join(
params.work_dir,
"xds_" + os.path.splitext(os.path.basename(img_file))[0])
xdsinp = os.path.join(work_dir, "XDS.INP")
spot_xds = os.path.join(work_dir, "SPOT.XDS")
if not os.path.exists(work_dir):
os.mkdir(work_dir)
template_str, min_frame, max_frame = dataset.group_img_files_template(
[img_file])[0]
im = XIO.Image(img_file)
# Remove lines if None (means to use default)
params_maps = [
("strong_pixel", "STRONG_PIXEL="),
("minimum_number_of_pixels_in_a_spot",
"MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT="),
("background_pixel", "BACKGROUND_PIXEL="),
("maximum_number_of_strong_pixels",
"MAXIMUM_NUMBER_OF_STRONG_PIXELS="),
("spot_maximum_centroid", "SPOT_MAXIMUM-CENTROID="),
("reflecting_range", "REFLECTING_RANGE="),
("value_range_for_trusted_detector_pixels",
"VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS="),
]
tmp = xds_inp_template.splitlines()
for p, x in params_maps:
if getattr(params.xds, p) is None:
tmp = filter(lambda s: not s.startswith(x), tmp)
inp_template = "\n".join(tmp)
# Prepare XDS.INP
inp_str = inp_template % dict(
template=os.path.relpath(template_str, work_dir),
framenum=min_frame,
orgx=im.header["BeamX"] / im.header["PixelX"],
orgy=im.header["BeamY"] / im.header["PixelY"],
distance=im.header["Distance"],
osc_range=im.header["PhiWidth"],
wavelength=im.header["Wavelength"],
strong_pixel=params.xds.strong_pixel,
min_pixels=params.xds.minimum_number_of_pixels_in_a_spot,
background_pixel=params.xds.background_pixel,
max_strong_pixels=params.xds.maximum_number_of_strong_pixels,
spot_maximum_centroid=params.xds.spot_maximum_centroid,
reflecting_range=params.xds.reflecting_range,
nx=im.header["Width"],
ny=im.header["Height"],
qx=im.header["PixelX"],
qy=im.header["PixelY"],
defpix_trusted1=params.xds.value_range_for_trusted_detector_pixels[0],
defpix_trusted2=params.xds.value_range_for_trusted_detector_pixels[1])
open(xdsinp, "w").write(inp_str)
if params.xds.do_defpix:
xp = xparm.XPARM()
xp.set_info_from_xdsinp(xdsinp)
open(os.path.join(work_dir, "XPARM.XDS"), "w").write(xp.xparm_str())
modify_xdsinp(xdsinp,
inp_params=[("JOB", "XYCORR INIT DEFPIX"),
("INCLUDE_RESOLUTION_RANGE",
res_range_for_xds(params.distl.res.outer,
params.distl.res.inner))])
call("xds",
wdir=work_dir,
stdout=open(os.path.join(work_dir, "xds.log"), "w"))
shutil.copy(os.path.join(work_dir, "BKGPIX.cbf"),
os.path.join(work_dir, "BKGINIT.cbf"))
modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
else:
modify_xdsinp(xdsinp,
inp_params=[
("TRUSTED_REGION",
res_range_to_trusted_region(params.distl.res.outer,
params.distl.res.inner,
im.header))
])
open(os.path.join(work_dir, "xds.log"), "w").write("")
# Run XDS
rotate_file(spot_xds)
call("xds",
wdir=work_dir,
stdout=open(os.path.join(work_dir, "xds.log"), "a"))
# Extract results
spots = [] # (x, y, d, intensity)
if os.path.isfile(spot_xds):
for l in open(spot_xds):
x, y, z, intensity = map(lambda x: float(x), l.strip().split())
d = coord_to_resol(x, y, im.header)
spots.append((x, y, d, intensity))
# Delete dir
shutil.rmtree(work_dir)
return spots
def find_spots(img_file, params):
"""
Use XDS to locate spots.
If params.xds.do_defpix is true, DEFPIX will be run. For DEFPIX, dummy XPARM.XDS is needed. Thanks to DEFPIX, we can mask beam stop shadow and remove areas outside the resolution range.
If false, we need to set TRUSTED_REGION to exclude regions outside resolution range, but it is independent of beam center. Maybe we should remove spots outside the resolution range after XDS run?
"""
# Test if ramdisk available
if os.path.isdir("/dev/shm"):
work_dir = tempfile.mkdtemp(prefix="shika_x_"+os.path.splitext(os.path.basename(img_file))[0], dir="/dev/shm")
else:
work_dir = os.path.join(params.work_dir, "xds_"+os.path.splitext(os.path.basename(img_file))[0])
xdsinp = os.path.join(work_dir, "XDS.INP")
spot_xds = os.path.join(work_dir, "SPOT.XDS")
if not os.path.exists(work_dir):
os.mkdir(work_dir)
template_str, min_frame, max_frame = dataset.group_img_files_template([img_file])[0]
im = XIO.Image(img_file)
# Remove lines if None (means to use default)
params_maps = [("strong_pixel", "STRONG_PIXEL="),
("minimum_number_of_pixels_in_a_spot", "MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT="),
("background_pixel", "BACKGROUND_PIXEL="),
("maximum_number_of_strong_pixels", "MAXIMUM_NUMBER_OF_STRONG_PIXELS="),
("spot_maximum_centroid", "SPOT_MAXIMUM-CENTROID="),
("reflecting_range", "REFLECTING_RANGE="),
("value_range_for_trusted_detector_pixels", "VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS="),
]
tmp = xds_inp_template.splitlines()
for p, x in params_maps:
if getattr(params.xds, p) is None:
tmp = filter(lambda s:not s.startswith(x), tmp)
inp_template = "\n".join(tmp)
# Prepare XDS.INP
inp_str = inp_template%dict(template=os.path.relpath(template_str, work_dir),
framenum=min_frame,
orgx=im.header["BeamX"]/im.header["PixelX"],
orgy=im.header["BeamY"]/im.header["PixelY"],
distance=im.header["Distance"],
osc_range=im.header["PhiWidth"],
wavelength=im.header["Wavelength"],
strong_pixel=params.xds.strong_pixel,
min_pixels=params.xds.minimum_number_of_pixels_in_a_spot,
background_pixel=params.xds.background_pixel,
max_strong_pixels=params.xds.maximum_number_of_strong_pixels,
spot_maximum_centroid=params.xds.spot_maximum_centroid,
reflecting_range=params.xds.reflecting_range,
nx=im.header["Width"], ny=im.header["Height"],
qx=im.header["PixelX"], qy=im.header["PixelY"],
defpix_trusted1=params.xds.value_range_for_trusted_detector_pixels[0],
defpix_trusted2=params.xds.value_range_for_trusted_detector_pixels[1]
)
open(xdsinp, "w").write(inp_str)
if params.xds.do_defpix:
xp = xparm.XPARM()
xp.set_info_from_xdsinp(xdsinp)
open(os.path.join(work_dir, "XPARM.XDS"), "w").write(xp.xparm_str())
modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT DEFPIX"),
("INCLUDE_RESOLUTION_RANGE", res_range_for_xds(params.distl.res.outer, params.distl.res.inner))
])
call("xds", wdir=work_dir, stdout=open(os.path.join(work_dir, "xds.log"), "w"))
shutil.copy(os.path.join(work_dir, "BKGPIX.cbf"), os.path.join(work_dir, "BKGINIT.cbf"))
modify_xdsinp(xdsinp, inp_params=[("JOB","COLSPOT")])
else:
modify_xdsinp(xdsinp, inp_params=[("TRUSTED_REGION", res_range_to_trusted_region(params.distl.res.outer, params.distl.res.inner, im.header))
])
open(os.path.join(work_dir, "xds.log"), "w").write("")
# Run XDS
rotate_file(spot_xds)
call("xds", wdir=work_dir, stdout=open(os.path.join(work_dir, "xds.log"), "a"))
# Extract results
spots = [] # (x, y, d, intensity)
if os.path.isfile(spot_xds):
for l in open(spot_xds):
x, y, z, intensity = map(lambda x:float(x), l.strip().split())
d = coord_to_resol(x, y, im.header)
spots.append((x, y, d, intensity))
# Delete dir
shutil.rmtree(work_dir)
return spots
def reindex_with_specified_symm_worker(wd,
wdr,
topdir,
log_out,
reference_symm,
sgnum_laue,
prep_dials_files=False):
"""
wd: directory where XDS file exists
wdr: wd to return; a directory where transformed file should be saved.
If wd!=wdr, files in wd/ are unchanged during procedure. Multiprocessing is unsafe when wd==wdr.
"""
out = StringIO()
print >> out, "%s:" % os.path.relpath(wd, topdir),
# Find appropriate data
xac_file = util.return_first_found_file(
("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
"XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
"XDS_ASCII.HKL.org", "XDS_ASCII.HKL", "DIALS.HKL.org", "DIALS.HKL"),
wd=wd)
if xac_file is None:
print >> out, "Can't find XDS_ASCII file in %s" % wd
log_out.write(out.getvalue())
log_out.flush()
return (wdr, None)
if xac_file.endswith(".org"): xac_file_out = xac_file[:-4]
else: xac_file_out = xac_file
xac = XDS_ASCII(xac_file, read_data=False)
print >> out, "%s %s (%s)" % (os.path.basename(xac_file),
xac.symm.space_group_info(), ",".join(
map(lambda x: "%.2f" % x,
xac.symm.unit_cell().parameters())))
if xac.symm.reflection_intensity_symmetry(
False).space_group_info().type().number() == sgnum_laue:
if xac.symm.unit_cell().is_similar_to(
reference_symm.unit_cell(), 0.1,
10): # XXX Check unit cell consistency!!
print >> out, " Already scaled with specified symmetry"
log_out.write(out.getvalue())
log_out.flush()
if wd != wdr: shutil.copy2(xac_file, wdr)
if prep_dials_files and "DIALS.HKL" not in xac_file:
prepare_dials_files(wd, out, moveto=wdr)
return (wdr, (numpy.array(xac.symm.unit_cell().parameters()),
os.path.join(wdr, os.path.basename(xac_file))))
cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
20) # XXX ISN'T THIS TOO LARGE?
if len(cosets.combined_cb_ops()) == 0:
print >> out, "Can't find operator:"
xac.symm.show_summary(out, " ")
reference_symm.show_summary(out, " ")
log_out.write(out.getvalue())
log_out.flush()
return (wdr, None)
if wd == wdr:
dest = tempfile.mkdtemp(prefix="multiprep", dir=wd)
else:
dest = wdr
hklout = os.path.join(dest, os.path.basename(xac_file_out))
newcell = xac.write_reindexed(op=cosets.combined_cb_ops()[0],
space_group=reference_symm.space_group(),
hklout=hklout)
if "DIALS.HKL" in os.path.basename(xac_file):
outstr = 'output.experiments="%sreindexed_experiments.json" ' % os.path.join(
dest, "")
outstr += 'output.reflections="%sreindexed_reflections.pickle" ' % os.path.join(
dest, "")
for f in ("experiments.json", "indexed.pickle"):
if not os.path.isfile(os.path.join(os.path.dirname(xac_file), f)):
continue
util.call(
'dials.reindex %s change_of_basis_op=%s space_group="%s" %s' %
(f, cosets.combined_cb_ops()[0].as_abc(),
reference_symm.space_group_info(), outstr),
wdir=os.path.dirname(xac_file))
elif prep_dials_files:
prepare_dials_files(wd,
out,
space_group=reference_symm.space_group(),
reindex_op=cosets.combined_cb_ops()[0],
moveto=dest)
newcell_str = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
print >> out, " Reindexed to transformed cell: %s with %s" % (
newcell_str, cosets.combined_cb_ops()[0].as_hkl())
log_out.write(out.getvalue())
log_out.flush()
if wd == wdr:
for f in glob.glob(os.path.join(dest, "*")):
f_in_wd = os.path.join(wd, os.path.basename(f))
if os.path.exists(f_in_wd) and not os.path.exists(f_in_wd +
".org"):
os.rename(f_in_wd, f_in_wd + ".org")
os.rename(f, f_in_wd)
shutil.rmtree(dest)
ret = (numpy.array(newcell.parameters()),
os.path.join(wd, os.path.basename(xac_file_out)))
else:
ret = (numpy.array(newcell.parameters()), hklout)
return (wdr, ret)
# Check all needed files exist
if not check_needed_files():
sys.exit(1)
xdsinp = "XDS.INP"
workdir = "scale_exp"
os.mkdir(workdir)
os.mkdir(os.path.join(workdir, "scaled"))
anomalous = is_anomalous(xdsinp)
# *. INTEGRATE_forscale.mtz
call("pointless -copy xdsin INTEGRATE.HKL hklout %s/INTEGRATE_forscale.mtz" % workdir)
# 1. Backup XDS.INP, etc. (Make copies; not renaming)
bk_prefix = make_backup(backup_needed)
try:
# CORRECT with a
modify_xdsinp(xdsinp, inp_params=[("JOB","CORRECT"),
("CORRECTIONS", None),
("NBATCH", None),
("MINIMUM_I/SIGMA", None),
("WFAC1", None)
])
call("xds", stdout=open("CORRECT_a_xds.log", "w"))