Example #1
0
    def run_cycles(self, xds_files):
        self.removed_files = []
        self.removed_reason = {}
        print >> self.out, "********************* START FUNCTION ***********************"

        # Remove unconnected files here.
        remove_idxes = find_unconnected_xds_files(xds_files, min_ios=3, d_min=self.d_min)
        print >> self.out, "DEBUG:: Need to remove unconnected %d files" % len(remove_idxes)
        for i in sorted(remove_idxes):
            print >> self.out, " %.3d %s" % (i + 1, xds_files[i])
            self.removed_files.append(xds_files[i])
            self.removed_reason[xds_files[i]] = "no_common_refls"

        keep_idxes = filter(lambda x: x not in remove_idxes, xrange(len(xds_files)))
        self.run_cycle(map(lambda i: xds_files[i], keep_idxes))

        # Final cycle; average cells and merge again.
        used_files = set(xds_files).difference(set(self.removed_files))
        sg, cell = self.average_cells(used_files)
        print >> self.out, "Final scaling with averaged cell."
        print >> self.out, "Averaged cell= %s" % (cell)

        # call mtzutils to change cell!
        mtz_finalcell = "pointless_finalcell.mtz"
        util.call(
            cmd="mtzutils hklin %s hklout %s" % (os.path.relpath(self.mtzin, self.workdir), mtz_finalcell),
            wdir=self.workdir,
            expects_in=[os.path.relpath(self.mtzin, self.workdir)],
            expects_out=[mtz_finalcell],
            stdin="cell %s\nend\n" % cell,
            stdout=open(os.path.join(self.workdir, "mtzutils.log"), "w"),
        )

        inp_str = ""
        for i, f in enumerate(used_files):
            brange = self.batch_info[f]
            inp_str += "RUN %3d BATCH %4d to %4d\n" % (i + 1, brange[0], brange[1])

        aimless.run_aimless(
            mtzin=mtz_finalcell,
            wdir=self.workdir,
            anomalous=self.anomalous_flag,
            d_min=self.d_min,
            prefix="aimless",
            add_stdin=inp_str,
        )

        ctruncate_arg = "-hklin aimless.mtz -hklout ctruncate.mtz -colin '/*/*/[IMEAN,SIGIMEAN]'"
        if self.anomalous_flag:
            ctruncate_arg += " -colano '/*/*/[I(+),SIGI(+),I(-),SIGI(-)]'"

        util.call(
            cmd="ctruncate %s" % ctruncate_arg,
            wdir=self.workdir,
            expects_in=["aimless.mtz"],
            expects_out=["ctruncate.mtz"],
            stdout=open(os.path.join(self.workdir, "ctruncate.log"), "w"),
        )

        return self.removed_files, self.removed_reason
Example #2
0
def run_blend(wdir, xds_ascii_files, logout="blend_a.log"):
    ofs_lst = open(os.path.join(wdir, "files.lst"), "w")
    map(lambda x: ofs_lst.write(os.path.relpath(x, wdir)+"\n"), xds_ascii_files)
    ofs_lst.close()
    util.call(blend_comm, "-aDO files.lst", stdin="tolerance 10\n",
              wdir=wdir,
              stdout=open(os.path.join(wdir, logout), "w"))
Example #3
0
def run_blend(wdir, xds_ascii_files, logout="blend_a.log"):
    ofs_lst = open(os.path.join(wdir, "files.lst"), "w")
    map(lambda x: ofs_lst.write(os.path.relpath(x, wdir)+"\n"), xds_ascii_files)
    ofs_lst.close()
    util.call(blend_comm, "-aDO files.lst", stdin="tolerance 10\n",
              wdir=wdir,
              stdout=open(os.path.join(wdir, logout), "w"))
Example #4
0
def run(search_dir, pf_logfiles=None):
    for img_template, min_frame, max_frame in dataset.find_data_sets(search_dir,skip_symlinks=False):

        print "Dataset found:"
        print " NAME_TEMPLATE_OF_DATA_FRAMES= %s" % img_template
        print " DATA_RNAGE= %d %d" % (min_frame, max_frame)

        pdir, ppath = os.path.split(img_template)
        if not os.path.exists(os.path.join(pdir, "xds")):
            os.mkdir(os.path.join(pdir, "xds"))
        else:
            assert os.path.isdir(os.path.join(pdir, "xds"))

        prefix = ppath[:ppath.index("?")]
        if not prefix.endswith("_"):
            prefix += "_"

        if pf_logfiles is None:
            wdir = os.path.join(pdir, "xds", "xds_%s%d-%d" % (prefix, min_frame, max_frame))
            if os.path.isfile(os.path.join(wdir, "XDS.INP")):
                print " Already exist:", os.path.join(wdir, "XDS.INP")
                continue

            os.mkdir(wdir)

            cmd = 'generate_XDS.INP "%s"' % os.path.relpath(img_template, wdir)
            print " launching", cmd
            call(cmd=cmd, wdir=wdir)
        else:
            for minf, maxf in split_runs_based_on_pf_logfiles(pf_logfiles):
                if maxf < min_frame or max_frame < minf:
                    continue
                if minf < min_frame:
                    minf = min_frame
                if maxf > max_frame:
                    maxf = max_frame

                wdir = os.path.join(pdir, "xds", "xds_%s%d-%d" % (prefix, minf, maxf))

                if os.path.isfile(os.path.join(wdir, "XDS.INP")):
                    print "Already exist:", os.path.isfile(os.path.join(wdir, "XDS.INP"))
                    continue

                os.mkdir(wdir)

                cmd = 'generate_XDS.INP "%s"' % os.path.relpath(img_template, wdir)
                print " launching", cmd
                call(cmd=cmd, wdir=wdir)
                # XXX dirty hacks..
                inp = open(os.path.join(wdir, "XDS.INP")).readlines()
                ofs = open(os.path.join(wdir, "XDS.INP"), "w")
                for l in inp:
                    if l.startswith("DATA_RANGE="):
                        l = "DATA_RANGE= %d %d\n" % (minf, maxf)
                    if l.startswith("SPOT_RANGE="):
                        l = "SPOT_RANGE= %d %d\n" % (minf, (maxf-minf)/2+minf)
                    ofs.write(l)
                ofs.close()
        print
Example #5
0
def run_blend0R(wdir, xds_ascii_files, logout="blend0.log"):
    ofs_cell = open(os.path.join(wdir, "forR_macropar.dat"), "w")
    ofs_lst = open(os.path.join(wdir, "xds_lookup_table.txt"), "w")
    ofs_files = open(os.path.join(wdir, "NEW_list_of_files.dat"), "w") # Just to avoid error
    for i, xac in enumerate(xds_ascii_files):
        try:
            symm = xds_ascii.XDS_ASCII(xac, read_data=False).symm
        except:
            symm =  any_reflection_file(xac).as_miller_arrays()[0].crystal_symmetry()
        cell = " ".join(map(lambda x: "%7.3f"%x, symm.unit_cell().parameters()))
        sgnum =   int(symm.space_group().type().number())
        ofs_lst.write("dataset_%.3d %s\n" % (i, xac))
        ofs_cell.write("%4d %s 0 0 0 \"%s\"\n" % (i+1, cell, sg_to_cent[sgnum]))
        ofs_files.write("%s\n"%xac)

    ofs_lst.close()
    ofs_cell.close()
    ofs_files.close()

    shutil.copyfile(os.path.join(wdir, "forR_macropar.dat"),
                    os.path.join(wdir, "forR_macropar.dat.bak"))

    pathblend = "modules/yamtbx/blend/R/blend0.R"
    useNC = os.getenv("BLEND_USE_NCDIST","no")
    useSF = os.getenv("BLEND_USE_SFDIST","no")
    if useNC != "no":
        pathblend = "modules/yamtbx/blend/R/blend0NC.R"

    if useSF != "no":
        pathblend = "modules/yamtbx/blend/R/blend0SF.R"

    util.call("Rscript", os.path.join(os.environ["PHENIX"], pathblend),
              wdir=wdir,
              stdout=open(os.path.join(wdir, logout), "w"))

    open(os.path.join(wdir, "hctojson.R"), "w").write("""\
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
  labels<-hc$labels
  merge<-data.frame(hc$merge)
  for (i in (1:nrow(merge))) {
    if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
    else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
  }
  eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
  return(JSON)
}

load("BLEND0.RData",.GlobalEnv)
JSON<-HCtoJSON(npar.hc_ward)
cat(JSON, file="dendro.json")
""")
    util.call("Rscript", "hctojson.R",
              wdir=wdir,
              stdout=open(os.path.join(wdir, logout), "a"))
Example #6
0
def run_xscale(xscale_inp):
    ftable = {}
    outfile = None
    count = 0
    wdir = os.path.dirname(os.path.abspath(xscale_inp))

    os.rename(xscale_inp, xscale_inp+".org")
    ofs = open(xscale_inp, "w")

    # Check line length and make symlink if needed
    for l in open(xscale_inp+".org"):
        ll = l[:l.index("!")] if "!" in l else l
        if "OUTPUT_FILE=" in ll:
            outfile = ll[ll.index("=")+1:].strip()
        if "INPUT_FILE=" in ll and len(l) > 132: # one line is limited to 131 characters!
            filename = ll[ll.index("=")+1:].strip()
            if "*" in filename: filename = filename[filename.index("*")+1:].strip()
            assert " " not in filename
            lnkf = "lnk%.6d.hkl" % count
            assert not os.path.isfile(os.path.join(wdir, lnkf))
            filename_abs = os.path.normpath(os.path.join(wdir, filename)) if not os.path.isabs(filename) else filename
            os.symlink(filename_abs, os.path.join(wdir, lnkf))
            print "xscale: %s -> %s" % (lnkf, filename)
            count += 1
            ftable[lnkf] = filename
            l = l.replace(filename, lnkf)
        ofs.write(l)

    ofs.close()
    assert outfile is not None

    if len(ftable) == 0:
        os.rename(xscale_inp+".org", xscale_inp)

    # Run xscale
    util.call(xscale_comm, wdir=wdir)

    # Replace file names if needed
    if len(ftable) > 0:
        for f in "XSCALE.LP", outfile:
            f = os.path.join(wdir, f)
            if not os.path.isfile(f): continue

            os.rename(f, f+".org")
            ofs = open(f, "w")
            for l in open(f+".org"):
                for lfn in ftable: l = l.replace(lfn, ftable[lfn])
                ofs.write(l)
            ofs.close()

        for lfn in ftable:
            os.remove(os.path.join(wdir, lfn))

        os.rename(xscale_inp+".org", xscale_inp)
def work(rootdir, xdsinp, orgxy):
    workdir = os.path.join(rootdir, "bs_x%+.1f_y%+.1f" % orgxy)
    inpdir = os.path.normpath(os.path.dirname(xdsinp))
    print workdir
    os.makedirs(workdir)
    shutil.copyfile(os.path.join(inpdir, "SPOT.XDS"), os.path.join(workdir, "SPOT.XDS"))
    shutil.copyfile(xdsinp, os.path.join(workdir, "XDS.INP"))
    modify_xdsinp(os.path.join(workdir, "XDS.INP"), inp_params=[("JOB", "IDXREF"),
                                                                ("ORGX", orgxy[0]),
                                                                ("ORGY", orgxy[1]),
                                                                ])
    call("xds", wdir=workdir)
Example #8
0
def run_blend0R(wdir, xds_ascii_files, logout="blend0.log"):
    ofs_cell = open(os.path.join(wdir, "forR_macropar.dat"), "w")
    ofs_lst = open(os.path.join(wdir, "xds_lookup_table.txt"), "w")
    ofs_files = open(os.path.join(wdir, "NEW_list_of_files.dat"),
                     "w")  # Just to avoid error
    for i, xac in enumerate(xds_ascii_files):
        symm = xds_ascii.XDS_ASCII(xac, read_data=False).symm
        cell = " ".join(
            map(lambda x: "%7.3f" % x,
                symm.unit_cell().parameters()))
        ofs_lst.write("dataset_%.3d %s\n" % (i, xac))
        ofs_cell.write("%4d %s 0 0 0\n" % (i + 1, cell))
        ofs_files.write("%s\n" % xac)

    ofs_lst.close()
    ofs_cell.close()
    ofs_files.close()

    shutil.copyfile(os.path.join(wdir, "forR_macropar.dat"),
                    os.path.join(wdir, "forR_macropar.dat.bak"))

    util.call("Rscript",
              os.path.join(os.environ["CCP4"], "share/blend/R/blend0.R"),
              wdir=wdir,
              stdout=open(os.path.join(wdir, logout), "w"))

    open(os.path.join(wdir, "hctojson.R"), "w").write("""\
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
  labels<-hc$labels
  merge<-data.frame(hc$merge)
  for (i in (1:nrow(merge))) {
    if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
    else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
  }
  eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
  return(JSON)
}

load("BLEND0.RData",.GlobalEnv)
JSON<-HCtoJSON(npar.hc_ward)
cat(JSON, file="dendro.json")
""")
    util.call("Rscript",
              "hctojson.R",
              wdir=wdir,
              stdout=open(os.path.join(wdir, logout), "a"))
Example #9
0
def run_xtriage_in_module_if_possible(args,  wdir):
    try:
        from mmtbx.scaling import xtriage
        cwd_org = os.getcwd()
        try:
            os.chdir(wdir)
            xtriage.run(args, command_name="mmtbx.xtriage")
        except:
            print traceback.format_exc()
        finally:
            os.chdir(cwd_org)
    except ImportError:
        call("phenix.xtriage", arg=" ".join(args),
             stdin=None, stdout=sys.stdout, wdir=wdir)
Example #10
0
def tst_h5toxds():
    print "Testing H5ToXds.."
    rcode, out, err = util.call("H5ToXds")
    ignore_msg = "(You can ignore this if you don't process hdf5 files which usually mean Eiger data)"

    if rcode == 127:  # 127 is "command not found".
        print "  Not installed. NG %s" % ignore_msg
        return False

    import numpy
    from yamtbx.dataproc.eiger import make_dummy_h5_for_test
    from yamtbx.dataproc import cbf

    tmpdir = util.get_temp_local_dir("h5test")
    data = numpy.random.randint(0, 65535,
                                size=100).astype(numpy.uint32).reshape(
                                    (1, 10, 10))
    master_h5 = make_dummy_h5_for_test(tmpdir, data)
    rcode, out, err = util.call("H5ToXds",
                                "%s 1 1.cbf" % os.path.basename(master_h5),
                                wdir=tmpdir)
    cbfout = os.path.join(tmpdir, "1.cbf")
    if not os.path.isfile(cbfout):
        print "  H5ToXds exists, but not works. Probably Dectris original H5ToXds? Test it with real h5 file. %s" % ignore_msg
        if out.strip():
            print "  -- stdout:"
            print out
        if err.strip():
            print "  -- stderr:"
            print err

        shutil.rmtree(tmpdir)
        return False

    data_read, _, _ = cbf.load_minicbf_as_numpy(cbfout)
    shutil.rmtree(tmpdir)

    if numpy.all(data_read.flatten() == data.flatten()):
        print "  OK"
        return True
    else:
        print "  H5ToXds exists, but not correctly works. Probably Dectris original H5ToXds? Test it with real h5 file. %s" % ignore_msg
        if out.strip():
            print "  -- stdout:"
            print out
        if err.strip():
            print "  -- stderr:"
            print err

        return False
Example #11
0
def work(rootdir, xdsinp, orgxy):
    workdir = os.path.join(rootdir, "bs_x%+.1f_y%+.1f" % orgxy)
    inpdir = os.path.normpath(os.path.dirname(xdsinp))
    print workdir
    os.makedirs(workdir)
    shutil.copyfile(os.path.join(inpdir, "SPOT.XDS"),
                    os.path.join(workdir, "SPOT.XDS"))
    shutil.copyfile(xdsinp, os.path.join(workdir, "XDS.INP"))
    modify_xdsinp(os.path.join(workdir, "XDS.INP"),
                  inp_params=[
                      ("JOB", "IDXREF"),
                      ("ORGX", orgxy[0]),
                      ("ORGY", orgxy[1]),
                  ])
    call("xds", wdir=workdir)
Example #12
0
def tst_R():
    print "Testing R.."

    rcode, out, err = util.call("Rscript", '-e "print(cos(pi))"')
    if rcode != 0 or out.strip() != '[1] -1':
        print "  Rscript is not avaiable. NG"
        return False

    rcode, out, err = util.call("Rscript", '-e "library(rjson)"')
    if rcode != 0:
        print "  rjson is not installed. NG"
        return False

    print "  OK"
    return True
def tst_R():
    print "Testing R.."

    rcode, out, err = util.call("Rscript", '-e "print(cos(pi))"')
    if rcode != 0 or out.strip() != '[1] -1':
        print "  Rscript is not avaiable. NG"
        return False
    
    rcode, out, err = util.call("Rscript", '-e "library(rjson)"')
    if rcode != 0:
        print "  rjson is not installed. NG"
        return False
    
    print "  OK"
    return True
Example #14
0
def run_aimless(prefix, wdir):
    script_body = """\
pref=%s
mtzin=../${pref}_forscale.mtz
scaled=${pref}_aimless.mtz
truncated=${pref}_aimless_truncate.mtz

#lowres=$2
#highres=$3

aimless \
 HKLIN $mtzin \
 HKLOUT $scaled \
 SCALES ${pref}.scala \
 ROGUES ${pref}_scala_rogues.log \
 NORMPLOT ${pref}_scala_normplot.xmgr \
 ANOMPLOT ${pref}_scala_anomplot.xmgr \
 PLOT ${pref}_scala_surface_plot.plt \
 CORRELPLOT ${pref}_scala_correlplot.xmgr \
 ROGUEPLOT ${pref}_scala_rogueplot.xmgr \
<<eoi
!resolution low $lowres high $highres
!exclude batch ** to **
bins 20
!run 1 batch 1 to 750
!scales rotation spacing 5.000000 bfactor on tails
!cycles 5
""" % (prefix)

    if anomalous:
        script_body += """\
anomalous on
eoi

ctruncate -hklin $scaled -hklout $truncated -colin "/*/*/[IMEAN,SIGIMEAN]" -colano "/*/*/[I(+),SIGI(+),I(-),SIGI(-)]"
"""
    else:
        script_body += """\
anomalous off
eoi

ctruncate -hklin $scaled -hklout $truncated -colin "/*/*/[IMEAN,SIGIMEAN]"
"""
    open(os.path.join(wdir, "run_aimless.sh"),"w").write(script_body)

    call("sh run_aimless.sh",
         stdout=open(os.path.join(wdir, "aimless_truncate.log"), "w"),
         wdir=wdir)
Example #15
0
def run_aimless(prefix, wdir):
    script_body = """\
pref=%s
mtzin=../${pref}_forscale.mtz
scaled=${pref}_aimless.mtz
truncated=${pref}_aimless_truncate.mtz

#lowres=$2
#highres=$3

aimless \
 HKLIN $mtzin \
 HKLOUT $scaled \
 SCALES ${pref}.scala \
 ROGUES ${pref}_scala_rogues.log \
 NORMPLOT ${pref}_scala_normplot.xmgr \
 ANOMPLOT ${pref}_scala_anomplot.xmgr \
 PLOT ${pref}_scala_surface_plot.plt \
 CORRELPLOT ${pref}_scala_correlplot.xmgr \
 ROGUEPLOT ${pref}_scala_rogueplot.xmgr \
<<eoi
!resolution low $lowres high $highres
!exclude batch ** to **
bins 20
!run 1 batch 1 to 750
!scales rotation spacing 5.000000 bfactor on tails
!cycles 5
""" % (prefix)

    if anomalous:
        script_body += """\
anomalous on
eoi

ctruncate -hklin $scaled -hklout $truncated -colin "/*/*/[IMEAN,SIGIMEAN]" -colano "/*/*/[I(+),SIGI(+),I(-),SIGI(-)]"
"""
    else:
        script_body += """\
anomalous off
eoi

ctruncate -hklin $scaled -hklout $truncated -colin "/*/*/[IMEAN,SIGIMEAN]"
"""
    open(os.path.join(wdir, "run_aimless.sh"), "w").write(script_body)

    call("sh run_aimless.sh",
         stdout=open(os.path.join(wdir, "aimless_truncate.log"), "w"),
         wdir=wdir)
Example #16
0
def run(params, xac_files):
    if len(xac_files) == 0:
        print "No XDS_ASCII.HKL files provided."
        return

    xscale_inp_head = "!MINIMUM_I/SIGMA= 3\n\n" 

    infos = {}
    d_max, d_min = 0, 100
    cells = []
    for xds_ascii in xac_files:
        info = get_xac_info(xds_ascii)
        infos[xds_ascii] = info

        resrng = map(float, info["resol_range"].split())
        d_max = max(d_max, resrng[0])
        d_min = min(d_min, resrng[1])
        cells.append(map(float, info["cell"].split()))

    if params.d_min is not None:
        d_min = max(params.d_min, d_min)

    if params.cell == "average":
        cell_sum = reduce(lambda x,y: map(lambda a: a[0]+a[1], zip(x,y)), cells)
        cell_mean = map(lambda x: x/float(len(cells)), cell_sum)

        if params.sgnum is not None: sgnum = str(params.sgnum)
        else: sgnum = infos[xac_files[0]]["spgr_num"]

        xscale_inp_head += " SPACE_GROUP_NUMBER= %s\n" % sgnum
        xscale_inp_head += " UNIT_CELL_CONSTANTS= %s\n" % " ".join(map(lambda x: "%.3f"%x, cell_mean))

    #if anomalous_flag is not None:
    #    xscale_inp_head += " FRIEDEL'S_LAW= %s\n" % ("FALSE" if anomalous_flag else "TRUE")
    
    xscale_inp_head += make_shells(d_max, d_min, params.nbins) + "\n"
    xscale_inp_head += " OUTPUT_FILE= %s\n\n" % params.output

    xscale_inp = os.path.join(params.workdir, "XSCALE.INP")
    inp_out = open(xscale_inp, "w")
    inp_out.write(xscale_inp_head)

    for xds_ascii in xac_files:
        inp_out.write("  INPUT_FILE= %s\n" % os.path.relpath(xds_ascii, params.workdir))
        inp_out.write("  ! INCLUDE_RESOLUTION_RANGE= %s\n\n" % infos[xds_ascii]["resol_range"])

    inp_out.close()
    util.call(xscale_comm, wdir=params.workdir, stdout=sys.stdout)
Example #17
0
    def btnUnmount_click(self, ev):
        def df(d):
            ret, out, err = util.call("df", '"%s"'%d) # XXX Linux specific?
            out_lines = filter(lambda x: x, map(lambda x: x.strip(), out.splitlines()))
            if len(out_lines) < 2: return None
            sp = out_lines[1].split()
            if not sp: return None
            return sp[0]

        ddir = self.txtDestDir.GetValue()
        if not ddir or not os.path.isabs(ddir) or not os.path.isdir(ddir):
            wx.MessageDialog(None, "Invalid directory.", "Error", style=wx.OK).ShowModal()
            return

        mounted_dev = df(ddir)

        if not mounted_dev:
            wx.MessageDialog(None, "Invalid directory.", "Error", style=wx.OK).ShowModal()
            return

        if wx.MessageDialog(None, "Unmounting %s. Are you sure?" % mounted_dev,
                            "Confirmation", style=wx.OK|wx.CANCEL).ShowModal() != wx.ID_OK:
            return

        ret, out, err = util.call("udisks", "--unmount %s" % mounted_dev) # XXX Linux specific?

        if out:
            wx.MessageDialog(None, out, "Error", style=wx.OK).ShowModal()
        elif mounted_dev == df(ddir):
            wx.MessageDialog(None, "Unmount failed.", "Error", style=wx.OK).ShowModal()
        else:
            wx.MessageDialog(None, "Successfully unmounted.", "OK", style=wx.OK).ShowModal()
Example #18
0
def run_aimless(mtzin, wdir, anomalous=False, d_min=None, prefix=None, add_stdin=None):
    if prefix is None:
        prefix = os.path.splitext(os.path.basename(mtzin))[0] + "_aimless"

    args = """\
HKLIN %(mtzin)s \
HKLOUT %(prefix)s.mtz \
SCALES %(prefix)s.scala \
ROGUES %(prefix)s_rogues.log \
NORMPLOT %(prefix)s_normplot.xmgr \
ANOMPLOT %(prefix)s_anomplot.xmgr \
PLOT %(prefix)s_surface_plot.plt \
CORRELPLOT %(prefix)s_correlplot.xmgr \
ROGUEPLOT %(prefix)s_rogueplot.xmgr \
""" % dict(mtzin=mtzin, prefix=prefix)

    stdin = ""
    if d_min is not None:
        stdin += "resolution high %.3f\n" % d_min
    if anomalous:
        stdin += "anomalous on\n"
    if add_stdin is not None:
        stdin += add_stdin

    exitcode, output, err = util.call(aimless_comm, args,
                                      stdin=stdin, wdir=wdir)

    open(os.path.join(wdir, prefix+".log"), "w").write(output)
Example #19
0
 def df(d):
     ret, out, err = util.call("df", '"%s"'%d) # XXX Linux specific?
     out_lines = filter(lambda x: x, map(lambda x: x.strip(), out.splitlines()))
     if len(out_lines) < 2: return None
     sp = out_lines[1].split()
     if not sp: return None
     return sp[0]
Example #20
0
def run_xtriage_in_module_if_possible(args, wdir):
    try:
        from mmtbx.scaling import xtriage
        cwd_org = os.getcwd()
        try:
            os.chdir(wdir)
            xtriage.run(args, command_name="mmtbx.xtriage")
        except:
            print traceback.format_exc()
        finally:
            os.chdir(cwd_org)
    except ImportError:
        call("phenix.xtriage",
             arg=" ".join(args),
             stdin=None,
             stdout=sys.stdout,
             wdir=wdir)
Example #21
0
def run_blend0R(wdir, xds_ascii_files, logout="blend0.log"):
    ofs_cell = open(os.path.join(wdir, "forR_macropar.dat"), "w")
    ofs_lst = open(os.path.join(wdir, "xds_lookup_table.txt"), "w")
    ofs_files = open(os.path.join(wdir, "NEW_list_of_files.dat"), "w") # Just to avoid error
    for i, xac in enumerate(xds_ascii_files):
        symm = xds_ascii.XDS_ASCII(xac, read_data=False).symm
        cell = " ".join(map(lambda x: "%7.3f"%x, symm.unit_cell().parameters()))
        ofs_lst.write("dataset_%.3d %s\n" % (i, xac))
        ofs_cell.write("%4d %s 0 0 0\n" % (i+1, cell))
        ofs_files.write("%s\n"%xac)

    ofs_lst.close()
    ofs_cell.close()
    ofs_files.close()

    shutil.copyfile(os.path.join(wdir, "forR_macropar.dat"),
                    os.path.join(wdir, "forR_macropar.dat.bak"))

    util.call("Rscript", os.path.join(os.environ["CCP4"], "share/blend/R/blend0.R"),
              wdir=wdir,
              stdout=open(os.path.join(wdir, logout), "w"))

    open(os.path.join(wdir, "hctojson.R"), "w").write("""\
# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
  labels<-hc$labels
  merge<-data.frame(hc$merge)
  for (i in (1:nrow(merge))) {
    if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
    else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
  }
  eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
  return(JSON)
}

load("BLEND0.RData",.GlobalEnv)
JSON<-HCtoJSON(npar.hc_ward)
cat(JSON, file="dendro.json")
""")
    util.call("Rscript", "hctojson.R",
              wdir=wdir,
              stdout=open(os.path.join(wdir, logout), "a"))
Example #22
0
def calc_cchalf_by_removing(wdir,
                            inp_head,
                            inpfiles,
                            with_sigma=False,
                            stat_bin="total",
                            nproc=1,
                            nproc_each=None,
                            batchjobs=None):
    assert not with_sigma  # Not supported now
    assert stat_bin in ("total", "outer")

    if not os.path.exists(wdir): os.makedirs(wdir)

    datout = open(os.path.join(wdir, "cchalf.dat"), "w")
    datout.write("idx exfile cc1/2(%s) Nuniq\n" % stat_bin)

    cchalf_list = []  # (i_ex, CC1/2, Nuniq)

    # Prep runs
    tmpdirs = map(
        lambda x: _calc_cchalf_by_removing_worker_1(wdir, inp_head, inpfiles,
                                                    x, nproc_each),
        xrange(len(inpfiles)))
    # Run XSCALE
    if batchjobs is not None:
        jobs = []
        for tmpdir in tmpdirs:
            job = batchjob.Job(tmpdir, "xscale.sh", nproc=nproc_each)
            job.write_script(xscale_comm)
            batchjobs.submit(job)
            jobs.append(job)

        batchjobs.wait_all(jobs)
    else:
        easy_mp.pool_map(fixed_func=lambda x: util.call(xscale_comm, wdir=x),
                         args=tmpdirs,
                         processes=nproc)
    # Finish runs
    cchalf_list = map(
        lambda x: _calc_cchalf_by_removing_worker_2(wdir, x[1], x[0], stat_bin
                                                    ), enumerate(tmpdirs))

    for iex, cchalf_exi, nuniq in cchalf_list:
        datout.write("%3d %s %.4f %d\n" %
                     (iex, inpfiles[iex], cchalf_exi, nuniq))

    cchalf_list.sort(key=lambda x: -x[1])
    print
    print "# Sorted table"
    for idx, cch, nuniq in cchalf_list:
        print "%3d %-.4f %4d %s" % (idx, cch, nuniq, inpfiles[idx])

    # Remove unuseful (failed) data
    cchalf_list = filter(lambda x: x[1] == x[1], cchalf_list)

    return cchalf_list
def tst_adxv():
    print "Testing Adxv.."

    rcode, out, err = util.call("adxv", '-help')
    if rcode != 0:
        print "  Adxv is not avaiable. NG"
        return False
    
    print "  OK"
    return True
Example #24
0
def tst_adxv():
    print "Testing Adxv.."

    rcode, out, err = util.call("adxv", '-help')
    if rcode != 0:
        print "  Adxv is not avaiable. NG"
        return False

    print "  OK"
    return True
Example #25
0
def run(params):
    xdsinp = "XDS.INP"
    workdir = os.getcwd()

    params.wfac1.sort()
    params.minpk.sort()
    digit_wfac1 = str(get_digit(params.wfac1))
    digit_minpk = str(get_digit(params.minpk))

    backup_needed = files.generated_by_CORRECT + ("XDS.INP", )

    bk_prefix = make_backup(backup_needed)

    try:
        for wfac1 in params.wfac1:
            for minpk in params.minpk:
                work_name = ("CORRECT_wfac1_%." + digit_wfac1 + "f_minpk_%." +
                             digit_minpk + "f") % (wfac1, minpk)
                modify_xdsinp(xdsinp,
                              inp_params=[
                                  ("JOB", "CORRECT"),
                                  ("WFAC1", wfac1),
                                  ("MINPK", minpk),
                              ])
                call("xds")
                # make .mtz file
                xds2mtz.xds2mtz(os.path.abspath("XDS_ASCII.HKL"),
                                dir_name=os.path.join(workdir,
                                                      "ccp4_" + work_name),
                                run_ctruncate=True,
                                run_xtriage=True)

                if params.mtz_free is not None:
                    copy_testflag(mtzfree=params.mtz_free,
                                  mtzin=os.path.join(workdir,
                                                     "ccp4_" + work_name,
                                                     "XDS_ASCII.mtz"))

                make_backup(backup_needed, work_name + "_")

    finally:
        # 6. Revert XDS.INP, etc.
        revert_files(backup_needed, bk_prefix)
Example #26
0
def run(hklin):
    arrays = filter(lambda x:x.is_xray_intensity_array(),
                    iotbx.file_reader.any_file(hklin).file_server.miller_arrays)

    # TODO Copy non-intensity arrays to new mtz!

    if len(arrays) == 0:
        print "No intensity array"
        return

    print "Intensity arrays:"
    for ar in arrays:
        print "", ar.info().label_string()
    print

    colin, colano = "", ""

    if len(arrays) == 1:
        coltmp = arrays[0].info().label_string()
        if arrays[0].anomalous_flag():
            colano = "/*/*/[%s]" % coltmp.replace(",merged","")
        else:
            colin = "/*/*/[%s]" % coltmp
    elif len(arrays) == 2 and sum(map(lambda x: x.anomalous_flag(), arrays)) == 1:
        tmpano = filter(lambda x: x.anomalous_flag(), arrays)[0]
        tmpnoano = filter(lambda x: not x.anomalous_flag(), arrays)[0]
        colano = "/*/*/[%s]" % tmpano.info().label_string().replace(",merged","")
        colin = "/*/*/[%s]" % tmpnoano.info().label_string()
    else:
        print "What should I do.. exiting."
        return

    hklout = os.path.splitext(os.path.basename(hklin))[0] + "_ctruncate.mtz"
    cmd = "ctruncate -hklin %s -hklout %s " % (hklin, hklout)
    if colin != "":
        cmd += "-colin '%s' " % colin
    if colano != "":
        cmd += "-colano '%s' " % colano

    call(cmd=cmd,
         stdout=open("ctruncate_%s.log" % os.path.splitext(os.path.basename(hklin))[0], "w")
         )
Example #27
0
def tst_dials():
    print "Testing dials (package).."
    rcode, out, err = util.call("dials.version")

    if rcode != 0:
        print "  Not installed. NG"
        return False

    print "\n".join(map(lambda x: "  " + x, out.splitlines()))
    print "  OK"
    return True
def tst_xdsstat():
    print "Testing XDSSTAT.."
    rcode, out, err = util.call("xdsstat")
    if rcode != 2:
        print "  Not installed. NG"
        return False

    if "XDSSTAT version" not in err:
        print "  Seems not working. NG"
        return False

    print "  OK"
    return True
Example #29
0
    def run_copy(self, hklout, wdir, hklin=None, xdsin=None, logout=None, tolerance=None):
        assert (hklin, xdsin).count(None) == 1
        stdin, junk = self.input_str(hklin, xdsin)
        

        if tolerance is not None:
            stdin += "tolerance %f\n" % tolerance
        
        exitcode, output, err = util.call(pointless_comm, "-copy hklout %s" % hklout,
                                          stdin=stdin, wdir=wdir)
        if logout is not None:
            open(logout, "w").write(output)
            return parse_pointless_output_for_runs(output)
def tst_xds():
    print "Testing XDS.."
    rcode, out, err = util.call("xds_par")
    if rcode != 0:
        print "  Not installed. NG"
        return False

    if "license expired" in out:
        print "  license expired. Get latest version. NG"
        return False

    print "  OK"
    return True
Example #31
0
def tst_xdsstat():
    print "Testing XDSSTAT.."
    rcode, out, err = util.call("xdsstat")
    if rcode != 2:
        print "  Not installed. NG"
        return False

    if "XDSSTAT version" not in err:
        print "  Seems not working. NG"
        return False

    print "  OK"
    return True
Example #32
0
def run(hklin):
    arrays = filter(lambda x: x.is_xray_intensity_array(), iotbx.file_reader.any_file(hklin).file_server.miller_arrays)

    # TODO Copy non-intensity arrays to new mtz!

    if len(arrays) == 0:
        print "No intensity array"
        return

    print "Intensity arrays:"
    for ar in arrays:
        print "", ar.info().label_string()
    print

    colin, colano = "", ""

    if len(arrays) == 1:
        coltmp = arrays[0].info().label_string()
        if arrays[0].anomalous_flag():
            colano = "/*/*/[%s]" % coltmp.replace(",merged", "")
        else:
            colin = "/*/*/[%s]" % coltmp
    elif len(arrays) == 2 and sum(map(lambda x: x.anomalous_flag(), arrays)) == 1:
        tmpano = filter(lambda x: x.anomalous_flag(), arrays)[0]
        tmpnoano = filter(lambda x: not x.anomalous_flag(), arrays)[0]
        colano = "/*/*/[%s]" % tmpano.info().label_string().replace(",merged", "")
        colin = "/*/*/[%s]" % tmpnoano.info().label_string()
    else:
        print "What should I do.. exiting."
        return

    hklout = os.path.splitext(os.path.basename(hklin))[0] + "_ctruncate.mtz"
    cmd = "ctruncate -hklin %s -hklout %s " % (hklin, hklout)
    if colin != "":
        cmd += "-colin '%s' " % colin
    if colano != "":
        cmd += "-colano '%s' " % colano

    call(cmd=cmd, stdout=open("ctruncate_%s.log" % os.path.splitext(os.path.basename(hklin))[0], "w"))
Example #33
0
def tst_xds():
    print "Testing XDS.."
    rcode, out, err = util.call("xds_par")
    if rcode != 0:
        print "  Not installed. NG"
        return False

    if "license expired" in out:
        print "  license expired. Get latest version. NG"
        return False

    print "  OK"
    return True
Example #34
0
def run(params):
    xdsinp = "XDS.INP"
    kwds = dict(get_xdsinp_keyword(xdsinp))
    orgx_org, orgy_org = map(float, (kwds["ORGX"], kwds["ORGY"]))

    dx, dy = params.dx, params.dy
    if params.unit == "mm":
        assert "QX" in kwds
        assert "QY" in kwds
        dx /= float(kwds["QX"])
        dy /= float(kwds["QY"])

    backup_needed = files.generated_by_IDXREF + ("XDS.INP",)
    bk_prefix = make_backup(backup_needed)
    try:
        results = []
        for i in xrange(-params.nx, params.nx+1):
            for j in xrange(-params.ny, params.ny+1):
                work_name = "bs_x%+.2d_y%+.2d" % (i, j)
                orgx = orgx_org + i * dx
                orgy = orgy_org + j * dy
                print "Trying", orgx, orgy

                modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                                  ("ORGX", orgx),
                                                  ("ORGY", orgy),
                                                  ])
                call("xds")
                make_backup(backup_needed, work_name+"_")

                results.append([work_name, orgx, orgy])

        for ret in results:
                print ret,
                analyze_result(ret[0]+"_IDXREF.LP")
                

    finally:
        revert_files(backup_needed, bk_prefix)
Example #35
0
    def run_copy(self, hklout, wdir, hklin=None, xdsin=None, logout=None, tolerance=None):
        assert (hklin, xdsin).count(None) == 1
        stdin, junk = self.input_str(hklin, xdsin)
        

        if tolerance is not None:
            stdin += "tolerance %f\n" % tolerance
        
        exitcode, output, err = util.call(pointless_comm, "-copy hklout %s" % hklout,
                                          stdin=stdin, wdir=wdir)
        if logout is not None:
            open(logout, "w").write(output)
            return parse_pointless_output_for_runs(output)
Example #36
0
def run(params):
    xdsinp = "XDS.INP"
    workdir = os.getcwd()

    params.wfac1.sort()
    params.minpk.sort()
    digit_wfac1 = str(get_digit(params.wfac1))
    digit_minpk = str(get_digit(params.minpk))

    backup_needed = files.generated_by_CORRECT + ("XDS.INP",)

    bk_prefix = make_backup(backup_needed)

    try:
        for wfac1 in params.wfac1:
            for minpk in params.minpk:
                work_name = ("CORRECT_wfac1_%." + digit_wfac1 + "f_minpk_%." + digit_minpk + "f") % (wfac1, minpk)
                modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"), ("WFAC1", wfac1), ("MINPK", minpk)])
                call("xds")
                # make .mtz file
                xds2mtz.xds2mtz(
                    os.path.abspath("XDS_ASCII.HKL"),
                    dir_name=os.path.join(workdir, "ccp4_" + work_name),
                    run_ctruncate=True,
                    run_xtriage=True,
                )

                if params.mtz_free is not None:
                    copy_testflag(
                        mtzfree=params.mtz_free, mtzin=os.path.join(workdir, "ccp4_" + work_name, "XDS_ASCII.mtz")
                    )

                make_backup(backup_needed, work_name + "_")

    finally:
        # 6. Revert XDS.INP, etc.
        revert_files(backup_needed, bk_prefix)
Example #37
0
    def run_for_symm(self, hklin=None, xdsin=None, logout=None, tolerance=None, d_min=None, choose_laue=None, xdsin_to_p1=False, setting="c2"):
        assert (hklin, xdsin).count(None) == 1
        assert setting.lower().startswith(("c2", "cell", "symm"))
        tmpfile = None

        out = open(logout, "w") if logout else null_out()
        
        # Merge in P1 before pointless for quick evaluation
        if xdsin and xdsin_to_p1:
            out.write("Merging in P1 symmetry before Pointless..\n")
            xac = xds_ascii.XDS_ASCII(xdsin)
            xac.remove_rejected()
            i_obs = xac.i_obs(anomalous_flag=True)
            i_obs.show_summary(out, "  ")
            i_obs = i_obs.apply_change_of_basis(change_of_basis="to_niggli_cell", out=out)[0]
            i_obs = i_obs.customized_copy(space_group_info=sgtbx.space_group_info("P1"))
            i_obs = i_obs.merge_equivalents(use_internal_variance=False).array()
            tmpfile = util.get_temp_filename(prefix="", suffix=os.path.basename(xdsin) + "_p1.mtz")
            i_obs.as_mtz_dataset(column_root_label="I").mtz_object().write(tmpfile)
            xdsin, hklin = None, tmpfile
            out.write("\n")
        
        filein, wdir = self.input_str(hklin, xdsin)

        # Need centre keyword?
        stdin = """\
SETTING %s
%s
""" % (setting, filein)

        if tolerance is not None:
            stdin += "tolerance %f\n" % tolerance
        if d_min is not None:
            stdin += "resolution high %f\n" % d_min
        if choose_laue is not None:
            stdin += "choose lauegroup %s\n" % choose_laue

        exitcode, output, err = util.call(pointless_comm, 
                                          stdin=stdin, wdir=wdir)
        
        parsed = parse_pointless_output_for_symm(output)
        out.write(output)

        if tmpfile and os.path.isfile(tmpfile):
            os.remove(tmpfile)
        
        return parsed
Example #38
0
def tst_xds():
    print "Testing XDS.."
    tmpdir = util.get_temp_local_dir("xdstest")
    rcode, out, err = util.call("xds_par", wdir=tmpdir)
    if tmpdir:
        shutil.rmtree(tmpdir)  # just in case; xds shouldn't make any files

    if rcode != 0:
        print "  Not installed. NG"
        return False

    if "license expired" in out:
        print "  license expired. Get latest version. NG"
        return False

    print "  OK"
    return True
Example #39
0
def check_xds_version():
    tmpdir = util.get_temp_local_dir("xdstest")
    rcode, out, err = util.call("xds", wdir=tmpdir)
    if tmpdir:
        shutil.rmtree(tmpdir)  # just in case; xds shouldn't make any files

    # Expected outout:
    # ***** XDS ***** (VERSION Mar 15, 2019  BUILT=20191211)   6-Jan-2020
    # Author: Wolfgang Kabsch
    # Copy licensed until 30-Sep-2020 to

    r = re.search("VERSION (.*[0-9]) *BUILT=(.*)\)", out)
    if r:
        ver, built = r.groups()
        return ver, built

    return None, None
Example #40
0
def calc_cchalf_by_removing(wdir, inp_head, inpfiles, with_sigma=False, stat_bin="total", nproc=1, nproc_each=None, batchjobs=None):
    assert not with_sigma # Not supported now
    assert stat_bin in ("total", "outer")

    if not os.path.exists(wdir): os.makedirs(wdir)

    datout = open(os.path.join(wdir, "cchalf.dat"), "w")
    datout.write("idx exfile cc1/2(%s) Nuniq\n" % stat_bin)

    cchalf_list = [] # (i_ex, CC1/2, Nuniq)

    # Prep runs
    tmpdirs = map(lambda x: _calc_cchalf_by_removing_worker_1(wdir, inp_head, inpfiles, x, nproc_each),
                  xrange(len(inpfiles)))
    # Run XSCALE 
    if batchjobs is not None:
        jobs = []
        for tmpdir in tmpdirs:
            job = batchjob.Job(tmpdir, "xscale.sh", nproc=nproc_each)
            job.write_script(xscale_comm)
            batchjobs.submit(job)
            jobs.append(job)

        batchjobs.wait_all(jobs)
    else:
        easy_mp.pool_map(fixed_func=lambda x: util.call(xscale_comm, wdir=x),
                         args=tmpdirs,
                         processes=nproc)
    # Finish runs
    cchalf_list = map(lambda x: _calc_cchalf_by_removing_worker_2(wdir, x[1], x[0], stat_bin), enumerate(tmpdirs))

    for iex, cchalf_exi, nuniq in cchalf_list:
        datout.write("%3d %s %.4f %d\n" % (iex, inpfiles[iex], cchalf_exi, nuniq))

    cchalf_list.sort(key=lambda x: -x[1])
    print
    print "# Sorted table"
    for idx, cch, nuniq in cchalf_list:
        print "%3d %-.4f %4d %s" % (idx, cch, nuniq, inpfiles[idx])

    # Remove unuseful (failed) data
    cchalf_list = filter(lambda x: x[1]==x[1], cchalf_list)

    return cchalf_list
Example #41
0
def unique(mtzin, mtzout, sg, wdir, logout):
    ##
    # unique -> cad -> mtzutil (EXCLUDE FUNI SIGFUNI)
    #
    m = mtzutil.MtzFile(os.path.join(wdir, mtzin))
    cell = m.get_cell_str()
    sg_org = m.get_spacegroup()[1]
    resol = min(m.get_resolution())

    call(
        cmd="unique",
        arg="hklout unique.mtz",
        wdir=wdir,
        stdin=
        "CELL %s\nSYMMETRY '%s'\nLABOUT F=FUNI SIGF=SIGFUNI\nRESOLUTION %f" %
        (cell, sg_org, resol),
        expects_in=[],
        expects_out=["unique.mtz"],
        stdout=logout,
    )

    call(
        cmd="cad",
        arg="hklin1 %s hklin2 %s hklout %s" %
        (mtzin, "unique.mtz", "unique_cad.mtz"),
        wdir=wdir,
        stdin="labin file 1 all\nlabin file 2 all\nend\n",
        expects_in=[mtzin, "unique.mtz"],
        expects_out=["unique_cad.mtz"],
        stdout=logout,
    )

    call(
        cmd="mtzutils",
        arg="hklin %s hklout %s" % ("unique_cad.mtz", mtzout),
        wdir=wdir,
        stdin="EXCLUDE FUNI SIGFUNI\nSYMMETRY %s\nRUN\n" % sg,
        expects_in=["unique_cad.mtz"],
        expects_out=[mtzout],
        stdout=logout,
    )

    os.remove(os.path.join(wdir, "unique.mtz"))
    os.remove(os.path.join(wdir, "unique_cad.mtz"))
Example #42
0
    def run_for_symm(self, hklin=None, xdsin=None, logout=None, tolerance=None, d_min=None):
        assert (hklin, xdsin).count(None) == 1
        filein, wdir = self.input_str(hklin, xdsin)

        # Need centre keyword?
        stdin = """\
SETTING SYMMETRY-BASED
%s
""" % filein

        if tolerance is not None:
            stdin += "tolerance %f\n" % tolerance
        if d_min is not None:
            stdin += "resolution high %f\n" % d_min

        exitcode, output, err = util.call(pointless_comm, 
                                          stdin=stdin, wdir=wdir)
        
        parsed = parse_pointless_output_for_symm(output)

        if logout is not None:
            open(logout, "w").write(output)
        
        return parsed
Example #43
0
def unique(mtzin, mtzout, wdir, logout):
    ##
    # unique -> cad -> mtzutil (EXCLUDE FUNI SIGFUNI)
    #
    m = mtzutil.MtzFile(os.path.join(wdir, mtzin))
    cell = m.get_cell_str()
    sg = m.get_spacegroup()[1]
    resol = min(m.get_resolution())

    call(
        cmd="unique",
        arg="hklout unique.mtz",
        wdir=wdir,
        stdin="CELL %s\nSYMMETRY '%s'\nLABOUT F=FUNI SIGF=SIGFUNI\nRESOLUTION %f" % (cell, sg, resol),
        expects_in=[],
        expects_out=["unique.mtz"],
        stdout=logout,
    )

    call(
        cmd="cad",
        arg="hklin1 %s hklin2 %s hklout %s" % (mtzin, "unique.mtz", "unique_cad.mtz"),
        wdir=wdir,
        stdin="labin file 1 all\nlabin file 2 all\nend\n",
        expects_in=[mtzin, "unique.mtz"],
        expects_out=["unique_cad.mtz"],
        stdout=logout,
    )

    call(
        cmd="mtzutils",
        arg="hklin %s hklout %s" % ("unique_cad.mtz", mtzout),
        wdir=wdir,
        stdin="EXCLUDE FUNI SIGFUNI\nRUN\n",
        expects_in=["unique_cad.mtz"],
        expects_out=[mtzout],
        stdout=logout,
    )

    os.remove(os.path.join(wdir, "unique.mtz"))
    os.remove(os.path.join(wdir, "unique_cad.mtz"))
Example #44
0
def run(search_dir, pf_logfiles=None):
    for img_template, min_frame, max_frame in dataset.find_data_sets(
            search_dir, skip_symlinks=False):

        print "Dataset found:"
        print " NAME_TEMPLATE_OF_DATA_FRAMES= %s" % img_template
        print " DATA_RNAGE= %d %d" % (min_frame, max_frame)

        pdir, ppath = os.path.split(img_template)
        if not os.path.exists(os.path.join(pdir, "xds")):
            os.mkdir(os.path.join(pdir, "xds"))
        else:
            assert os.path.isdir(os.path.join(pdir, "xds"))

        prefix = ppath[:ppath.index("?")]
        if not prefix.endswith("_"):
            prefix += "_"

        if pf_logfiles is None:
            wdir = os.path.join(pdir, "xds",
                                "xds_%s%d-%d" % (prefix, min_frame, max_frame))
            if os.path.isfile(os.path.join(wdir, "XDS.INP")):
                print " Already exist:", os.path.join(wdir, "XDS.INP")
                continue

            os.mkdir(wdir)

            cmd = 'generate_XDS.INP "%s"' % os.path.relpath(img_template, wdir)
            print " launching", cmd
            call(cmd=cmd, wdir=wdir)
        else:
            for minf, maxf in split_runs_based_on_pf_logfiles(pf_logfiles):
                if maxf < min_frame or max_frame < minf:
                    continue
                if minf < min_frame:
                    minf = min_frame
                if maxf > max_frame:
                    maxf = max_frame

                wdir = os.path.join(pdir, "xds",
                                    "xds_%s%d-%d" % (prefix, minf, maxf))

                if os.path.isfile(os.path.join(wdir, "XDS.INP")):
                    print "Already exist:", os.path.isfile(
                        os.path.join(wdir, "XDS.INP"))
                    continue

                os.mkdir(wdir)

                cmd = 'generate_XDS.INP "%s"' % os.path.relpath(
                    img_template, wdir)
                print " launching", cmd
                call(cmd=cmd, wdir=wdir)
                # XXX dirty hacks..
                inp = open(os.path.join(wdir, "XDS.INP")).readlines()
                ofs = open(os.path.join(wdir, "XDS.INP"), "w")
                for l in inp:
                    if l.startswith("DATA_RANGE="):
                        l = "DATA_RANGE= %d %d\n" % (minf, maxf)
                    if l.startswith("SPOT_RANGE="):
                        l = "SPOT_RANGE= %d %d\n" % (minf,
                                                     (maxf - minf) / 2 + minf)
                    ofs.write(l)
                ofs.close()
        print
Example #45
0
    def do_clustering(self, nproc=1, b_scale=False, use_normalized=False, html_maker=None):
        self.clusters = {}
        prefix = os.path.join(self.wdir, "cctable")
        assert (b_scale, use_normalized).count(True) <= 1

        if len(self.arrays) < 2:
            print "WARNING: less than two data! can't do cc-based clustering"
            self.clusters[1] = [float("nan"), [0]]
            return

        # Absolute scaling using Wilson-B factor 
        if b_scale:
            from mmtbx.scaling.matthews import p_vm_calculator
            from mmtbx.scaling.absolute_scaling import ml_iso_absolute_scaling
            
            ofs_wilson = open("%s_wilson_scales.dat"%prefix, "w")
            n_residues = p_vm_calculator(self.arrays.values()[0], 1, 0).best_guess
            ofs_wilson.write("# guessed n_residues= %d\n" % n_residues)
            ofs_wilson.write("file wilsonB\n")
            for f in self.arrays:
                arr = self.arrays[f]
                iso_scale_and_b = ml_iso_absolute_scaling(arr, n_residues, 0)
                wilson_b = iso_scale_and_b.b_wilson
                ofs_wilson.write("%s %.3f\n" % (f, wilson_b))
                if wilson_b > 0: # Ignoring data with B<0? is a bad idea.. but how..?
                    tmp = flex.exp(-2. * wilson_b * arr.unit_cell().d_star_sq(arr.indices())/4.)
                    self.arrays[f] = arr.customized_copy(data=arr.data()*tmp,
                                                         sigmas=arr.sigmas()*tmp)
            ofs_wilson.close()

        elif use_normalized:
            from mmtbx.scaling.absolute_scaling import kernel_normalisation
            for f in self.arrays:
                arr = self.arrays[f]
                normaliser = kernel_normalisation(arr, auto_kernel=True)
                self.arrays[f] = arr.customized_copy(data=arr.data()/normaliser.normalizer_for_miller_array,
                                                     sigmas=arr.sigmas()/normaliser.normalizer_for_miller_array)
        # Prep 
        args = []
        for i in xrange(len(self.arrays)-1):
            for j in xrange(i+1, len(self.arrays)):
                args.append((i,j))
           
        # Calc all CC
        worker = lambda x: calc_cc(self.arrays.values()[x[0]], self.arrays.values()[x[1]])
        results = easy_mp.pool_map(fixed_func=worker,
                                   args=args,
                                   processes=nproc)

        # Check NaN and decide which data to remove
        idx_bad = {}
        nans = []
        cc_data_for_html = []
        for (i,j), (cc,nref) in zip(args, results):
            cc_data_for_html.append((i,j,cc,nref))
            if cc==cc: continue
            idx_bad[i] = idx_bad.get(i, 0) + 1
            idx_bad[j] = idx_bad.get(j, 0) + 1
            nans.append([i,j])

        if html_maker is not None:
            html_maker.add_cc_clustering_details(cc_data_for_html)

        idx_bad = idx_bad.items()
        idx_bad.sort(key=lambda x:x[1])
        remove_idxes = set()
        
        for idx, badcount in reversed(idx_bad):
            if len(filter(lambda x: idx in x, nans)) == 0: continue
            remove_idxes.add(idx)
            nans = filter(lambda x: idx not in x, nans)
            if len(nans) == 0: break

        use_idxes = filter(lambda x: x not in remove_idxes, xrange(len(self.arrays)))

        # Make table: original index (in file list) -> new index (in matrix)
        count = 0
        org2now = collections.OrderedDict()
        for i in xrange(len(self.arrays)):
            if i in remove_idxes: continue
            org2now[i] = count
            count += 1

        if len(remove_idxes) > 0:
            open("%s_notused.lst"%prefix, "w").write("\n".join(map(lambda x: self.arrays.keys()[x], remove_idxes)))

        # Make matrix
        mat = numpy.zeros(shape=(len(use_idxes), len(use_idxes)))
        for (i,j), (cc,nref) in zip(args, results):
            if i in remove_idxes or j in remove_idxes: continue
            mat[org2now[j], org2now[i]] = cc
            
        open("%s.matrix"%prefix, "w").write(" ".join(map(lambda x:"%.4f"%x, mat.flatten())))

        ofs = open("%s.dat"%prefix, "w")
        ofs.write("   i    j     cc  nref\n")
        for (i,j), (cc,nref) in zip(args, results):
            ofs.write("%4d %4d %.4f %4d\n" % (i,j,cc,nref))

        open("%s_ana.R"%prefix, "w").write("""\
treeToList2 <- function(htree)
{  # stolen from $CCP4/share/blend/R/blend0.R
 groups <- list()
 itree <- dim(htree$merge)[1]
 for (i in 1:itree)
 { 
  il <- htree$merge[i,1]
  ir <- htree$merge[i,2]
  if (il < 0) lab1 <- htree$labels[-il]
  if (ir < 0) lab2 <- htree$labels[-ir]
  if (il > 0) lab1 <- groups[[il]]
  if (ir > 0) lab2 <- groups[[ir]]
  lab <- c(lab1,lab2)
  lab <- as.integer(lab)
  groups <- c(groups,list(lab))
 }
 return(groups)
}

cc<-scan("%(prefix)s.matrix")
md<-matrix(1-cc, ncol=%(ncol)d, byrow=TRUE)
hc <- hclust(as.dist(md),method="ward")
pdf("tree.pdf")
plot(hc)
dev.off()
png("tree.png",height=1000,width=1000)
plot(hc)
dev.off()

hc$labels <- c(%(hclabels)s)
groups <- treeToList2(hc)
cat("ClNumber             Nds         Clheight   IDs\\n",file="./CLUSTERS.txt")
for (i in 1:length(groups))
{
 sorted_groups <- sort(groups[[i]])
 linea <- sprintf("%%04d %%4d %%7.3f %%s\\n",
                  i,length(groups[[i]]),hc$height[i], paste(sorted_groups,collapse=" "))
 cat(linea, file="./CLUSTERS.txt", append=TRUE)
}

# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
  labels<-hc$labels
  merge<-data.frame(hc$merge)
  for (i in (1:nrow(merge))) {
    if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
    else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
  }
  eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
  return(JSON)
}

JSON<-HCtoJSON(hc)
cat(JSON, file="dendro.json")

q(save="yes")
""" % dict(prefix=os.path.basename(prefix),
           ncol=len(self.arrays),
           hclabels=",".join(map(lambda x: "%d"%(x+1), org2now.keys()))))

        call(cmd="Rscript", arg="%s_ana.R" % os.path.basename(prefix),
             wdir=self.wdir)

        output = open(os.path.join(self.wdir, "CLUSTERS.txt")).readlines()
        for l in output[1:]:
            sp = l.split()
            clid, clheight, ids = sp[0], sp[2], sp[3:]
            self.clusters[int(clid)] = [float(clheight), map(int,ids)]
Example #46
0
def copy_testflag(mtzfree, mtzin):
    call(cmd="copy_free_R_flag.py -r %s %s" % (mtzfree, mtzin), wdir=os.path.dirname(mtzin))
Example #47
0
def run_xdsstat(wdir):
    comm = "xdsstat"
    out = open(os.path.join(wdir, "XDSSTAT.LP"), "w")
    sys.stdout.write("\r\x1b[K Running XDSSTAT")
    util.call(comm, wdir=wdir, stdin="\n", stdout=out)
Example #48
0
def run(params):
    xdsinp = "XDS.INP"
    workdir = os.getcwd()

    # concerning DELPHI=
    if len(params.delphi) == 0:
        params.delphi = [get_delphi_defined(xdsinp)]

    params.delphi.sort()
    digit_delphi = str(get_digit(params.delphi))

    # concerning REFINE(INTEGRATE)=
    if params.all_refine_combinations:
        refine_params = make_all_refine_combinations()
    else:
        refine_params = get_refine_defined(xdsinp)

    backup_needed = files.generated_by_INTEGRATE + ("XDS.INP",)
    if params.run_correct:
        backup_needed += files.generated_by_CORRECT

    bk_prefix = None

    try:
        for rp in refine_params:
            rp_str = "refine_none" if len(rp) == 0 else "refine_" + "+".join(rp)

            for delphi in params.delphi:
                delphi_str = ("delphi_%."+digit_delphi+"f") % delphi

                bk_prefix = make_backup(backup_needed) # Backup existing XDS.INP and others

                work_name = "INTEGRATE_%s_%s" % (rp_str, delphi_str)

                inp_params = [("JOB","INTEGRATE"),
                              ("DELPHI", delphi),
                              ("REFINE(INTEGRATE)", " ".join(rp))
                              ]
                if params.run_correct:
                    inp_params[0] = ("JOB","INTEGRATE CORRECT")

                modify_xdsinp(xdsinp, inp_params=inp_params)
                call("xds_par", stdout=sys.stdout)

                # Move files into a new directory.
                os.mkdir(work_name)
                for f in backup_needed:
                    shutil.move(f, work_name)

                if params.run_correct:
                    # make .mtz file
                    try:
                        call("xdsstat", stdin="\n", stdout=open(os.path.join(work_name, "XDSSTAT.LP"),"w"), wdir=work_name)

                        xds2mtz.xds2mtz(os.path.abspath(os.path.join(work_name, "XDS_ASCII.HKL")),
                                        dir_name=os.path.join(workdir, work_name, "ccp4"),
                                        run_ctruncate=False, run_xtriage=True
                                        )

                        if params.mtz_free is not None:
                            copy_testflag(mtzfree=params.mtz_free,
                                          mtzin=os.path.join(workdir, work_name, "ccp4", "XDS_ASCII.mtz"))
                    except:
                        print traceback.format_exc()
                        print "Ignoring xds2mtz error.."
                        print

                revert_files(backup_needed, bk_prefix) # Revert XDS.INP and others

    finally:
        # 6. Revert XDS.INP, etc.
        revert_files(backup_needed, bk_prefix)
Example #49
0
def run_dials_sequence(filename_template, prefix, nr_range, wdir, known_xs,
                       overrides, scan_varying, nproc):
    log_out = open(os.path.join(wdir, "dials_sequence.log"), "w")
    pointless_log = os.path.join(wdir, "pointless.log")

    # Prepare
    img_files = find_existing_files_in_template(
        filename_template,
        nr_range[0],
        nr_range[1],
        datadir=os.path.dirname(prefix),
        check_compressed=True)
    if len(img_files) == 0:
        mylog.error("No files found for %s %s" % (filename_template, nr_range))
        return

    nproc_str = "nproc=%d" % nproc

    log_out.write("Importing %s range=%s\n" % (img_files, nr_range))
    log_out.write(" Overrides: %s\n" % overrides)
    log_out.flush()

    override_str = ""  # TODO support other stuff.. (wavelength, distance, osc_range, rotation_axis,..)
    if "orgx" in overrides and "orgy" in overrides:
        override_str += "slow_fast_beam_centre=%.2f,%.2f " % (
            overrides["orgy"], overrides["orgx"])

    if len(img_files) == 1 and img_files[0].endswith(".h5"):
        util.call('dials.import "%s" %s image_range=%d,%d' %
                  (img_files[0], override_str, nr_range[0], nr_range[1]),
                  wdir=wdir,
                  stdout=log_out,
                  expects_out=[os.path.join(wdir, "datablock.json")])
    else:
        util.call('dials.import %s template="%s" image_range=%d,%d' %
                  (override_str, filename_template.replace(
                      "?", "#"), nr_range[0], nr_range[1]),
                  wdir=wdir,
                  stdout=log_out,
                  expects_out=[os.path.join(wdir, "datablock.json")])

    util.call(
        "dials.find_spots datablock.json filter.d_max=30 %s" %
        nproc_str,  # global_threshold=200
        wdir=wdir,
        stdout=log_out,
        expects_out=[os.path.join(wdir, "strong.pickle")])

    util.call("dials.export strong.pickle format=xds xds.directory=.",
              wdir=wdir,
              stdout=log_out)

    index_ok = False
    for index_meth in ("fft3d", "fft1d", "real_space_grid_search"):
        for index_assi in ("local", "simple"):
            if index_ok: break
            cmd = "dials.index datablock.json strong.pickle verbosity=3 "
            cmd += "indexing.method=%s index_assignment.method=%s " % (
                index_meth, index_assi)
            if known_xs is not None:  # not in (known.space_group, known.unit_cell):
                cmd += "unit_cell=%s space_group=%d " % (
                    ",".join(
                        map(lambda x: "%.3f" % x,
                            known_xs.unit_cell().parameters())),
                    known_xs.space_group().type().number())
            elif index_meth == "real_space_grid_search":
                continue

            log_out.write(
                "Trying indexing.method=%s index_assignment.method=%s\n" %
                (index_meth, index_assi))
            log_out.flush()
            util.call(cmd, wdir=wdir, stdout=log_out)
            if os.path.isfile(os.path.join(wdir, "experiments.json")):
                index_ok = True
            else:
                for f in ("dials.index.log", "dials.index.debug.log"):
                    util.rotate_file(os.path.join(wdir, f))

    if not index_ok:
        return

    files_for_integration = "experiments.json indexed.pickle"

    if scan_varying:
        util.call(
            "dials.refine experiments.json indexed.pickle scan_varying=true",
            wdir=wdir,
            stdout=log_out)
        if os.path.isfile(os.path.join(wdir, "refined.pickle")):
            files_for_integration = "refined_experiments.json refined.pickle"
        else:
            log_out.write("dials.refine failed. using intedexed results.\n")

    util.call("dials.integrate %s min_spots.per_degree=10 %s" %
              (files_for_integration, nproc_str),
              wdir=wdir,
              stdout=log_out)
    util.call(
        "dials.export integrated.pickle integrated_experiments.json mtz.hklout=integrated.mtz",
        wdir=wdir,
        stdout=log_out)
    util.call("pointless integrated.mtz hklout pointless.mtz",
              wdir=wdir,
              stdin="SETTING SYMMETRY-BASED\ntolerance 10\n",
              stdout=open(pointless_log, "w"))
    util.call(
        "dials.export integrated_experiments.json integrated.pickle format=xds_ascii xds_ascii.hklout=DIALS.HKL",
        wdir=wdir,
        stdout=log_out)
    util.call("aimless hklin pointless.mtz hklout aimless.mtz",
              wdir=wdir,
              stdin="output UNMERGED\n",
              stdout=open(os.path.join(wdir, "aimless.log"), "w"))

    #job_str += "touch dials_job_finished\n"

    ret = calc_merging_stats(os.path.join(wdir, "aimless_unmerged.mtz"))
    ret["symm"] = get_most_possible_symmetry(wdir)

    pickle.dump(ret, open(os.path.join(wdir, "kamo_dials.pkl"), "w"), -1)
Example #50
0
    def do_clustering(self,
                      nproc=1,
                      b_scale=False,
                      use_normalized=False,
                      html_maker=None):
        self.clusters = {}
        prefix = os.path.join(self.wdir, "cctable")
        assert (b_scale, use_normalized).count(True) <= 1

        if len(self.arrays) < 2:
            print "WARNING: less than two data! can't do cc-based clustering"
            self.clusters[1] = [float("nan"), [0]]
            return

        # Absolute scaling using Wilson-B factor
        if b_scale:
            from mmtbx.scaling.matthews import p_vm_calculator
            from mmtbx.scaling.absolute_scaling import ml_iso_absolute_scaling

            ofs_wilson = open("%s_wilson_scales.dat" % prefix, "w")
            n_residues = p_vm_calculator(self.arrays.values()[0], 1,
                                         0).best_guess
            ofs_wilson.write("# guessed n_residues= %d\n" % n_residues)
            ofs_wilson.write("file wilsonB\n")
            for f in self.arrays:
                arr = self.arrays[f]
                iso_scale_and_b = ml_iso_absolute_scaling(arr, n_residues, 0)
                wilson_b = iso_scale_and_b.b_wilson
                ofs_wilson.write("%s %.3f\n" % (f, wilson_b))
                if wilson_b > 0:  # Ignoring data with B<0? is a bad idea.. but how..?
                    tmp = flex.exp(-2. * wilson_b *
                                   arr.unit_cell().d_star_sq(arr.indices()) /
                                   4.)
                    self.arrays[f] = arr.customized_copy(data=arr.data() * tmp,
                                                         sigmas=arr.sigmas() *
                                                         tmp)
            ofs_wilson.close()

        elif use_normalized:
            from mmtbx.scaling.absolute_scaling import kernel_normalisation
            for f in self.arrays:
                arr = self.arrays[f]
                normaliser = kernel_normalisation(arr, auto_kernel=True)
                self.arrays[f] = arr.customized_copy(
                    data=arr.data() / normaliser.normalizer_for_miller_array,
                    sigmas=arr.sigmas() /
                    normaliser.normalizer_for_miller_array)
        # Prep
        args = []
        for i in xrange(len(self.arrays) - 1):
            for j in xrange(i + 1, len(self.arrays)):
                args.append((i, j))

        # Calc all CC
        if self.use_sfdist:
            worker = lambda x: calc_sfdist(self.arrays.values()[x[0]],
                                           self.arrays.values()[x[1]])
        else:
            worker = lambda x: calc_cc(self.arrays.values()[x[0]],
                                       self.arrays.values()[x[1]])
        results = easy_mp.pool_map(fixed_func=worker,
                                   args=args,
                                   processes=nproc)

        # Check NaN and decide which data to remove
        idx_bad = {}
        nans = []
        cc_data_for_html = []
        for (i, j), (cc, nref) in zip(args, results):
            cc_data_for_html.append((i, j, cc, nref))
            if cc == cc: continue
            idx_bad[i] = idx_bad.get(i, 0) + 1
            idx_bad[j] = idx_bad.get(j, 0) + 1
            nans.append([i, j])

        if html_maker is not None:
            html_maker.add_cc_clustering_details(cc_data_for_html)

        idx_bad = idx_bad.items()
        idx_bad.sort(key=lambda x: x[1])
        remove_idxes = set()

        for idx, badcount in reversed(idx_bad):
            if len(filter(lambda x: idx in x, nans)) == 0: continue
            remove_idxes.add(idx)
            nans = filter(lambda x: idx not in x, nans)
            if len(nans) == 0: break

        use_idxes = filter(lambda x: x not in remove_idxes,
                           xrange(len(self.arrays)))

        # Make table: original index (in file list) -> new index (in matrix)
        count = 0
        org2now = collections.OrderedDict()
        for i in xrange(len(self.arrays)):
            if i in remove_idxes: continue
            org2now[i] = count
            count += 1

        if len(remove_idxes) > 0:
            open("%s_notused.lst" % prefix, "w").write("\n".join(
                map(lambda x: self.arrays.keys()[x], remove_idxes)))

        # Make matrix
        mat = numpy.zeros(shape=(len(use_idxes), len(use_idxes)))
        for (i, j), (cc, nref) in zip(args, results):
            if i in remove_idxes or j in remove_idxes: continue
            mat[org2now[j], org2now[i]] = cc

        open("%s.matrix" % prefix,
             "w").write(" ".join(map(lambda x: "%.4f" % x, mat.flatten())))

        ofs = open("%s.dat" % prefix, "w")
        ofs.write("   i    j     cc  nref\n")
        for (i, j), (cc, nref) in zip(args, results):
            ofs.write("%4d %4d %.4f %4d\n" % (i, j, cc, nref))

        open("%s_ana.R" % prefix, "w").write("""\
treeToList2 <- function(htree)
{  # stolen from $CCP4/share/blend/R/blend0.R
 groups <- list()
 itree <- dim(htree$merge)[1]
 for (i in 1:itree)
 { 
  il <- htree$merge[i,1]
  ir <- htree$merge[i,2]
  if (il < 0) lab1 <- htree$labels[-il]
  if (ir < 0) lab2 <- htree$labels[-ir]
  if (il > 0) lab1 <- groups[[il]]
  if (ir > 0) lab2 <- groups[[ir]]
  lab <- c(lab1,lab2)
  lab <- as.integer(lab)
  groups <- c(groups,list(lab))
 }
 return(groups)
}

cc<-scan("%(prefix)s.matrix")
md<-matrix(1-cc, ncol=%(ncol)d, byrow=TRUE)
hc <- hclust(as.dist(md),method="ward")
pdf("tree.pdf")
plot(hc)
dev.off()
png("tree.png",height=1000,width=1000)
plot(hc)
dev.off()

hc$labels <- c(%(hclabels)s)
groups <- treeToList2(hc)
cat("ClNumber             Nds         Clheight   IDs\\n",file="./CLUSTERS.txt")
for (i in 1:length(groups))
{
 sorted_groups <- sort(groups[[i]])
 linea <- sprintf("%%04d %%4d %%7.3f %%s\\n",
                  i,length(groups[[i]]),hc$height[i], paste(sorted_groups,collapse=" "))
 cat(linea, file="./CLUSTERS.txt", append=TRUE)
}

# reference: http://www.coppelia.io/2014/07/converting-an-r-hclust-object-into-a-d3-js-dendrogram/
library(rjson)
HCtoJSON<-function(hc){
  labels<-hc$labels
  merge<-data.frame(hc$merge)
  for (i in (1:nrow(merge))) {
    if (merge[i,1]<0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]),list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]>0 & merge[i,2]<0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node", merge[i,1], ", list(name=labels[-merge[i,2]])))")))}
    else if (merge[i,1]<0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(list(name=labels[-merge[i,1]]), node", merge[i,2],"))")))}
    else if (merge[i,1]>0 & merge[i,2]>0) {eval(parse(text=paste0("node", i, "<-list(name=\\"", i, "\\", children=list(node",merge[i,1] , ", node" , merge[i,2]," ))")))}
  }
  eval(parse(text=paste0("JSON<-toJSON(node",nrow(merge), ")")))
  return(JSON)
}

JSON<-HCtoJSON(hc)
cat(JSON, file="dendro.json")

q(save="yes")
""" % dict(prefix=os.path.basename(prefix),
           ncol=len(self.arrays),
           hclabels=",".join(map(lambda x: "%d" % (x + 1), org2now.keys()))))

        call(cmd="Rscript",
             arg="%s_ana.R" % os.path.basename(prefix),
             wdir=self.wdir)

        output = open(os.path.join(self.wdir, "CLUSTERS.txt")).readlines()
        for l in output[1:]:
            sp = l.split()
            clid, clheight, ids = sp[0], sp[2], sp[3:]
            self.clusters[int(clid)] = [float(clheight), map(int, ids)]
Example #51
0
def copy_testflag(mtzfree, mtzin, mtzout):
    call(cmd="copy_free_R_flag.py -r %s -o %s %s" % (mtzfree, mtzout, mtzin))
Example #52
0
def run_xscale(xscale_inp, cbf_to_dat=False, use_tmpdir_if_available=False):
    ftable = {}
    outfile = None
    count = 0
    inpdir = os.path.dirname(os.path.abspath(xscale_inp))
    wdir = inpdir # may be overridden
    tmpdir = None

    if use_tmpdir_if_available:
        tmpdir = util.get_temp_local_dir("xscale", min_gb=1) # TODO guess required tempdir size
        print tmpdir
        if tmpdir is None:
            print "Can't get temp dir with sufficient size."

    if tmpdir is not None:
        shutil.copy2(xscale_inp, tmpdir)
        xscale_inp = os.path.join(tmpdir, os.path.basename(xscale_inp))
        wdir = tmpdir

    os.rename(xscale_inp, xscale_inp+".org")
    ofs = open(xscale_inp, "w")

    # Check line length and make symlink if needed
    for l in open(xscale_inp+".org"):
        ll = l[:l.index("!")] if "!" in l else l
        if "OUTPUT_FILE=" in ll:
            outfile = ll[ll.index("=")+1:].strip() # TODO what if the file is not in current directory?
        if "INPUT_FILE=" in ll: #  and len(l) > 132: # one line is limited to 131 characters!
            filename = ll[ll.index("=")+1:].strip()
            if "*" in filename: filename = filename[filename.index("*")+1:].strip()
            assert " " not in filename
            lnkf = "lnk%.6d.hkl" % count
            assert not os.path.isfile(os.path.join(wdir, lnkf))
            filename_abs = os.path.normpath(os.path.join(inpdir, filename)) if not os.path.isabs(filename) else filename
            os.symlink(filename_abs, os.path.join(wdir, lnkf))
            print "xscale: %s -> %s" % (lnkf, filename)
            count += 1
            ftable[lnkf] = filename
            l = l.replace(filename, lnkf)
        ofs.write(l)

    ofs.close()
    assert outfile is not None

    if len(ftable) == 0:
        os.rename(xscale_inp+".org", xscale_inp)

    # Run xscale
    util.call(xscale_comm, wdir=wdir)

    # Replace file names if needed
    if len(ftable) > 0:
        for i, f in enumerate(("XSCALE.LP", outfile)):
            f = os.path.join(wdir, f)
            if not os.path.isfile(f): continue

            os.rename(f, f+".org")
            ofs = open(f, "w")

            if i == 0:
                for l in open(f+".org"):
                    if ".hkl" in l:
                        for lfn in ftable: l = l.replace(lfn, ftable[lfn])
                    ofs.write(l)
            else:
                ifs = open(f+".org")
                while True:
                    l = ifs.readline() 
                    if ".hkl" in l:
                        for lfn in ftable: l = l.replace(lfn, ftable[lfn])
                    ofs.write(l)
                    if l.startswith("!END_OF_HEADER"): break
                shutil.copyfileobj(ifs, ofs)
                
            ofs.close()
            os.remove(f+".org")

        for lfn in ftable:
            os.remove(os.path.join(wdir, lfn))

        os.rename(xscale_inp+".org", xscale_inp)

    if cbf_to_dat:
        xscale_lp = os.path.join(wdir, "XSCALE.LP")
        cbfouts = glob.glob(os.path.join(wdir, "*.cbf"))
        if len(cbfouts) > 0:
            xscalelp.cbf_to_dat(xscale_lp)
            for f in cbfouts: os.remove(f)

    # Move to original directory
    if tmpdir is not None:
        for f in glob.glob(os.path.join(tmpdir, "*")):
            shutil.copy2(f, inpdir)

        shutil.rmtree(tmpdir)
Example #53
0
def calc_delta_cchalf(prev_lp, tmpdir, with_sigma=False, precalc_cchalf_all=None):
    """
    Obsolete function. Maybe useful when starting with the last XSCALE.LP...
    """

    assert not with_sigma # Not supported now

    if not os.path.exists(tmpdir): os.makedirs(tmpdir)

    newinp = os.path.join(tmpdir, "XSCALE.INP")
    newlp = os.path.join(tmpdir, "XSCALE.LP")

    rel_org = os.path.relpath(os.path.dirname(prev_lp), tmpdir)

    if precalc_cchalf_all is None:
        # read CC1/2(all) from cwd
        orgtable = xscalelp.read_stats_table(prev_lp)
        assert orgtable["dmin"][-1] is None # None for total
        cchalf_all = orgtable["cc_half"][-1]
    else:
        cchalf_all = precalc_cchalf_all

    datout = open(os.path.join(tmpdir, "delta_cchalf.dat"), "w")
    datout.write("# CC1/2(all)= %.4f\n" % cchalf_all)
    datout.write("idx exfile cc1/2 delta_cc1/2\n")

    # Read inp and extract input files.
    # XXX What if reference file is included???
    orgkwds = xscalelp.read_control_cards(prev_lp)
    inpfiles = map(lambda x:x[1],
                   filter(lambda y: y[0]=="INPUT_FILE", orgkwds))
                          
    # XXX Need to take care of xscale specific inp manner - order matters!!

    delta_cchalf = []

    for iex in xrange(len(inpfiles)):
        print "Doing", iex
        files = inpfiles[:iex] + inpfiles[iex+1:]
        ofs = open(newinp, "w")
        for k, v in orgkwds:
            if k not in ("INPUT_FILE", "INCLUDE_RESOLUTION_RANGE"):
                ofs.write("%s= %s\n" % (k,v))

        for f in files:
            if not os.path.isabs(f): f = os.path.join(rel_org, f)
            ofs.write("INPUT_FILE= %s\n" % f)
        ofs.close()

        util.call(xscale_comm, wdir=tmpdir)
        table = xscalelp.read_stats_table(newlp)
        assert table["dmin"][-1] is None # None for total
        cchalf_exi = table["cc_half"][-1]
        delta_cchalf.append((iex, cchalf_exi - cchalf_all))

        os.rename(newinp, newinp+".ex%.3d"%iex)
        os.rename(newlp, newlp+".ex%.3d"%iex)

        datout.write("%3d %s %.4f %.4f\n" % (iex, inpfiles[iex], cchalf_exi, cchalf_exi-cchalf_all))

    delta_cchalf.sort(key=lambda x: -x[1])
    print
    print "# Sorted table"
    for idx, dch in delta_cchalf:
        print "%3d %-.4f %s" % (idx, dch, inpfiles[idx])

    return delta_cchalf, cchalf_all
Example #54
0
def run(hklin, pdbin, wdir, anisotropy_correction=False):
    arrays = iotbx.file_reader.any_file(hklin).file_server.miller_arrays
    i_arrays = filter(
        lambda x: x.is_xray_intensity_array() and x.anomalous_flag(), arrays)
    f_arrays = filter(
        lambda x: x.is_xray_amplitude_array() and x.anomalous_flag(), arrays)

    if not i_arrays and not f_arrays:
        print "No anomalous observation data"
        return

    if os.path.exists(wdir):
        print "%s already exists. quiting." % wdir
        return

    os.mkdir(wdir)

    xs = crystal_symmetry_from_any.extract_from(pdbin)

    sh_out = open(os.path.join(wdir, "run_anode.sh"), "w")
    sh_out.write("#!/bin/sh\n\n")
    sh_out.write("shelxc anode <<+ > shelxc.log 2>&1\n")
    sh_out.write("cell %s\n" % format_unit_cell(xs.unit_cell()))
    sh_out.write("spag %s\n" % str(xs.space_group_info()).replace(" ", ""))

    if i_arrays:
        obs_array = i_arrays[0]
        infile = "%s.hkl" % os.path.splitext(os.path.basename(hklin))[0]
        in_opt = "%s" % infile
        print "Using intensity array:", obs_array.info().label_string()
    else:
        obs_array = f_arrays[0]
        infile = "%s_f.hkl" % os.path.splitext(os.path.basename(hklin))[0]
        in_opt = "-f %s" % infile
        print "No intensity arrays. Using amplitude arrays instead:", obs_array.info(
        ).label_string()

    sh_out.write("! data from %s : %s\n" %
                 (os.path.abspath(hklin), obs_array.info().label_string()))
    obs_array.crystal_symmetry().show_summary(sh_out, prefix="! ")
    check_symm(obs_array.crystal_symmetry(), xs)

    n_org = obs_array.size()
    obs_array = obs_array.eliminate_sys_absent()
    n_sys_abs = n_org - obs_array.size()
    if n_sys_abs > 0:
        print "  %d systematic absences removed." % n_sys_abs

    if anisotropy_correction:
        print "Correcting anisotropy.."
        n_residues = p_vm_calculator(obs_array, 1, 0).best_guess
        abss = ml_aniso_absolute_scaling(obs_array, n_residues=n_residues)
        abss.show()
        tmp = -2. if i_arrays else -1.
        b_cart = map(lambda x: x * tmp, abss.b_cart)
        obs_array = obs_array.apply_debye_waller_factors(b_cart=b_cart)

    sh_out.write("sad %s\n" % in_opt)
    iotbx.shelx.hklf.miller_array_export_as_shelx_hklf(
        obs_array,
        open(os.path.join(wdir, infile), "w"),
        normalise_if_format_overflow=True)
    sh_out.write("+\n\n")
    sh_out.write('ln -s "%s" anode.pdb\n\n' % os.path.relpath(pdbin, wdir))
    sh_out.write("anode anode\n")
    sh_out.close()

    call(cmd="sh", arg="./run_anode.sh", wdir=wdir)

    pha_file = os.path.join(wdir, "anode.pha")
    if os.path.isfile(pha_file):
        pha2mtz(pha_file, xs, os.path.join(wdir, "anode.pha.mtz"))

    print "Done. See %s/" % wdir

    fa_file = os.path.join(wdir, "anode_fa.hkl")
    if os.path.isfile(fa_file):
        r = iotbx.shelx.hklf.reader(open(fa_file))
        fa_array = r.as_miller_arrays(crystal_symmetry=xs)[0]
        print "\nData stats:"
        print " # Cmpl.o = Anomalous completeness in original data"
        print " # Cmpl.c = Anomalous completeness in shelxc result (rejections)"
        print " # SigAno = <d''/sigma> in shelxc result"
        print " d_max d_min Cmpl.o Cmpl.c SigAno"
        binner = obs_array.setup_binner(n_bins=12)
        for i_bin in binner.range_used():
            d_max_bin, d_min_bin = binner.bin_d_range(i_bin)
            obs_sel = obs_array.resolution_filter(d_max_bin, d_min_bin)
            obs_sel_ano = obs_sel.anomalous_differences()
            fa_sel = fa_array.resolution_filter(d_max_bin, d_min_bin)
            cmplset = obs_sel_ano.complete_set(
                d_max=d_max_bin, d_min=d_min_bin).select_acentric()
            n_acentric = cmplset.size()
            sigano = flex.mean(
                fa_sel.data() /
                fa_sel.sigmas()) if fa_sel.size() else float("nan")
            print " %5.2f %5.2f %6.2f %6.2f %6.2f" % (
                d_max_bin, d_min_bin, 100. * obs_sel_ano.size() / n_acentric,
                100. * fa_sel.size() / n_acentric, sigano)

    lsa_file = os.path.join(wdir, "anode.lsa")
    if os.path.isfile(lsa_file):
        print ""
        flag = False
        for l in open(lsa_file):
            if "Strongest unique anomalous peaks" in l:
                flag = True
            elif "Reflections written to" in l:
                flag = False
            if flag:
                print l.rstrip()

    if os.path.isfile(("anode_fa.res")):
        x = iotbx.shelx.cctbx_xray_structure_from(file=open("anode_fa.res"))
        open("anode_fa.pdb", "w").write(x.as_pdb_file())
Example #55
0
def xds2mtz_normal(refl, mtzout, sg, wavelen, logout, use_ctruncate=False, dmin=None, dmax=None):
    wdir = os.path.dirname(mtzout)

    if not os.path.exists(os.path.join(wdir, "original")):
        os.symlink(refl, os.path.join(wdir, "original"))


    ##
    # prepare XDSCONV.INP and run
    #

    # for I
    print "generating MTZ for IMEAN,SIGIMEAN"

    ofs = open(os.path.join(wdir, "XDSCONV.INP"), "w")
    ofs.write("OUTPUT_FILE=tmp.hkl CCP4_I\n")
    ofs.write("INPUT_FILE=original\n")
    ofs.write("GENERATE_FRACTION_OF_TEST_REFLECTIONS=0.0\n")
    ofs.write("WILSON_STATISTICS= TRUE\n")
    if None not in (dmin, dmax):
        ofs.write("INCLUDE_RESOLUTION_RANGE= %s %s\n" % (dmax, dmin))

    ofs.close()

    call(cmd="xdsconv",
         wdir=wdir,
         expects_in=["original"],
         expects_out=["F2MTZ.INP", "tmp.hkl"],
         stdout=logout
         )

    call(cmd="f2mtz",
         arg="hklout CCP4_I.mtz",
         stdin=open(os.path.join(wdir, "F2MTZ.INP")).read(),
         wdir=wdir,
         expects_in=["tmp.hkl"],
         expects_out=["CCP4_I.mtz"],
         stdout=logout
         )

    # for F
    print "generating MTZ for FP,SIGFP"
    ctruncate_ok = False
    if use_ctruncate:
        call(cmd="ctruncate -hklin CCP4_I.mtz -hklout ctruncate.mtz -colin '/*/*/[IMEAN,SIGIMEAN]'",
             wdir=wdir,
             expects_in=["CCP4_I.mtz"],
             stdout=open(os.path.join(wdir, "ctruncate.log"), "w")
             )

        ctruncate_ok = os.path.isfile(os.path.join(wdir, "ctruncate.mtz"))

        if ctruncate_ok:
            call(cmd="cad",
                 arg="hklin1 ctruncate.mtz hklout CCP4_FI.mtz",
                 stdin="""\
    labin file 1 all
    xname file 1 ALL=XDS
    dname file 1 ALL=XDS
    dwavelength file 1 XDS XDS %s
    symmetry %s
    end
    """%(wavelen,sg),
                 wdir=wdir,
                 expects_in=["ctruncate.mtz"],
                 expects_out=["CCP4_FI.mtz"],
                 stdout=logout
                 )
        else:
            print "Ctruncate failed. Use xdsconv instead."

    if not ctruncate_ok:
        ofs = open(os.path.join(wdir, "XDSCONV.INP"), "w")
        ofs.write("OUTPUT_FILE=tmp.hkl CCP4_F\n")
        ofs.write("INPUT_FILE=original\n")
        ofs.write("GENERATE_FRACTION_OF_TEST_REFLECTIONS=0.0\n")
        ofs.write("WILSON_STATISTICS= TRUE\n")
        if None not in (dmin, dmax):
            ofs.write("INCLUDE_RESOLUTION_RANGE= %s %s\n" % (dmax, dmin))

        ofs.close()

        call(cmd="xdsconv",
             wdir=wdir,
             expects_in=["original"],
             expects_out=["F2MTZ.INP", "tmp.hkl"],
             stdout=logout
             )

        call(cmd="f2mtz",
             arg="hklout CCP4_F.mtz",
             stdin=open(os.path.join(wdir, "F2MTZ.INP")).read(),
             wdir=wdir,
             expects_in=["tmp.hkl"],
             expects_out=["CCP4_F.mtz"],
             stdout=logout
             )

        ##
        # CAD all mtz files
        print "concatenating MTZ files"

        call(cmd="cad",
             arg="hklin1 CCP4_I.mtz hklin2 CCP4_F.mtz hklout CCP4_FI.mtz",
             stdin="""\
labin file 1 all
labin file 2 all
xname file 1 ALL=XDS
xname file 2 ALL=XDS
dname file 1 ALL=XDS
dname file 2 ALL=XDS
dwavelength file 1 XDS XDS %s
symmetry %s
end
"""%(wavelen,sg),
             wdir=wdir,
             expects_in=["CCP4_I.mtz", "CCP4_F.mtz"],
             expects_out=["CCP4_FI.mtz"],
             stdout=logout
             )

    ##
    # Generate all unique reflections
    print "Genrating all unique reflections"
    unique(mtzin="CCP4_FI.mtz", mtzout=os.path.basename(mtzout), wdir=wdir, logout=logout)


    # remove files
    os.remove(os.path.join(wdir, "CCP4_I.mtz"))
    os.remove(os.path.join(wdir, "CCP4_FI.mtz"))
    os.remove(os.path.join(wdir, "tmp.hkl"))
    os.remove(os.path.join(wdir, "XDSCONV.INP"))
    os.remove(os.path.join(wdir, "XDSCONV.LP"))
    os.remove(os.path.join(wdir, "F2MTZ.INP"))
    os.remove(os.path.join(wdir, "original"))

    if ctruncate_ok:
        os.remove(os.path.join(wdir, "ctruncate.mtz"))
    else:
        os.remove(os.path.join(wdir, "CCP4_F.mtz"))
Example #56
0
def copy_testflag(mtzfree, mtzin):
    call(cmd="copy_free_R_flag.py -r %s %s" % (mtzfree, mtzin),
         wdir=os.path.dirname(mtzin))
Example #57
0
def find_spots(img_file, params):
    """
    Use XDS to locate spots.
    If params.xds.do_defpix is true, DEFPIX will be run. For DEFPIX, dummy XPARM.XDS is needed. Thanks to DEFPIX, we can mask beam stop shadow and remove areas outside the resolution range.
    If false, we need to set TRUSTED_REGION to exclude regions outside resolution range, but it is independent of beam center. Maybe we should remove spots outside the resolution range after XDS run?
    """

    # Test if ramdisk available
    if os.path.isdir("/dev/shm"):
        work_dir = tempfile.mkdtemp(
            prefix="shika_x_" +
            os.path.splitext(os.path.basename(img_file))[0],
            dir="/dev/shm")
    else:
        work_dir = os.path.join(
            params.work_dir,
            "xds_" + os.path.splitext(os.path.basename(img_file))[0])

    xdsinp = os.path.join(work_dir, "XDS.INP")
    spot_xds = os.path.join(work_dir, "SPOT.XDS")

    if not os.path.exists(work_dir):
        os.mkdir(work_dir)

    template_str, min_frame, max_frame = dataset.group_img_files_template(
        [img_file])[0]

    im = XIO.Image(img_file)

    # Remove lines if None (means to use default)
    params_maps = [
        ("strong_pixel", "STRONG_PIXEL="),
        ("minimum_number_of_pixels_in_a_spot",
         "MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT="),
        ("background_pixel", "BACKGROUND_PIXEL="),
        ("maximum_number_of_strong_pixels",
         "MAXIMUM_NUMBER_OF_STRONG_PIXELS="),
        ("spot_maximum_centroid", "SPOT_MAXIMUM-CENTROID="),
        ("reflecting_range", "REFLECTING_RANGE="),
        ("value_range_for_trusted_detector_pixels",
         "VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS="),
    ]
    tmp = xds_inp_template.splitlines()
    for p, x in params_maps:
        if getattr(params.xds, p) is None:
            tmp = filter(lambda s: not s.startswith(x), tmp)
    inp_template = "\n".join(tmp)

    # Prepare XDS.INP
    inp_str = inp_template % dict(
        template=os.path.relpath(template_str, work_dir),
        framenum=min_frame,
        orgx=im.header["BeamX"] / im.header["PixelX"],
        orgy=im.header["BeamY"] / im.header["PixelY"],
        distance=im.header["Distance"],
        osc_range=im.header["PhiWidth"],
        wavelength=im.header["Wavelength"],
        strong_pixel=params.xds.strong_pixel,
        min_pixels=params.xds.minimum_number_of_pixels_in_a_spot,
        background_pixel=params.xds.background_pixel,
        max_strong_pixels=params.xds.maximum_number_of_strong_pixels,
        spot_maximum_centroid=params.xds.spot_maximum_centroid,
        reflecting_range=params.xds.reflecting_range,
        nx=im.header["Width"],
        ny=im.header["Height"],
        qx=im.header["PixelX"],
        qy=im.header["PixelY"],
        defpix_trusted1=params.xds.value_range_for_trusted_detector_pixels[0],
        defpix_trusted2=params.xds.value_range_for_trusted_detector_pixels[1])

    open(xdsinp, "w").write(inp_str)

    if params.xds.do_defpix:
        xp = xparm.XPARM()
        xp.set_info_from_xdsinp(xdsinp)
        open(os.path.join(work_dir, "XPARM.XDS"), "w").write(xp.xparm_str())
        modify_xdsinp(xdsinp,
                      inp_params=[("JOB", "XYCORR INIT DEFPIX"),
                                  ("INCLUDE_RESOLUTION_RANGE",
                                   res_range_for_xds(params.distl.res.outer,
                                                     params.distl.res.inner))])
        call("xds",
             wdir=work_dir,
             stdout=open(os.path.join(work_dir, "xds.log"), "w"))
        shutil.copy(os.path.join(work_dir, "BKGPIX.cbf"),
                    os.path.join(work_dir, "BKGINIT.cbf"))
        modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
    else:
        modify_xdsinp(xdsinp,
                      inp_params=[
                          ("TRUSTED_REGION",
                           res_range_to_trusted_region(params.distl.res.outer,
                                                       params.distl.res.inner,
                                                       im.header))
                      ])
        open(os.path.join(work_dir, "xds.log"), "w").write("")

    # Run XDS
    rotate_file(spot_xds)
    call("xds",
         wdir=work_dir,
         stdout=open(os.path.join(work_dir, "xds.log"), "a"))

    # Extract results
    spots = []  # (x, y, d, intensity)
    if os.path.isfile(spot_xds):
        for l in open(spot_xds):
            x, y, z, intensity = map(lambda x: float(x), l.strip().split())
            d = coord_to_resol(x, y, im.header)
            spots.append((x, y, d, intensity))

    # Delete dir
    shutil.rmtree(work_dir)

    return spots
Example #58
0
def find_spots(img_file, params):
    """
    Use XDS to locate spots.
    If params.xds.do_defpix is true, DEFPIX will be run. For DEFPIX, dummy XPARM.XDS is needed. Thanks to DEFPIX, we can mask beam stop shadow and remove areas outside the resolution range.
    If false, we need to set TRUSTED_REGION to exclude regions outside resolution range, but it is independent of beam center. Maybe we should remove spots outside the resolution range after XDS run?
    """

    # Test if ramdisk available
    if os.path.isdir("/dev/shm"):
        work_dir = tempfile.mkdtemp(prefix="shika_x_"+os.path.splitext(os.path.basename(img_file))[0], dir="/dev/shm")
    else:
        work_dir = os.path.join(params.work_dir, "xds_"+os.path.splitext(os.path.basename(img_file))[0])

    xdsinp = os.path.join(work_dir, "XDS.INP")
    spot_xds = os.path.join(work_dir, "SPOT.XDS")

    if not os.path.exists(work_dir):
        os.mkdir(work_dir)

    template_str, min_frame, max_frame = dataset.group_img_files_template([img_file])[0]

    im = XIO.Image(img_file)

    # Remove lines if None (means to use default)
    params_maps = [("strong_pixel", "STRONG_PIXEL="),
                   ("minimum_number_of_pixels_in_a_spot", "MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT="),
                   ("background_pixel", "BACKGROUND_PIXEL="),
                   ("maximum_number_of_strong_pixels", "MAXIMUM_NUMBER_OF_STRONG_PIXELS="),
                   ("spot_maximum_centroid", "SPOT_MAXIMUM-CENTROID="),
                   ("reflecting_range", "REFLECTING_RANGE="),
                   ("value_range_for_trusted_detector_pixels", "VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS="),
                   ]
    tmp = xds_inp_template.splitlines()
    for p, x in params_maps:
        if getattr(params.xds, p) is None:
            tmp = filter(lambda s:not s.startswith(x), tmp)
    inp_template = "\n".join(tmp)

    # Prepare XDS.INP
    inp_str = inp_template%dict(template=os.path.relpath(template_str, work_dir),
                                framenum=min_frame,
                                orgx=im.header["BeamX"]/im.header["PixelX"],
                                orgy=im.header["BeamY"]/im.header["PixelY"],
                                distance=im.header["Distance"],
                                osc_range=im.header["PhiWidth"],
                                wavelength=im.header["Wavelength"],
                                strong_pixel=params.xds.strong_pixel,
                                min_pixels=params.xds.minimum_number_of_pixels_in_a_spot,
                                background_pixel=params.xds.background_pixel,
                                max_strong_pixels=params.xds.maximum_number_of_strong_pixels,
                                spot_maximum_centroid=params.xds.spot_maximum_centroid,
                                reflecting_range=params.xds.reflecting_range,
                                nx=im.header["Width"], ny=im.header["Height"],
                                qx=im.header["PixelX"], qy=im.header["PixelY"],
                                defpix_trusted1=params.xds.value_range_for_trusted_detector_pixels[0],
                                defpix_trusted2=params.xds.value_range_for_trusted_detector_pixels[1]
                                )

    open(xdsinp, "w").write(inp_str)

    if params.xds.do_defpix:
        xp = xparm.XPARM()
        xp.set_info_from_xdsinp(xdsinp)
        open(os.path.join(work_dir, "XPARM.XDS"), "w").write(xp.xparm_str())
        modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT DEFPIX"),
                                          ("INCLUDE_RESOLUTION_RANGE", res_range_for_xds(params.distl.res.outer, params.distl.res.inner))
                                          ])
        call("xds", wdir=work_dir, stdout=open(os.path.join(work_dir, "xds.log"), "w"))
        shutil.copy(os.path.join(work_dir, "BKGPIX.cbf"), os.path.join(work_dir, "BKGINIT.cbf"))
        modify_xdsinp(xdsinp, inp_params=[("JOB","COLSPOT")])
    else:
        modify_xdsinp(xdsinp, inp_params=[("TRUSTED_REGION", res_range_to_trusted_region(params.distl.res.outer, params.distl.res.inner, im.header))
                                          ])
        open(os.path.join(work_dir, "xds.log"), "w").write("")

    # Run XDS
    rotate_file(spot_xds)
    call("xds", wdir=work_dir, stdout=open(os.path.join(work_dir, "xds.log"), "a"))

    # Extract results
    spots = [] # (x, y, d, intensity)
    if os.path.isfile(spot_xds):
        for l in open(spot_xds):
            x, y, z, intensity = map(lambda x:float(x), l.strip().split())
            d = coord_to_resol(x, y, im.header)
            spots.append((x, y, d, intensity))

    # Delete dir
    shutil.rmtree(work_dir)

    return spots
Example #59
0
def reindex_with_specified_symm_worker(wd,
                                       wdr,
                                       topdir,
                                       log_out,
                                       reference_symm,
                                       sgnum_laue,
                                       prep_dials_files=False):
    """
    wd: directory where XDS file exists
    wdr: wd to return; a directory where transformed file should be saved.

    If wd!=wdr, files in wd/ are unchanged during procedure. Multiprocessing is unsafe when wd==wdr.
    """

    out = StringIO()
    print >> out, "%s:" % os.path.relpath(wd, topdir),

    # Find appropriate data
    xac_file = util.return_first_found_file(
        ("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
         "XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
         "XDS_ASCII.HKL.org", "XDS_ASCII.HKL", "DIALS.HKL.org", "DIALS.HKL"),
        wd=wd)
    if xac_file is None:
        print >> out, "Can't find XDS_ASCII file in %s" % wd
        log_out.write(out.getvalue())
        log_out.flush()
        return (wdr, None)

    if xac_file.endswith(".org"): xac_file_out = xac_file[:-4]
    else: xac_file_out = xac_file

    xac = XDS_ASCII(xac_file, read_data=False)
    print >> out, "%s %s (%s)" % (os.path.basename(xac_file),
                                  xac.symm.space_group_info(), ",".join(
                                      map(lambda x: "%.2f" % x,
                                          xac.symm.unit_cell().parameters())))

    if xac.symm.reflection_intensity_symmetry(
            False).space_group_info().type().number() == sgnum_laue:
        if xac.symm.unit_cell().is_similar_to(
                reference_symm.unit_cell(), 0.1,
                10):  # XXX Check unit cell consistency!!
            print >> out, "  Already scaled with specified symmetry"
            log_out.write(out.getvalue())
            log_out.flush()

            if wd != wdr: shutil.copy2(xac_file, wdr)

            if prep_dials_files and "DIALS.HKL" not in xac_file:
                prepare_dials_files(wd, out, moveto=wdr)

            return (wdr, (numpy.array(xac.symm.unit_cell().parameters()),
                          os.path.join(wdr, os.path.basename(xac_file))))

    cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
                                          20)  # XXX ISN'T THIS TOO LARGE?

    if len(cosets.combined_cb_ops()) == 0:
        print >> out, "Can't find operator:"
        xac.symm.show_summary(out, " ")
        reference_symm.show_summary(out, " ")
        log_out.write(out.getvalue())
        log_out.flush()
        return (wdr, None)

    if wd == wdr:
        dest = tempfile.mkdtemp(prefix="multiprep", dir=wd)
    else:
        dest = wdr

    hklout = os.path.join(dest, os.path.basename(xac_file_out))

    newcell = xac.write_reindexed(op=cosets.combined_cb_ops()[0],
                                  space_group=reference_symm.space_group(),
                                  hklout=hklout)

    if "DIALS.HKL" in os.path.basename(xac_file):
        outstr = 'output.experiments="%sreindexed_experiments.json" ' % os.path.join(
            dest, "")
        outstr += 'output.reflections="%sreindexed_reflections.pickle" ' % os.path.join(
            dest, "")
        for f in ("experiments.json", "indexed.pickle"):
            if not os.path.isfile(os.path.join(os.path.dirname(xac_file), f)):
                continue
            util.call(
                'dials.reindex %s change_of_basis_op=%s space_group="%s" %s' %
                (f, cosets.combined_cb_ops()[0].as_abc(),
                 reference_symm.space_group_info(), outstr),
                wdir=os.path.dirname(xac_file))
    elif prep_dials_files:
        prepare_dials_files(wd,
                            out,
                            space_group=reference_symm.space_group(),
                            reindex_op=cosets.combined_cb_ops()[0],
                            moveto=dest)

    newcell_str = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
    print >> out, "  Reindexed to transformed cell: %s with %s" % (
        newcell_str, cosets.combined_cb_ops()[0].as_hkl())
    log_out.write(out.getvalue())
    log_out.flush()

    if wd == wdr:
        for f in glob.glob(os.path.join(dest, "*")):
            f_in_wd = os.path.join(wd, os.path.basename(f))
            if os.path.exists(f_in_wd) and not os.path.exists(f_in_wd +
                                                              ".org"):
                os.rename(f_in_wd, f_in_wd + ".org")
            os.rename(f, f_in_wd)

        shutil.rmtree(dest)
        ret = (numpy.array(newcell.parameters()),
               os.path.join(wd, os.path.basename(xac_file_out)))
    else:
        ret = (numpy.array(newcell.parameters()), hklout)

    return (wdr, ret)
Example #60
0
    # Check all needed files exist
    if not check_needed_files():
        sys.exit(1)

    xdsinp = "XDS.INP"

    workdir = "scale_exp"
    os.mkdir(workdir)
    os.mkdir(os.path.join(workdir, "scaled"))


    anomalous = is_anomalous(xdsinp)

    # *. INTEGRATE_forscale.mtz
    call("pointless -copy xdsin INTEGRATE.HKL hklout %s/INTEGRATE_forscale.mtz" % workdir)

    # 1. Backup XDS.INP, etc. (Make copies; not renaming)
    bk_prefix = make_backup(backup_needed)

    try:
        # CORRECT with a
        modify_xdsinp(xdsinp, inp_params=[("JOB","CORRECT"),
                                          ("CORRECTIONS", None),
                                          ("NBATCH", None),
                                          ("MINIMUM_I/SIGMA", None),
                                          ("WFAC1", None)
                                          ])

        call("xds", stdout=open("CORRECT_a_xds.log", "w"))