Пример #1
0
def get_colspot_result(frame_ranges, wdir):
    assert all(map(lambda x: len(x) == 2 and x[0] <= x[1], frame_ranges))

    # Check all needed files exist
    if not check_needed_files(xds_files.needed_by_COLSPOT, wdir):
        return

    xdsinp = os.path.join(wdir, "XDS.INP")

    backup_needed = xds_files.generated_by_COLSPOT + ("XDS.INP",)

    # 1. Backup XDS.INP, etc. (Make copies; not renaming)
    bk_prefix = make_backup(backup_needed, wdir=wdir)

    try:
        # 2. Modify XDS.INP
        spot_ranges = map(lambda x: ("SPOT_RANGE", "%d %d" % tuple(x)), frame_ranges)
        data_range = "%d %d" % (min(map(lambda x: min(x), frame_ranges)), max(map(lambda x: max(x), frame_ranges)))
        modify_xdsinp(xdsinp, [("JOB", "COLSPOT"), ("DATA_RANGE", data_range)] + spot_ranges)

        # 3. Run xds
        p = subprocess.Popen("xds", cwd=wdir)
        p.wait()

        spotxds = SpotXds(os.path.join(wdir, "SPOT.XDS"))

    finally:
        # 6. Revert XDS.INP, etc.
        revert_files(backup_needed, bk_prefix, wdir=wdir)

    return spotxds
Пример #2
0
def run_xds_sequence(root, params):
    tmpdir = None
    
    if params.use_tmpdir_if_available:
        tmpdir = util.get_temp_local_dir("xdskamo", min_gb=2) # TODO guess required tempdir size
        if tmpdir is None:
            print "Can't get temp dir with sufficient size."

    # If tmpdir is not used
    if tmpdir is None:
        return xds_sequence(root, params)

    print "Using %s as temp dir.." % tmpdir

    # If tempdir is used
    for f in glob.glob(os.path.join(root, "*")): shutil.copy2(f, tmpdir)
    xdsinp = os.path.join(tmpdir, "XDS.INP")
    xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))

    # Make a link to data dir
    org_data_template = xdsinp_dict["NAME_TEMPLATE_OF_DATA_FRAMES"]
    ord_data_dir = os.path.dirname(org_data_template)
    if not os.path.isabs(ord_data_dir): ord_data_dir = os.path.join(root, ord_data_dir)
    datadir_lns = os.path.join(tmpdir, "data_loc")
    os.symlink(ord_data_dir, datadir_lns)

    # Modify XDS.INP
    modify_xdsinp(xdsinp, inp_params=[("NAME_TEMPLATE_OF_DATA_FRAMES",
                                       os.path.join("data_loc", os.path.basename(org_data_template)))])

    try:
        ret = xds_sequence(tmpdir, params)
    finally:
        # Revert XDS.INP
        modify_xdsinp(xdsinp, inp_params=[("NAME_TEMPLATE_OF_DATA_FRAMES", org_data_template)])

        # Remove link
        os.remove(datadir_lns)
        
        # Move to original directory
        for f in glob.glob(os.path.join(tmpdir, "*")):
            f_dest = os.path.join(root, os.path.relpath(f, tmpdir))
            if os.path.isfile(f_dest):
                # Copy only if newer
                if os.stat(f).st_mtime > os.stat(f_dest).st_mtime:
                    shutil.copy2(f, root)
                else:
                    print "%s already exists and not modified. skip." % f

                os.remove(f)
            else:
                shutil.move(f, root)
        
        # Remove tmpdir
        shutil.rmtree(tmpdir)

    return ret
Пример #3
0
def work(rootdir, xdsinp, orgxy):
    workdir = os.path.join(rootdir, "bs_x%+.1f_y%+.1f" % orgxy)
    inpdir = os.path.normpath(os.path.dirname(xdsinp))
    print workdir
    os.makedirs(workdir)
    shutil.copyfile(os.path.join(inpdir, "SPOT.XDS"), os.path.join(workdir, "SPOT.XDS"))
    shutil.copyfile(xdsinp, os.path.join(workdir, "XDS.INP"))
    modify_xdsinp(os.path.join(workdir, "XDS.INP"), inp_params=[("JOB", "IDXREF"),
                                                                ("ORGX", orgxy[0]),
                                                                ("ORGY", orgxy[1]),
                                                                ])
    call("xds", wdir=workdir)
Пример #4
0
def work(rootdir, xdsinp, orgxy):
    workdir = os.path.join(rootdir, "bs_x%+.1f_y%+.1f" % orgxy)
    inpdir = os.path.normpath(os.path.dirname(xdsinp))
    print workdir
    os.makedirs(workdir)
    shutil.copyfile(os.path.join(inpdir, "SPOT.XDS"),
                    os.path.join(workdir, "SPOT.XDS"))
    shutil.copyfile(xdsinp, os.path.join(workdir, "XDS.INP"))
    modify_xdsinp(os.path.join(workdir, "XDS.INP"),
                  inp_params=[
                      ("JOB", "IDXREF"),
                      ("ORGX", orgxy[0]),
                      ("ORGY", orgxy[1]),
                  ])
    call("xds", wdir=workdir)
Пример #5
0
    def cut_resolution(self, cycle_number):
        def est_resol(xscale_hkl, res_params, plt_out):
            iobs = XDS_ASCII(xscale_hkl, i_only=True).i_obs()
            est = estimate_resolution_based_on_cc_half(iobs, res_params.cc_one_half_min,
                                                       res_params.cc_half_tol,
                                                       res_params.n_bins, log_out=self.out)
            est.show_plot(False, plt_out)
            if None not in (est.d_min, est.cc_at_d_min):
                self.out.write("Best resolution cutoff= %.2f A @CC1/2= %.4f\n" % (est.d_min, est.cc_at_d_min))
            else:
                self.out.write("Can't decide resolution cutoff. No reflections??\n")
            return est.d_min
        # est_resol()

        print >>self.out, "**** Determining resolution cutoff in run_%.2d ****" % cycle_number
        last_wd = os.path.join(self.workdir_org, "run_%.2d"%cycle_number)
        xscale_hkl = os.path.abspath(os.path.join(last_wd, "xscale.hkl"))

        tmpwd = os.path.join(self.workdir_org, "run_%.2d_tmp"%cycle_number)
        os.mkdir(tmpwd)

        for i, cc_cut in enumerate((self.res_params.cc_one_half_min*.7, self.res_params.cc_one_half_min)):
            self.res_params.cc_one_half_min = cc_cut
            d_min = est_resol(xscale_hkl, self.res_params,
                              os.path.join(tmpwd, "ccfit_%d.pdf"%(i+1)))
            if d_min is not None and d_min > self.d_min + 0.001:
                for f in "XSCALE.INP", "XSCALE.LP": util.rotate_file(os.path.join(tmpwd, f))
                inp_new = os.path.join(tmpwd, "XSCALE.INP")
                shutil.copyfile(os.path.join(last_wd, "XSCALE.INP"), inp_new)
                modify_xdsinp(inp_new, [make_bin_str(d_min, self.d_max).split("= ")])

                try:
                    xscale.run_xscale(inp_new, cbf_to_dat=True, aniso_analysis=True,
                                      use_tmpdir_if_available=self.xscale_params.use_tmpdir_if_available)
                except:
                    print >>self.out, traceback.format_exc()

                xscale_hkl = os.path.abspath(os.path.join(tmpwd, "xscale.hkl"))

        if not os.path.isfile(os.path.join(tmpwd, "XSCALE.INP")):
            for f in "XSCALE.INP", "XSCALE.LP", "xscale.hkl", "pointless.log", "ccp4":
                os.symlink(os.path.relpath(os.path.join(last_wd, f), tmpwd),
                           os.path.join(tmpwd, f))

        if d_min is not None:
            self.dmin_est_at_cycles[cycle_number] = d_min
            os.rename(tmpwd, os.path.join(self.workdir_org, "run_%.2d_%.2fA"%(cycle_number, d_min)))
Пример #6
0
def run(params):
    xdsinp = "XDS.INP"
    kwds = dict(get_xdsinp_keyword(xdsinp))
    orgx_org, orgy_org = map(float, (kwds["ORGX"], kwds["ORGY"]))

    dx, dy = params.dx, params.dy
    if params.unit == "mm":
        assert "QX" in kwds
        assert "QY" in kwds
        dx /= float(kwds["QX"])
        dy /= float(kwds["QY"])

    backup_needed = files.generated_by_IDXREF + ("XDS.INP",)
    bk_prefix = make_backup(backup_needed)
    try:
        results = []
        for i in xrange(-params.nx, params.nx+1):
            for j in xrange(-params.ny, params.ny+1):
                work_name = "bs_x%+.2d_y%+.2d" % (i, j)
                orgx = orgx_org + i * dx
                orgy = orgy_org + j * dy
                print "Trying", orgx, orgy

                modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                                  ("ORGX", orgx),
                                                  ("ORGY", orgy),
                                                  ])
                call("xds")
                make_backup(backup_needed, work_name+"_")

                results.append([work_name, orgx, orgy])

        for ret in results:
                print ret,
                analyze_result(ret[0]+"_IDXREF.LP")
                

    finally:
        revert_files(backup_needed, bk_prefix)
Пример #7
0
def try_indexing_hard(wdir,
                      show_progress,
                      decilog,
                      known_sgnum=None,
                      known_cell=None,
                      tol_length=None,
                      tol_angle=None):
    idxref_lp = os.path.join(wdir, "IDXREF.LP")
    xdsinp = os.path.join(wdir, "XDS.INP")

    lp_org = idxreflp.IdxrefLp(idxref_lp)

    if lp_org.is_cell_maybe_half():
        backup_needed = ("XDS.INP", ) + xds_files.generated_by_IDXREF

        print >> decilog, " !! Cell may be halved. Trying doubled cell."
        bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)

        cell = lp_org.deduce_correct_cell_based_on_integerness()
        cell = " ".join(map(lambda x: "%.2f" % x, cell.parameters()))
        modify_xdsinp(xdsinp,
                      inp_params=[("JOB", "IDXREF"),
                                  ("SPACE_GROUP_NUMBER", "1"),
                                  ("UNIT_CELL_CONSTANTS", cell)])
        run_xds(wdir=wdir, show_progress=show_progress)

        if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
            revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)

            print >> decilog, "  .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
            bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)

            modify_xdsinp(xdsinp,
                          inp_params=[("JOB", "IDXREF"), ("SEPMIN", "4"),
                                      ("CLUSTER_RADIUS", "2")])
            run_xds(wdir=wdir, show_progress=show_progress)

            if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
                print >> decilog, "  .. not solved. Give up."
                revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
        else:
            print >> decilog, "  Now OK."
            remove_backups(backup_needed, bk_prefix, wdir=wdir)
            modify_xdsinp(xdsinp, inp_params=[
                ("SPACE_GROUP_NUMBER", "0"),
            ])

    # If Cell hint exists, try to use it..
    if known_sgnum > 0:
        flag_try_cell_hint = False
        xparm = os.path.join(wdir, "XPARM.XDS")
        if not os.path.isfile(xparm):
            flag_try_cell_hint = True
        else:
            xsxds = XPARM(xparm).crystal_symmetry()

            xsref = crystal.symmetry(known_cell, known_sgnum)
            cosets = reindex.reindexing_operators(xsref, xsxds, tol_length,
                                                  tol_angle)

            if cosets.double_cosets is None: flag_try_cell_hint = True

        if flag_try_cell_hint:
            print >> decilog, " Worth trying to use prior cell for indexing."
            modify_xdsinp(xdsinp,
                          inp_params=[
                              ("JOB", "IDXREF"),
                              ("UNIT_CELL_CONSTANTS",
                               " ".join(map(lambda x: "%.3f" % x,
                                            known_cell))),
                              ("SPACE_GROUP_NUMBER", "%d" % known_sgnum),
                          ])
            run_xds(wdir=wdir, show_progress=False)
            modify_xdsinp(xdsinp, inp_params=[
                ("SPACE_GROUP_NUMBER", "0"),
            ])
Пример #8
0
def run(params, out):
    print >> out, "Frames:", params.frames
    backup_needed = xds_files.generated_by_DEFPIX + (
        "XDS.INP",
        "BKGINIT.cbf",
    )
    bk_prefix = make_backup(backup_needed, wdir=params.xdsdir)

    ret = {}  # {frame: [matches, spots, predicted]}

    try:
        # run DEFPIX to limit resolution.
        modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"),
                      [("JOB", "DEFPIX"),
                       ("INCLUDE_RESOLUTION_RANGE", "50 %.2f" % params.d_min)])

        p = subprocess.Popen("xds", cwd=params.xdsdir)
        p.wait()

        # copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT)
        shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"),
                        os.path.join(params.xdsdir, "BKGINIT.cbf"))

        for frame in params.frames:
            print >> out, "Frame %d" % frame
            print >> out, "====================\n"
            # search spots
            if params.spotfinder == "xds":
                spotxds = get_colspot_result(frame_ranges=[
                    [frame, frame],
                ],
                                             wdir=params.xdsdir)
                spots = map(lambda x: x[:2],
                            spotxds.collected_spots(with_resolution=False))
            else:
                raise "Sorry!"

            # run INTEGRATE to collect predicted coords
            integrate_results = xds_predict_mitai.run(
                param_source=os.path.join(params.xdsdir, "XPARM.XDS"),
                frame_num=frame,
                wdir=params.xdsdir,
                need_adx=False,
                sigmar=params.sigmar,
                sigmab=params.sigmab)

            # read predicted coords
            tmp = filter(lambda x: x.endswith(".HKL"), integrate_results)
            if len(tmp) == 0:
                print >> out, "Integration failed!"
                ret[frame] = (0, len(spots), 0)
                continue

            integrate_hkl = tmp[0]
            cols = integrate_hkl_as_flex.reader(integrate_hkl, [],
                                                False).get_column_names()
            i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL")
            predicted = []

            for l in open(integrate_hkl):
                if l.startswith("!"): continue
                sp = l.split()
                predicted.append(map(float, (sp[i_xcal], sp[i_ycal])))

            # compare them
            nmatch = calc_matches(
                spots, predicted, params.distance_limit_in_px,
                open(
                    os.path.join(params.xdsdir,
                                 "matched_predicted_%.4d.adx" % frame), "w"))
            #nmatch = calc_matches(predicted, spots, params.distance_limit_in_px,
            #                      open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w"))

            ret[frame] = (nmatch, len(spots), len(predicted))

    finally:
        revert_files(backup_needed, bk_prefix, wdir=params.xdsdir)

    print >> out
    for frame in sorted(ret):
        nmatch, nspots, npredicted = ret[frame]
        print >> out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (
            frame, nmatch, npredicted, 100. * float(nmatch) /
            npredicted if npredicted > 0 else float("nan"))
        print >> out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (
            frame, nmatch, nspots,
            100. * float(nmatch) / nspots if nspots > 0 else float("nan"))
        print >> out

    return ret
Пример #9
0
def xds_sequence(root, params):
    print
    print os.path.relpath(root, params.topdir)

    init_lp = os.path.join(root, "INIT.LP")
    xparm = os.path.join(root, "XPARM.XDS")
    gxparm = os.path.join(root, "GXPARM.XDS")
    defpix_lp = os.path.join(root, "DEFPIX.LP")
    correct_lp = os.path.join(root, "CORRECT.LP")
    integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
    xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
    integrate_lp = os.path.join(root, "INTEGRATE.LP")
    spot_xds = os.path.join(root, "SPOT.XDS")
    xdsinp = os.path.join(root, "XDS.INP")

    assert os.path.isfile(xdsinp)
    if params.cell_prior.force: assert params.cell_prior.check

    xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))

    if params.cell_prior.sgnum > 0:
        xs_prior = crystal.symmetry(params.cell_prior.cell,
                                    params.cell_prior.sgnum)
    else:
        xs_prior = None

    decilog = multi_out()
    decilog.register("log",
                     open(os.path.join(root, "decision.log"), "a"),
                     atexit_send_to=None)
    try:
        print >> decilog, "xds_sequence started at %s in %s\n" % (
            time.strftime("%Y-%m-%d %H:%M:%S"), root)

        if not kamo_test_installation.tst_xds():
            print >> decilog, "XDS is not installed or expired!!"
            return

        if params.show_progress:
            decilog.register("stdout", sys.stdout)

        if params.mode == "initial" and params.resume and os.path.isfile(
                correct_lp):
            print >> decilog, " Already processed."
            return

        if params.mode == "recycle" and not os.path.isfile(gxparm):
            print >> decilog, "GXPARM.XDS not found. Cannot do recycle."
            return

        if params.fast_delphi and (params.nproc is None or params.nproc > 1):
            delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
            print >> decilog, " Setting delphi to ", delphi
            modify_xdsinp(xdsinp, inp_params=[
                ("DELPHI", str(delphi)),
            ])

        if params.nproc is not None and params.nproc > 1:
            modify_xdsinp(xdsinp,
                          inp_params=[
                              ("MAXIMUM_NUMBER_OF_PROCESSORS",
                               str(params.nproc)),
                          ])

        if params.mode == "initial":
            modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT")])
            run_xds(wdir=root, show_progress=params.show_progress)
            initlp = InitLp(init_lp)
            first_bad = initlp.check_bad_first_frames()
            if first_bad:
                print >> decilog, " first frames look bad (too weak) exposure:", first_bad
                new_data_range = map(
                    int,
                    dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
                new_data_range[0] = first_bad[-1] + 1
                print >> decilog, " changing DATA_RANGE= to", new_data_range
                modify_xdsinp(xdsinp,
                              inp_params=[("JOB", "INIT"),
                                          ("DATA_RANGE",
                                           "%d %d" % tuple(new_data_range))])
                for f in xds_files.generated_by_INIT:
                    util.rotate_file(os.path.join(root, f), copy=False)
                run_xds(wdir=root, show_progress=params.show_progress)

            # Peak search
            modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
            run_xds(wdir=root, show_progress=params.show_progress)
            if params.auto_frame_exclude_spot_based:
                sx = idxreflp.SpotXds(spot_xds)
                sx.set_xdsinp(xdsinp)
                spots = filter(lambda x: 5 < x[-1] < 30,
                               sx.collected_spots())  # low-res (5 A)
                frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1,
                                                spots))
                data_range = map(
                    int,
                    dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
                # XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
                h = numpy.histogram(frame_numbers,
                                    bins=numpy.arange(data_range[0],
                                                      data_range[1] + 2,
                                                      step=1))
                q14 = numpy.percentile(h[0], [25, 75])
                iqr = q14[1] - q14[0]
                cutoff = max(
                    h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5  # magic number
                print >> decilog, "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (
                    iqr, q14, cutoff)
                cut_frames = h[1][h[0] < cutoff]
                keep_frames = h[1][h[0] >= cutoff]
                print >> decilog, "DEBUG:: keep_frames=", keep_frames
                print >> decilog, "DEBUG::  cut_frames=", cut_frames

                if len(cut_frames) > 0:
                    cut_ranges = [
                        [cut_frames[0], cut_frames[0]],
                    ]
                    for fn in cut_frames:
                        if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
                        else: cut_ranges.append([fn, fn])

                    # Edit XDS.INP
                    cut_inp_str = "".join(
                        map(
                            lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(
                                x), cut_ranges))
                    open(xdsinp, "a").write("\n" + cut_inp_str)

                    # Edit SPOT.XDS
                    shutil.copyfile(spot_xds, spot_xds + ".org")
                    sx.write(open(spot_xds, "w"),
                             frame_selection=set(keep_frames))

            # Indexing
            if params.cell_prior.method == "use_first":
                modify_xdsinp(xdsinp,
                              inp_params=[
                                  ("JOB", "IDXREF"),
                                  ("UNIT_CELL_CONSTANTS", " ".join(
                                      map(lambda x: "%.3f" % x,
                                          params.cell_prior.cell))),
                                  ("SPACE_GROUP_NUMBER",
                                   "%d" % params.cell_prior.sgnum),
                              ])
            else:
                modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])

            run_xds(wdir=root, show_progress=params.show_progress)
            print >> decilog, ""  # TODO indexing stats like indexed percentage here.

            if params.tryhard:
                try_indexing_hard(root,
                                  params.show_progress,
                                  decilog,
                                  known_sgnum=params.cell_prior.sgnum,
                                  known_cell=params.cell_prior.cell,
                                  tol_length=params.cell_prior.tol_length,
                                  tol_angle=params.cell_prior.tol_angle)

            if not os.path.isfile(xparm):
                print >> decilog, " Indexing failed."
                return

            if params.cell_prior.sgnum > 0:
                # Check anyway
                xsxds = XPARM(xparm).crystal_symmetry()
                cosets = reindex.reindexing_operators(
                    xs_prior, xsxds, params.cell_prior.tol_length,
                    params.cell_prior.tol_angle)
                if cosets.double_cosets is None:
                    if params.cell_prior.check:
                        print >> decilog, " Incompatible cell. Indexing failed."
                        return
                    else:
                        print >> decilog, " Warning: Incompatible cell."

                elif params.cell_prior.method == "symm_constraint_only":
                    cell = xsxds.unit_cell().change_basis(
                        cosets.combined_cb_ops()[0])
                    print >> decilog, " Trying symmetry-constrained cell parameter:", cell
                    modify_xdsinp(xdsinp,
                                  inp_params=[
                                      ("JOB", "IDXREF"),
                                      ("UNIT_CELL_CONSTANTS", " ".join(
                                          map(lambda x: "%.3f" % x,
                                              cell.parameters()))),
                                      ("SPACE_GROUP_NUMBER",
                                       "%d" % params.cell_prior.sgnum),
                                  ])
                    for f in xds_files.generated_by_IDXREF:
                        util.rotate_file(os.path.join(root, f),
                                         copy=(f == "SPOT.XDS"))

                    run_xds(wdir=root, show_progress=params.show_progress)

                    if not os.path.isfile(xparm):
                        print >> decilog, " Indexing failed."
                        return

                    # Check again
                    xsxds = XPARM(xparm).crystal_symmetry()
                    if not xsxds.unit_cell().is_similar_to(
                            xs_prior.unit_cell(), params.cell_prior.tol_length,
                            params.cell_prior.tol_angle):
                        print >> decilog, "  Resulted in different cell. Indexing failed."
                        return

        elif params.mode == "recycle":
            print >> decilog, " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
                correct_lp, check_valid=True)
            for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
                                                         "plot_integrate.log"):
                util.rotate_file(os.path.join(root, f), copy=True)
            shutil.copyfile(gxparm + ".1", xparm)
        else:
            raise "Unknown mode (%s)" % params.mode

        # To Integration
        modify_xdsinp(xdsinp,
                      inp_params=[("JOB", "DEFPIX INTEGRATE"),
                                  ("INCLUDE_RESOLUTION_RANGE", "50 0")])
        run_xds(wdir=root, show_progress=params.show_progress)
        if os.path.isfile(integrate_lp):
            xds_plot_integrate.run(integrate_lp,
                                   os.path.join(root, "plot_integrate.log"))
        if not os.path.isfile(integrate_hkl):
            print >> decilog, " Integration failed."
            return

        # Make _noscale.HKL if needed
        if params.no_scaling:
            bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
            xparm_obj = XPARM(xparm)
            modify_xdsinp(xdsinp,
                          inp_params=[
                              ("JOB", "CORRECT"),
                              ("CORRECTIONS", ""),
                              ("NBATCH", "1"),
                              ("MINIMUM_I/SIGMA", "50"),
                              ("REFINE(CORRECT)", ""),
                              ("UNIT_CELL_CONSTANTS", " ".join(
                                  map(lambda x: "%.3f" % x,
                                      xparm_obj.unit_cell))),
                              ("SPACE_GROUP_NUMBER",
                               "%d" % xparm_obj.spacegroup),
                          ])
            print >> decilog, " running CORRECT without empirical scaling"
            run_xds(wdir=root, show_progress=params.show_progress)
            for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
                ff = os.path.join(root, f)
                if not os.path.isfile(ff): continue
                if ff.endswith(".cbf"):
                    os.remove(ff)
                else:
                    os.rename(ff, ff + "_noscale")

            revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)

        # Run pointless
        pointless_integrate = {}
        if params.use_pointless:
            worker = Pointless()
            pointless_integrate = worker.run_for_symm(
                xdsin=integrate_hkl,
                logout=os.path.join(root, "pointless_integrate.log"))
            if "symm" in pointless_integrate:
                symm = pointless_integrate["symm"]
                print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
                )
                if xs_prior:
                    if xtal.is_same_space_group_ignoring_enantiomorph(
                            symm.space_group(), xs_prior.space_group()):
                        print >> decilog, " which is consistent with given symmetry."
                    elif xtal.is_same_laue_symmetry(symm.space_group(),
                                                    xs_prior.space_group()):
                        print >> decilog, " which has consistent Laue symmetry with given symmetry."
                    else:
                        print >> decilog, " which is inconsistent with given symmetry."

                sgnum = symm.space_group_info().type().number()
                cell = " ".join(
                    map(lambda x: "%.2f" % x,
                        symm.unit_cell().parameters()))
                modify_xdsinp(xdsinp,
                              inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
                                          ("UNIT_CELL_CONSTANTS", cell)])
            else:
                print >> decilog, " pointless failed."

        flag_do_not_change_symm = False

        if xs_prior and params.cell_prior.force:
            modify_xdsinp(xdsinp,
                          inp_params=[("UNIT_CELL_CONSTANTS", " ".join(
                              map(lambda x: "%.3f" % x,
                                  params.cell_prior.cell))),
                                      ("SPACE_GROUP_NUMBER",
                                       "%d" % params.cell_prior.sgnum)])
            flag_do_not_change_symm = True
        elif params.cell_prior.method == "correct_only":
            xsxds = XPARM(xparm).crystal_symmetry()
            cosets = reindex.reindexing_operators(xs_prior, xsxds,
                                                  params.cell_prior.tol_length,
                                                  params.cell_prior.tol_angle)
            if cosets.double_cosets is not None:
                cell = xsxds.unit_cell().change_basis(
                    cosets.combined_cb_ops()[0])
                print >> decilog, " Using given symmetry in CORRECT with symmetry constraints:", cell
                modify_xdsinp(xdsinp,
                              inp_params=[
                                  ("UNIT_CELL_CONSTANTS", " ".join(
                                      map(lambda x: "%.3f" % x,
                                          cell.parameters()))),
                                  ("SPACE_GROUP_NUMBER",
                                   "%d" % params.cell_prior.sgnum),
                              ])
                flag_do_not_change_symm = True
            else:
                print >> decilog, " Tried to use given symmetry in CORRECT, but cell in integration is incompatible."

        # Do Scaling
        modify_xdsinp(xdsinp, inp_params=[
            ("JOB", "CORRECT"),
        ])

        run_xds(wdir=root, show_progress=params.show_progress)

        if not os.path.isfile(xac_hkl):
            print >> decilog, " CORRECT failed."
            return

        if not os.path.isfile(gxparm):
            print >> decilog, " Refinement in CORRECT failed."

        print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
            correct_lp, check_valid=True)

        ret = calc_merging_stats(xac_hkl)
        if params.cut_resolution:
            if ret is not None and ret[0] is not None:
                d_min = ret[0]
                modify_xdsinp(xdsinp,
                              inp_params=[("JOB", "CORRECT"),
                                          ("INCLUDE_RESOLUTION_RANGE",
                                           "50 %.2f" % d_min)])
                print >> decilog, " Re-scale at %.2f A" % d_min
                os.rename(os.path.join(root, "CORRECT.LP"),
                          os.path.join(root, "CORRECT_fullres.LP"))
                os.rename(xac_hkl, os.path.join(root, "XDS_ASCII_fullres.HKL"))
                run_xds(wdir=root, show_progress=params.show_progress)
                print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
                    correct_lp, check_valid=True)
                print >> decilog, " (Original files are saved as *_fullres.*)"
            else:
                print >> decilog, "error: Can't decide resolution."

        last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)

        # Run pointless and (if result is different from INTEGRATE) re-scale.
        if params.use_pointless:
            worker = Pointless()
            pointless_correct = worker.run_for_symm(
                xdsin=xac_hkl,
                logout=os.path.join(root, "pointless_correct.log"))
            pointless_best_symm = None

            if "symm" in pointless_correct:
                symm = pointless_correct["symm"]
                need_rescale = False

                if pointless_integrate.get("symm"):
                    symm_by_integrate = pointless_integrate["symm"]

                    if not xtal.is_same_laue_symmetry(
                            symm_by_integrate.space_group(),
                            symm.space_group()):
                        print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
                            symm.space_group_info(),
                            symm_by_integrate.space_group_info())
                        prob_integrate = pointless_integrate.get(
                            "laue_prob", float("nan"))
                        prob_correct = pointless_correct.get(
                            "laue_prob", float("nan"))

                        print >> decilog, " Prob(%s |INTEGRATE), Prob(%s |CORRECT) = %.4f, %.4f." % (
                            symm_by_integrate.space_group_info(),
                            symm.space_group_info(), prob_integrate,
                            prob_correct)
                        if prob_correct > prob_integrate:
                            need_rescale = True
                            pointless_best_symm = symm
                        else:
                            pointless_best_symm = symm_by_integrate
                else:
                    need_rescale = True
                    pointless_best_symm = symm
                    print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
                    )
                    if xs_prior:
                        if xtal.is_same_space_group_ignoring_enantiomorph(
                                symm.space_group(), xs_prior.space_group()):
                            print >> decilog, " which is consistent with given symmetry."
                        elif xtal.is_same_laue_symmetry(
                                symm.space_group(), xs_prior.space_group()):
                            print >> decilog, " which has consistent Laue symmetry with given symmetry."
                        else:
                            print >> decilog, " which is inconsistent with given symmetry."

                if need_rescale and not flag_do_not_change_symm:
                    sgnum = symm.space_group_info().type().number()
                    cell = " ".join(
                        map(lambda x: "%.2f" % x,
                            symm.unit_cell().parameters()))
                    modify_xdsinp(xdsinp,
                                  inp_params=[
                                      ("JOB", "CORRECT"),
                                      ("SPACE_GROUP_NUMBER", "%d" % sgnum),
                                      ("UNIT_CELL_CONSTANTS", cell),
                                      ("INCLUDE_RESOLUTION_RANGE", "50 0")
                                  ])

                    run_xds(wdir=root, show_progress=params.show_progress)

                    ret = calc_merging_stats(xac_hkl)

                    if params.cut_resolution:
                        if ret is not None and ret[0] is not None:
                            d_min = ret[0]
                            modify_xdsinp(xdsinp,
                                          inp_params=[
                                              ("JOB", "CORRECT"),
                                              ("INCLUDE_RESOLUTION_RANGE",
                                               "50 %.2f" % d_min)
                                          ])
                            print >> decilog, " Re-scale at %.2f A" % d_min
                            os.rename(os.path.join(root, "CORRECT.LP"),
                                      os.path.join(root, "CORRECT_fullres.LP"))
                            os.rename(
                                xac_hkl,
                                os.path.join(root, "XDS_ASCII_fullres.HKL"))
                            run_xds(wdir=root,
                                    show_progress=params.show_progress)
                            print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
                                correct_lp, check_valid=True)
                            print >> decilog, " (Original files are saved as *_fullres.*)"
                        else:
                            print >> decilog, "error: Can't decide resolution."
                            for f in ("CORRECT_fullres.LP",
                                      "XDS_ASCII_fullres.HKL"):
                                if os.path.isfile(os.path.join(root, f)):
                                    print >> decilog, "removing", f
                                    os.remove(os.path.join(root, f))

                    ISa = correctlp.get_ISa(correct_lp, check_valid=True)

                    if ISa >= last_ISa or last_ISa != last_ISa:  # if improved or last_ISa is nan
                        print >> decilog, "ISa improved= %.2f" % ISa
                    else:
                        print >> decilog, "ISa got worse= %.2f" % ISa

            if pointless_best_symm:
                xac_symm = XDS_ASCII(xac_hkl, read_data=False).symm
                if not xtal.is_same_space_group_ignoring_enantiomorph(
                        xac_symm.space_group(),
                        pointless_best_symm.space_group()):
                    if xtal.is_same_laue_symmetry(
                            xac_symm.space_group(),
                            pointless_best_symm.space_group()):
                        tmp = "same Laue symmetry"
                    else:
                        tmp = "different Laue symmetry"
                    print >> decilog, "WARNING: symmetry in scaling is different from Pointless result (%s)." % tmp

        run_xdsstat(wdir=root)
        print
        if params.make_report: html_report.make_individual_report(root, root)
    except:
        print >> decilog, traceback.format_exc()
    finally:
        print >> decilog, "\nxds_sequence finished at %s" % time.strftime(
            "%Y-%m-%d %H:%M:%S")
        decilog.close()
Пример #10
0
def find_spots(img_file, params):
    """
    Use XDS to locate spots.
    If params.xds.do_defpix is true, DEFPIX will be run. For DEFPIX, dummy XPARM.XDS is needed. Thanks to DEFPIX, we can mask beam stop shadow and remove areas outside the resolution range.
    If false, we need to set TRUSTED_REGION to exclude regions outside resolution range, but it is independent of beam center. Maybe we should remove spots outside the resolution range after XDS run?
    """

    # Test if ramdisk available
    if os.path.isdir("/dev/shm"):
        work_dir = tempfile.mkdtemp(
            prefix="shika_x_" +
            os.path.splitext(os.path.basename(img_file))[0],
            dir="/dev/shm")
    else:
        work_dir = os.path.join(
            params.work_dir,
            "xds_" + os.path.splitext(os.path.basename(img_file))[0])

    xdsinp = os.path.join(work_dir, "XDS.INP")
    spot_xds = os.path.join(work_dir, "SPOT.XDS")

    if not os.path.exists(work_dir):
        os.mkdir(work_dir)

    template_str, min_frame, max_frame = dataset.group_img_files_template(
        [img_file])[0]

    im = XIO.Image(img_file)

    # Remove lines if None (means to use default)
    params_maps = [
        ("strong_pixel", "STRONG_PIXEL="),
        ("minimum_number_of_pixels_in_a_spot",
         "MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT="),
        ("background_pixel", "BACKGROUND_PIXEL="),
        ("maximum_number_of_strong_pixels",
         "MAXIMUM_NUMBER_OF_STRONG_PIXELS="),
        ("spot_maximum_centroid", "SPOT_MAXIMUM-CENTROID="),
        ("reflecting_range", "REFLECTING_RANGE="),
        ("value_range_for_trusted_detector_pixels",
         "VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS="),
    ]
    tmp = xds_inp_template.splitlines()
    for p, x in params_maps:
        if getattr(params.xds, p) is None:
            tmp = filter(lambda s: not s.startswith(x), tmp)
    inp_template = "\n".join(tmp)

    # Prepare XDS.INP
    inp_str = inp_template % dict(
        template=os.path.relpath(template_str, work_dir),
        framenum=min_frame,
        orgx=im.header["BeamX"] / im.header["PixelX"],
        orgy=im.header["BeamY"] / im.header["PixelY"],
        distance=im.header["Distance"],
        osc_range=im.header["PhiWidth"],
        wavelength=im.header["Wavelength"],
        strong_pixel=params.xds.strong_pixel,
        min_pixels=params.xds.minimum_number_of_pixels_in_a_spot,
        background_pixel=params.xds.background_pixel,
        max_strong_pixels=params.xds.maximum_number_of_strong_pixels,
        spot_maximum_centroid=params.xds.spot_maximum_centroid,
        reflecting_range=params.xds.reflecting_range,
        nx=im.header["Width"],
        ny=im.header["Height"],
        qx=im.header["PixelX"],
        qy=im.header["PixelY"],
        defpix_trusted1=params.xds.value_range_for_trusted_detector_pixels[0],
        defpix_trusted2=params.xds.value_range_for_trusted_detector_pixels[1])

    open(xdsinp, "w").write(inp_str)

    if params.xds.do_defpix:
        xp = xparm.XPARM()
        xp.set_info_from_xdsinp(xdsinp)
        open(os.path.join(work_dir, "XPARM.XDS"), "w").write(xp.xparm_str())
        modify_xdsinp(xdsinp,
                      inp_params=[("JOB", "XYCORR INIT DEFPIX"),
                                  ("INCLUDE_RESOLUTION_RANGE",
                                   res_range_for_xds(params.distl.res.outer,
                                                     params.distl.res.inner))])
        call("xds",
             wdir=work_dir,
             stdout=open(os.path.join(work_dir, "xds.log"), "w"))
        shutil.copy(os.path.join(work_dir, "BKGPIX.cbf"),
                    os.path.join(work_dir, "BKGINIT.cbf"))
        modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
    else:
        modify_xdsinp(xdsinp,
                      inp_params=[
                          ("TRUSTED_REGION",
                           res_range_to_trusted_region(params.distl.res.outer,
                                                       params.distl.res.inner,
                                                       im.header))
                      ])
        open(os.path.join(work_dir, "xds.log"), "w").write("")

    # Run XDS
    rotate_file(spot_xds)
    call("xds",
         wdir=work_dir,
         stdout=open(os.path.join(work_dir, "xds.log"), "a"))

    # Extract results
    spots = []  # (x, y, d, intensity)
    if os.path.isfile(spot_xds):
        for l in open(spot_xds):
            x, y, z, intensity = map(lambda x: float(x), l.strip().split())
            d = coord_to_resol(x, y, im.header)
            spots.append((x, y, d, intensity))

    # Delete dir
    shutil.rmtree(work_dir)

    return spots
Пример #11
0
def find_spots(img_file, params):
    """
    Use XDS to locate spots.
    If params.xds.do_defpix is true, DEFPIX will be run. For DEFPIX, dummy XPARM.XDS is needed. Thanks to DEFPIX, we can mask beam stop shadow and remove areas outside the resolution range.
    If false, we need to set TRUSTED_REGION to exclude regions outside resolution range, but it is independent of beam center. Maybe we should remove spots outside the resolution range after XDS run?
    """

    # Test if ramdisk available
    if os.path.isdir("/dev/shm"):
        work_dir = tempfile.mkdtemp(prefix="shika_x_"+os.path.splitext(os.path.basename(img_file))[0], dir="/dev/shm")
    else:
        work_dir = os.path.join(params.work_dir, "xds_"+os.path.splitext(os.path.basename(img_file))[0])

    xdsinp = os.path.join(work_dir, "XDS.INP")
    spot_xds = os.path.join(work_dir, "SPOT.XDS")

    if not os.path.exists(work_dir):
        os.mkdir(work_dir)

    template_str, min_frame, max_frame = dataset.group_img_files_template([img_file])[0]

    im = XIO.Image(img_file)

    # Remove lines if None (means to use default)
    params_maps = [("strong_pixel", "STRONG_PIXEL="),
                   ("minimum_number_of_pixels_in_a_spot", "MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT="),
                   ("background_pixel", "BACKGROUND_PIXEL="),
                   ("maximum_number_of_strong_pixels", "MAXIMUM_NUMBER_OF_STRONG_PIXELS="),
                   ("spot_maximum_centroid", "SPOT_MAXIMUM-CENTROID="),
                   ("reflecting_range", "REFLECTING_RANGE="),
                   ("value_range_for_trusted_detector_pixels", "VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS="),
                   ]
    tmp = xds_inp_template.splitlines()
    for p, x in params_maps:
        if getattr(params.xds, p) is None:
            tmp = filter(lambda s:not s.startswith(x), tmp)
    inp_template = "\n".join(tmp)

    # Prepare XDS.INP
    inp_str = inp_template%dict(template=os.path.relpath(template_str, work_dir),
                                framenum=min_frame,
                                orgx=im.header["BeamX"]/im.header["PixelX"],
                                orgy=im.header["BeamY"]/im.header["PixelY"],
                                distance=im.header["Distance"],
                                osc_range=im.header["PhiWidth"],
                                wavelength=im.header["Wavelength"],
                                strong_pixel=params.xds.strong_pixel,
                                min_pixels=params.xds.minimum_number_of_pixels_in_a_spot,
                                background_pixel=params.xds.background_pixel,
                                max_strong_pixels=params.xds.maximum_number_of_strong_pixels,
                                spot_maximum_centroid=params.xds.spot_maximum_centroid,
                                reflecting_range=params.xds.reflecting_range,
                                nx=im.header["Width"], ny=im.header["Height"],
                                qx=im.header["PixelX"], qy=im.header["PixelY"],
                                defpix_trusted1=params.xds.value_range_for_trusted_detector_pixels[0],
                                defpix_trusted2=params.xds.value_range_for_trusted_detector_pixels[1]
                                )

    open(xdsinp, "w").write(inp_str)

    if params.xds.do_defpix:
        xp = xparm.XPARM()
        xp.set_info_from_xdsinp(xdsinp)
        open(os.path.join(work_dir, "XPARM.XDS"), "w").write(xp.xparm_str())
        modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT DEFPIX"),
                                          ("INCLUDE_RESOLUTION_RANGE", res_range_for_xds(params.distl.res.outer, params.distl.res.inner))
                                          ])
        call("xds", wdir=work_dir, stdout=open(os.path.join(work_dir, "xds.log"), "w"))
        shutil.copy(os.path.join(work_dir, "BKGPIX.cbf"), os.path.join(work_dir, "BKGINIT.cbf"))
        modify_xdsinp(xdsinp, inp_params=[("JOB","COLSPOT")])
    else:
        modify_xdsinp(xdsinp, inp_params=[("TRUSTED_REGION", res_range_to_trusted_region(params.distl.res.outer, params.distl.res.inner, im.header))
                                          ])
        open(os.path.join(work_dir, "xds.log"), "w").write("")

    # Run XDS
    rotate_file(spot_xds)
    call("xds", wdir=work_dir, stdout=open(os.path.join(work_dir, "xds.log"), "a"))

    # Extract results
    spots = [] # (x, y, d, intensity)
    if os.path.isfile(spot_xds):
        for l in open(spot_xds):
            x, y, z, intensity = map(lambda x:float(x), l.strip().split())
            d = coord_to_resol(x, y, im.header)
            spots.append((x, y, d, intensity))

    # Delete dir
    shutil.rmtree(work_dir)

    return spots
Пример #12
0
def rescale_with_specified_symm(topdir,
                                dirs,
                                symms,
                                out,
                                sgnum=None,
                                reference_symm=None):
    assert (sgnum, reference_symm).count(None) == 1

    if sgnum is not None:
        sgnum_laue = sgtbx.space_group_info(sgnum).group(
        ).build_derived_reflection_intensity_group(False).type().number()

        matches = filter(
            lambda x: x.reflection_intensity_symmetry(False).space_group_info(
            ).type().number() == sgnum_laue, symms)
        matched_cells = numpy.array(
            map(lambda x: x.unit_cell().parameters(), matches))
        median_cell = map(lambda x: numpy.median(matched_cells[:, x]),
                          xrange(6))

        reference_symm = crystal.symmetry(median_cell, sgnum)
    else:
        sgnum = reference_symm.space_group_info().type().number()
        sgnum_laue = reference_symm.space_group(
        ).build_derived_reflection_intensity_group(False).type().number()

    print >> out
    print >> out, "Re-scaling with specified symmetry:", reference_symm.space_group_info(
    ).symbol_and_number()
    print >> out, " reference cell:", reference_symm.unit_cell()
    print >> out
    print >> out

    cells = {}  # cell and file
    for sym, wd in zip(symms, dirs):
        print >> out, os.path.relpath(wd, topdir),

        # Find appropriate data
        xac_file = util.return_first_found_file(
            ("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
             "XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
             "XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
            wd=wd)
        if xac_file is None:
            print >> out, "Can't find XDS_ASCII file in %s" % wd
            continue

        xac = XDS_ASCII(xac_file, read_data=False)
        print >> out, "%s %s (%s)" % (
            os.path.basename(xac_file), xac.symm.space_group_info(), ",".join(
                map(lambda x: "%.2f" % x,
                    xac.symm.unit_cell().parameters())))

        if xac.symm.reflection_intensity_symmetry(
                False).space_group_info().type().number() == sgnum_laue:
            if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(),
                                                  0.1, 10):
                print >> out, "  Already scaled with specified symmetry"
                cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()),
                             xac_file)
                continue

        xdsinp = os.path.join(wd, "XDS.INP")
        cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
                                              20)

        if len(cosets.combined_cb_ops()) == 0:
            print >> out, "Can't find operator:"
            sym.show_summary(out, " ")
            reference_symm.show_summary(out, " ")
            continue

        newcell = reference_symm.space_group().average_unit_cell(
            xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
        newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
        print >> out, "Scaling with transformed cell:", newcell

        #for f in xds_files.generated_by_CORRECT:
        #    util.rotate_file(os.path.join(wd, f))
        bk_prefix = make_backup(xds_files.generated_by_CORRECT,
                                wdir=wd,
                                quiet=True)

        modify_xdsinp(
            xdsinp,
            inp_params=[
                ("JOB", "CORRECT"),
                ("SPACE_GROUP_NUMBER", "%d" % sgnum),
                ("UNIT_CELL_CONSTANTS", newcell),
                ("INCLUDE_RESOLUTION_RANGE", "50 0"),
                ("CORRECTIONS", ""),
                ("NBATCH", "1"),
                ("MINIMUM_I/SIGMA", None),  # use default
                ("REFINE(CORRECT)", None),  # use default
            ])
        run_xds(wd)
        for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
            if os.path.exists(os.path.join(wd, f)):
                shutil.copyfile(os.path.join(wd, f),
                                os.path.join(wd, f + "_rescale"))

        revert_files(xds_files.generated_by_CORRECT,
                     bk_prefix,
                     wdir=wd,
                     quiet=True)

        new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
        new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
        if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
            cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
            print "OK:", cells[wd][0]
        else:
            print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
            continue

    return cells, reference_symm
Пример #13
0
def run(params, out):
    print >>out, "Frames:", params.frames
    backup_needed = xds_files.generated_by_DEFPIX + ("XDS.INP","BKGINIT.cbf",)
    bk_prefix = make_backup(backup_needed, wdir=params.xdsdir)

    ret = {} # {frame: [matches, spots, predicted]}

    try:
        # run DEFPIX to limit resolution.
        modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"),
                      [("JOB", "DEFPIX"), ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%params.d_min)])

        p = subprocess.Popen("xds", cwd=params.xdsdir)
        p.wait()

        # copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT)
        shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"),
                        os.path.join(params.xdsdir, "BKGINIT.cbf"))
        
        for frame in params.frames:
            print >>out, "Frame %d" % frame
            print >>out, "====================\n"
            # search spots
            if params.spotfinder == "xds":
                spotxds = get_colspot_result(frame_ranges=[[frame, frame],], wdir=params.xdsdir)
                spots = map(lambda x: x[:2], spotxds.collected_spots(with_resolution=False))
            else:
                raise "Sorry!"

            # run INTEGRATE to collect predicted coords
            integrate_results = xds_predict_mitai.run(param_source=os.path.join(params.xdsdir, "XPARM.XDS"), 
                                                      frame_num=frame,
                                                      wdir=params.xdsdir, need_adx=False,
                                                      sigmar=params.sigmar, sigmab=params.sigmab)

            # read predicted coords
            tmp = filter(lambda x:x.endswith(".HKL"), integrate_results)
            if len(tmp) == 0:
                print >>out, "Integration failed!"
                ret[frame] = (0, len(spots), 0)
                continue

            integrate_hkl = tmp[0]
            cols = integrate_hkl_as_flex.reader(integrate_hkl, [], False).get_column_names()
            i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL")
            predicted = []

            for l in open(integrate_hkl):
                if l.startswith("!"): continue
                sp = l.split()
                predicted.append(map(float, (sp[i_xcal], sp[i_ycal])))

            # compare them
            nmatch = calc_matches(spots, predicted, params.distance_limit_in_px,
                                  open(os.path.join(params.xdsdir, "matched_predicted_%.4d.adx"%frame), "w"))
            #nmatch = calc_matches(predicted, spots, params.distance_limit_in_px,
            #                      open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w"))

            ret[frame] = (nmatch, len(spots), len(predicted))

    finally:
        revert_files(backup_needed, bk_prefix, wdir=params.xdsdir)

    print >>out
    for frame in sorted(ret):
        nmatch, nspots, npredicted = ret[frame]
        print >>out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (frame, nmatch, npredicted,
                                                                     100.*float(nmatch)/npredicted if npredicted>0 else float("nan"))
        print >>out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (frame, nmatch, nspots,
                                                                     100.*float(nmatch)/nspots if nspots>0 else float("nan"))
        print >>out


    return ret
Пример #14
0
def xds_sequence(root, params):
    print
    print os.path.relpath(root, params.topdir)

    xparm = os.path.join(root, "XPARM.XDS")
    gxparm = os.path.join(root, "GXPARM.XDS")
    defpix_lp = os.path.join(root, "DEFPIX.LP")
    correct_lp = os.path.join(root, "CORRECT.LP")
    integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
    xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
    integrate_lp = os.path.join(root, "INTEGRATE.LP")
    spot_xds = os.path.join(root, "SPOT.XDS")
    xdsinp = os.path.join(root, "XDS.INP")

    assert os.path.isfile(xdsinp)

    xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))

    decilog = multi_out()
    decilog.register("log",
                     open(os.path.join(root, "decision.log"), "a"),
                     atexit_send_to=None)

    print >> decilog, "xds_sequence started at %s in %s\n" % (
        time.strftime("%Y-%m-%d %H:%M:%S"), root)

    if params.show_progress:
        decilog.register("stdout", sys.stdout)

    if params.mode == "initial" and params.resume and os.path.isfile(
            correct_lp):
        print " Already processed."
        return

    if params.mode == "recycle" and not os.path.isfile(gxparm):
        print "GXPARM.XDS not found. Cannot do recycle."
        return

    if params.fast_delphi and (params.nproc is None or params.nproc > 1):
        delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
        print " Setting delphi to ", delphi
        modify_xdsinp(xdsinp, inp_params=[
            ("DELPHI", str(delphi)),
        ])

    if params.nproc is not None and params.nproc > 1:
        modify_xdsinp(xdsinp,
                      inp_params=[
                          ("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
                      ])

    if params.mode == "initial":
        # Peak search
        modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT")])
        run_xds(wdir=root, show_progress=params.show_progress)
        if params.auto_frame_exclude_spot_based:
            sx = idxreflp.SpotXds(spot_xds)
            sx.set_xdsinp(xdsinp)
            spots = filter(lambda x: 5 < x[-1] < 30,
                           sx.collected_spots())  # low-res (5 A)
            frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1, spots))
            data_range = map(int, xdsinp_dict["DATA_RANGE"].split())
            # XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
            h = numpy.histogram(frame_numbers,
                                bins=numpy.arange(data_range[0],
                                                  data_range[1] + 2,
                                                  step=1))
            q14 = numpy.percentile(h[0], [25, 75])
            iqr = q14[1] - q14[0]
            cutoff = max(h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5  # magic number
            print "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (iqr, q14,
                                                                 cutoff)
            cut_frames = h[1][h[0] < cutoff]
            keep_frames = h[1][h[0] >= cutoff]
            print "DEBUG:: keep_frames=", keep_frames
            print "DEBUG::  cut_frames=", cut_frames

            if len(cut_frames) > 0:
                cut_ranges = [
                    [cut_frames[0], cut_frames[0]],
                ]
                for fn in cut_frames:
                    if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
                    else: cut_ranges.append([fn, fn])

                # Edit XDS.INP
                cut_inp_str = "".join(
                    map(lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(x),
                        cut_ranges))
                open(xdsinp, "a").write("\n" + cut_inp_str)

                # Edit SPOT.XDS
                shutil.copyfile(spot_xds, spot_xds + ".org")
                sx.write(open(spot_xds, "w"), frame_selection=set(keep_frames))

        # Indexing
        modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
        run_xds(wdir=root, show_progress=params.show_progress)
        print  # indexing stats like indexed percentage here.

        if params.tryhard:
            try_indexing_hard(root,
                              params.show_progress,
                              decilog,
                              known_sgnum=params.cell_prior.sgnum,
                              known_cell=params.cell_prior.cell,
                              tol_length=params.cell_prior.tol_length,
                              tol_angle=params.cell_prior.tol_angle)

        if not os.path.isfile(xparm):
            print >> decilog, " Indexing failed."
            return

        if params.cell_prior.check and params.cell_prior.sgnum > 0:
            xsxds = XPARM(xparm).crystal_symmetry()
            xsref = crystal.symmetry(params.cell_prior.cell,
                                     params.cell_prior.sgnum)
            cosets = reindex.reindexing_operators(xsref, xsxds,
                                                  params.cell_prior.tol_length,
                                                  params.cell_prior.tol_angle)
            if cosets.double_cosets is None:
                print >> decilog, " Incompatible cell. Indexing failed."
                return

    elif params.mode == "recycle":
        print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
            correct_lp, check_valid=True)
        for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
                                                     "plot_integrate.log"):
            util.rotate_file(os.path.join(root, f), copy=True)
        shutil.copyfile(gxparm + ".1", xparm)
    else:
        raise "Unknown mode (%s)" % params.mode

    # To Integration
    modify_xdsinp(xdsinp,
                  inp_params=[("JOB", "DEFPIX INTEGRATE"),
                              ("INCLUDE_RESOLUTION_RANGE", "50 0")])
    run_xds(wdir=root, show_progress=params.show_progress)
    if os.path.isfile(integrate_lp):
        xds_plot_integrate.run(integrate_lp,
                               os.path.join(root, "plot_integrate.log"))
    if not os.path.isfile(integrate_hkl):
        print >> decilog, " Integration failed."
        return

    # Make _noscale.HKL if needed
    if params.no_scaling:
        bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
        xparm_obj = XPARM(xparm)
        modify_xdsinp(xdsinp,
                      inp_params=[
                          ("JOB", "CORRECT"),
                          ("CORRECTIONS", ""),
                          ("NBATCH", "1"),
                          ("MINIMUM_I/SIGMA", "50"),
                          ("REFINE(CORRECT)", ""),
                          ("UNIT_CELL_CONSTANTS", " ".join(
                              map(lambda x: "%.3f" % x, xparm_obj.unit_cell))),
                          ("SPACE_GROUP_NUMBER", "%d" % xparm_obj.spacegroup),
                      ])
        print >> decilog, " running CORRECT without empirical scaling"
        run_xds(wdir=root, show_progress=params.show_progress)
        for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
            ff = os.path.join(root, f)
            if not os.path.isfile(ff): continue
            if ff.endswith(".cbf"):
                os.remove(ff)
            else:
                os.rename(ff, ff + "_noscale")

        revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)

    # Run pointless
    symm_by_integrate = None
    if params.use_pointless:
        worker = Pointless()
        result = worker.run_for_symm(xdsin=integrate_hkl,
                                     logout=os.path.join(
                                         root, "pointless_integrate.log"))
        if "symm" in result:
            symm = result["symm"]
            print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
            )
            sgnum = symm.space_group_info().type().number()
            cell = " ".join(
                map(lambda x: "%.2f" % x,
                    symm.unit_cell().parameters()))
            modify_xdsinp(xdsinp,
                          inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
                                      ("UNIT_CELL_CONSTANTS", cell)])
            symm_by_integrate = symm
        else:
            print >> decilog, " pointless failed."

    # Do Scaling
    modify_xdsinp(xdsinp, inp_params=[
        ("JOB", "CORRECT"),
    ])

    run_xds(wdir=root, show_progress=params.show_progress)

    if not os.path.isfile(gxparm):
        print >> decilog, " Scaling failed."
        return

    print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp,
                                                           check_valid=True)

    ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
    if params.cut_resolution:
        if ret is not None and ret[0] is not None:
            d_min = ret[0]
            modify_xdsinp(xdsinp,
                          inp_params=[("JOB", "CORRECT"),
                                      ("INCLUDE_RESOLUTION_RANGE",
                                       "50 %.2f" % d_min)])
            print >> decilog, " Re-scale at %.2f A" % d_min
            os.rename(os.path.join(root, "CORRECT.LP"),
                      os.path.join(root, "CORRECT_fullres.LP"))
            os.rename(os.path.join(root, "XDS_ASCII.HKL"),
                      os.path.join(root, "XDS_ASCII_fullres.HKL"))
            run_xds(wdir=root, show_progress=params.show_progress)
            print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
                correct_lp, check_valid=True)
            print >> decilog, " (Original files are saved as *_fullres.*)"
        else:
            print >> decilog, "error: Can't decide resolution."

    last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)

    # Run pointless and (if result is different from INTEGRATE) re-scale.
    if params.use_pointless:
        worker = Pointless()
        result = worker.run_for_symm(xdsin=xac_hkl,
                                     logout=os.path.join(
                                         root, "pointless_correct.log"))
        if "symm" in result:
            symm = result["symm"]
            need_rescale = False

            if symm_by_integrate is not None:
                if not xtal.is_same_laue_symmetry(
                        symm_by_integrate.space_group(), symm.space_group()):
                    print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
                        symm.space_group_info(),
                        symm_by_integrate.space_group_info())
                    need_rescale = True
            else:
                print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
                )
                need_rescale = True

            if need_rescale:
                # make backup, and do correct and compare ISa
                # if ISa got worse, revert the result.
                backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL",
                                 "CORRECT_fullres.LP", "merging_stats.pkl",
                                 "merging_stats.log")
                backup_needed += xds_files.generated_by_CORRECT
                bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)

                sgnum = symm.space_group_info().type().number()
                cell = " ".join(
                    map(lambda x: "%.2f" % x,
                        symm.unit_cell().parameters()))
                modify_xdsinp(xdsinp,
                              inp_params=[("JOB", "CORRECT"),
                                          ("SPACE_GROUP_NUMBER", "%d" % sgnum),
                                          ("UNIT_CELL_CONSTANTS", cell),
                                          ("INCLUDE_RESOLUTION_RANGE", "50 0")
                                          ])

                run_xds(wdir=root, show_progress=params.show_progress)

                ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))

                if params.cut_resolution:
                    if ret is not None and ret[0] is not None:
                        d_min = ret[0]
                        modify_xdsinp(xdsinp,
                                      inp_params=[("JOB", "CORRECT"),
                                                  ("INCLUDE_RESOLUTION_RANGE",
                                                   "50 %.2f" % d_min)])
                        print >> decilog, " Re-scale at %.2f A" % d_min
                        os.rename(os.path.join(root, "CORRECT.LP"),
                                  os.path.join(root, "CORRECT_fullres.LP"))
                        os.rename(os.path.join(root, "XDS_ASCII.HKL"),
                                  os.path.join(root, "XDS_ASCII_fullres.HKL"))
                        run_xds(wdir=root, show_progress=params.show_progress)
                        print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
                            correct_lp, check_valid=True)
                        print >> decilog, " (Original files are saved as *_fullres.*)"
                    else:
                        print >> decilog, "error: Can't decide resolution."
                        for f in ("CORRECT_fullres.LP",
                                  "XDS_ASCII_fullres.HKL"):
                            if os.path.isfile(os.path.join(root, f)):
                                print >> decilog, "removing", f
                                os.remove(os.path.join(root, f))

                ISa = correctlp.get_ISa(correct_lp, check_valid=True)

                if ISa >= last_ISa or last_ISa != last_ISa:  # if improved or last_ISa is nan
                    print >> decilog, "ISa improved= %.2f" % ISa
                    remove_backups(backup_needed, bk_prefix, wdir=root)
                else:
                    print >> decilog, "ISa got worse= %.2f" % ISa
                    for f in backup_needed:
                        if os.path.isfile(os.path.join(root, f)):
                            os.remove(os.path.join(root, f))

                    revert_files(backup_needed,
                                 bk_prefix,
                                 wdir=root,
                                 quiet=True)

    run_xdsstat(wdir=root)
    print
    if params.make_report: html_report.make_individual_report(root, root)
    print >> decilog, "xds_sequence finished at %s\n" % time.strftime(
        "%Y-%m-%d %H:%M:%S")
    decilog.close()
Пример #15
0
def rescale_with_specified_symm(topdir, dirs, symms, out, sgnum=None, reference_symm=None):
    assert (sgnum, reference_symm).count(None) == 1

    if sgnum is not None:
        sgnum_laue = sgtbx.space_group_info(sgnum).group().build_derived_reflection_intensity_group(False).type().number()

        matches = filter(lambda x:x.reflection_intensity_symmetry(False).space_group_info().type().number()==sgnum_laue, symms)
        matched_cells = numpy.array(map(lambda x: x.unit_cell().parameters(), matches))
        median_cell = map(lambda x: numpy.median(matched_cells[:,x]), xrange(6))

        reference_symm = crystal.symmetry(median_cell, sgnum)
    else:
        sgnum = reference_symm.space_group_info().type().number()
        sgnum_laue = reference_symm.space_group().build_derived_reflection_intensity_group(False).type().number()
    
    print >>out
    print >>out,  "Re-scaling with specified symmetry:", reference_symm.space_group_info().symbol_and_number()
    print >>out,  " reference cell:", reference_symm.unit_cell()
    print >>out
    print >>out

    cells = {} # cell and file
    for sym, wd in zip(symms, dirs):
        print >>out,  os.path.relpath(wd, topdir),

        # Find appropriate data
        xac_file = util.return_first_found_file(("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale", 
                                                 "XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
                                                 "XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
                                                wd=wd)
        if xac_file is None:
            print >>out, "Can't find XDS_ASCII file in %s" % wd
            continue

        xac = XDS_ASCII(xac_file, read_data=False)
        print >>out, "%s %s (%s)" % (os.path.basename(xac_file), xac.symm.space_group_info(),
                                     ",".join(map(lambda x: "%.2f"%x, xac.symm.unit_cell().parameters())))

        if xac.symm.reflection_intensity_symmetry(False).space_group_info().type().number() == sgnum_laue:
            if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1, 10):
                print >>out,  "  Already scaled with specified symmetry"
                cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()), xac_file)
                continue

        xdsinp = os.path.join(wd, "XDS.INP")
        cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)

        if len(cosets.combined_cb_ops())==0:
            print >>out, "Can't find operator:"
            sym.show_summary(out, " ")
            reference_symm.show_summary(out, " ")
            continue

        newcell = reference_symm.space_group().average_unit_cell(xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
        newcell = " ".join(map(lambda x: "%.3f"%x, newcell.parameters()))
        print >>out,  "Scaling with transformed cell:", newcell

        #for f in xds_files.generated_by_CORRECT:
        #    util.rotate_file(os.path.join(wd, f))
        bk_prefix = make_backup(xds_files.generated_by_CORRECT, wdir=wd, quiet=True)

        modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                          ("SPACE_GROUP_NUMBER", "%d"%sgnum),
                                          ("UNIT_CELL_CONSTANTS", newcell),
                                          ("INCLUDE_RESOLUTION_RANGE", "50 0"),
                                          ("CORRECTIONS", ""),
                                          ("NBATCH", "1"),
                                          ("MINIMUM_I/SIGMA", None), # use default
                                          ("REFINE(CORRECT)", None), # use default
                                          ])
        run_xds(wd)
        for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
            if os.path.exists(os.path.join(wd, f)):
                shutil.copyfile(os.path.join(wd, f), os.path.join(wd, f+"_rescale"))

        revert_files(xds_files.generated_by_CORRECT, bk_prefix, wdir=wd, quiet=True)

        new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
        new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
        if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
            cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
            print "OK:", cells[wd][0]
        else:
            print >>out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
            continue

    return cells, reference_symm
Пример #16
0
def xds_sequence(img_in, topdir, data_root_dir, params):
    relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
    workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
    print workdir
    frame_num = None

    if not os.path.exists(img_in):
        if "<>" in img_in:
            img_in, num = img_in.split("<>")
            frame_num = int(num)
            if not os.path.exists(img_in):
                print "Error: Not found!!", img_in
                return
            workdir += "_%.6d" % frame_num
            assert img_in.endswith(".h5")
        else:
            for ext in (".bz2", ".gz", ".xz"):
                if os.path.exists(img_in+ext):
                    img_in += ext
                    break

    if params.tmpdir is not None:
        tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
    else:
        tmpdir = workdir
        
    if not os.path.exists(tmpdir): os.makedirs(tmpdir)

    xparm = os.path.join(tmpdir, "XPARM.XDS")
    xdsinp = os.path.join(tmpdir, "XDS.INP")
    integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
    decilog = open(os.path.join(tmpdir, "decision.log"), "w")

    try:
        print >>decilog, "Paramters:"
        libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")

        print >>decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")

        # Prepare XDS.INP
        if params.sfx.cheetah_mpccd:
            xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
        else:
            if frame_num is not None: # Eiger
                from yamtbx.dataproc import eiger
                img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
                eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
            else:
                ext = os.path.splitext(img_in)[1]
                img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
                os.symlink(img_in, img_in_work)
            xdsinp_str = xds_inp.generate_xds_inp(img_files=[img_in_work],
                                                  inp_dir=tmpdir,
                                                  reverse_phi=True, anomalous=params.anomalous,
                                                  spot_range=None, minimum=False,
                                                  crystal_symmetry=None,
                                                  integrate_nimages=None,
                                                  osc_range=params.osc_range,
                                                  orgx=params.orgx, orgy=params.orgy,
                                                  rotation_axis=params.rotation_axis,
                                                  distance=params.distance)

        ofs = open(xdsinp, "w")
        ofs.write(xdsinp_str)
        ofs.close()
        # Start processing
        modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF"),
                                          ("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
                                          ("MINPK", "%.2f"%params.minpk),
                                          ("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
                                          ("STRONG_PIXEL", "%.2f"%params.strong_pixel),
                                          ("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d"%params.minimum_number_of_pixels_in_a_spot),
                                          ("SEPMIN", "%.2f"%params.sepmin),
                                          ("CLUSTER_RADIUS", "%.2f"%params.cluster_radius),
                                          ("INDEX_ERROR", "%.4f"%params.index_error),
                                          ("INDEX_MAGNITUDE", "%d"%params.index_magnitude),
                                          ("INDEX_QUALITY", "%.2f"%params.index_quality),
                                          ("REFINE(IDXREF)", " ".join(params.refine_idxref)),
                                          ("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
                                          ("VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS", "%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels))
                                          ])

        if len(params.extra_inp_str) > 0:
            ofs = open(xdsinp, "a")
            ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
            ofs.close()

        run_xds(wdir=tmpdir, show_progress=False)

        if params.tryhard:
            try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)

            # If Cell hint exists, try to use it..
            if params.sgnum > 0:
                flag_try_cell_hint = False
                if not os.path.isfile(xparm):
                    flag_try_cell_hint = True
                else:
                    xsxds = XPARM(xparm).crystal_symmetry()
                    cosets = check_cell(params, xsxds)
                    if cosets.double_cosets is None: flag_try_cell_hint = True

                if flag_try_cell_hint:
                    print >>decilog, " Worth trying to use prior cell for indexing."
                    modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                                      ("UNIT_CELL_CONSTANTS",
                                                       " ".join(map(lambda x: "%.3f"%x, params.cell))),
                                                      ("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
                                                      ])
                    run_xds(wdir=tmpdir, show_progress=False)
                    modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
                                                      ])

        if not os.path.isfile(xparm):
            raise ProcFailed("Indexing failed")

        if params.checkcell.check and params.sgnum > 0:
            xsxds = XPARM(xparm).crystal_symmetry()
            cosets = check_cell(params, xsxds)
            if cosets.double_cosets is None:
                raise ProcFailed("Incompatible cell. Indexing failed.")

            if not cosets.combined_cb_ops()[0].is_identity_op():
                print >>decilog, "Re-idxref to match reference symmetry."
                xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
                modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                                  ("SPACE_GROUP_NUMBER", "%d"%params.sgnum),
                                                  ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))
                                                  ])
                run_xds(wdir=tmpdir, show_progress=False)

        modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min)),
                                          ])

        # To Integration
        modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),])
        run_xds(wdir=tmpdir, show_progress=False)

        if not os.path.isfile(integrate_hkl):
            raise ProcFailed("Integration failed.")

        # Determine unit cell in CORRECT
        if params.refine_correct:
            tmp = [("REFINE(CORRECT)", "ALL"),
                   ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, params.cell)))]
        else:
            # XXX What if CELL is refined in INTEGRATE?
            xsxds = XPARM(xparm).crystal_symmetry()
            cosets = check_cell(params, xsxds)
            if cosets.double_cosets is None:
                raise ProcFailed(" Incompatible cell. Failed before CORRECT.")

            xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0]) # Really OK??
            tmp = [("REFINE(CORRECT)", ""),
                   ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f"%x, xsxds_cb.unit_cell().parameters())))]

        # PEAK-corrected INTEGRATE.HKL
        ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
        ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
        ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
        inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
        inhkl.write_peak_corrected(ihk_part)
        os.rename(ihk, ihk_full)
        
        modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                          ("DATA_RANGE", "1 20000"),
                                          ("CORRECTIONS", ""),
                                          ("NBATCH", "1"),
                                          ("SPACE_GROUP_NUMBER", "%d"%params.sgnum)] + tmp)

        xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
        xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
        xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")

        # CORRECT for full
        os.symlink(ihk_full, ihk)
        run_xds(wdir=tmpdir, comm="xds", show_progress=False)
        if os.path.isfile(xac):
            os.rename(xac, xac_full)
            os.rename(os.path.join(tmpdir, "CORRECT.LP"),
                      os.path.join(tmpdir, "CORRECT_full.LP"))
        os.remove(ihk)

        # CORRECT for part
        os.symlink(ihk_part, ihk)
        run_xds(wdir=tmpdir, comm="xds", show_progress=False)
        if os.path.isfile(xac):
            os.rename(xac, xac_part)
            os.rename(os.path.join(tmpdir, "CORRECT.LP"),
                      os.path.join(tmpdir, "CORRECT_part.LP"))
        os.remove(ihk)

        if params.pickle_hkl:
            for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
                print >>decilog, "Pickling %s" % os.path.basename(f)
                x = xds_ascii.XDS_ASCII(f, log_out=decilog)
                if params.light_pickle: x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None # To make reading faster
                pickle.dump(x, open(f+".pkl", "w"), -1)
        if params.pickle_hkl:
            for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
                print >>decilog, "Pickling %s" % os.path.basename(f)
                inhkl = integrate_hkl_as_flex.reader(f, read_columns=["IOBS","SIGMA","XCAL","YCAL","RLP","PEAK","MAXC"])
                pickle.dump(inhkl, open(f+".pkl", "w"), -1)

    except ProcFailed, e:
        print >>decilog, "Processing failed: %s" % e.message
Пример #17
0
def try_indexing_hard(wdir, show_progress, decilog, 
                      known_sgnum=None, known_cell=None, tol_length=None, tol_angle=None):
    idxref_lp = os.path.join(wdir, "IDXREF.LP")
    xdsinp = os.path.join(wdir, "XDS.INP")

    lp_org = idxreflp.IdxrefLp(idxref_lp)

    if lp_org.is_cell_maybe_half():
        backup_needed = ("XDS.INP",) + xds_files.generated_by_IDXREF

        print >>decilog, " !! Cell may be halved. Trying doubled cell."
        bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)

        cell = lp_org.deduce_correct_cell_based_on_integerness()
        cell = " ".join(map(lambda x:"%.2f"%x, cell.parameters()))
        modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                          ("SPACE_GROUP_NUMBER", "1"),
                                          ("UNIT_CELL_CONSTANTS", cell)
                                          ])
        run_xds(wdir=wdir, show_progress=show_progress)

        if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
            revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)

            print >>decilog, "  .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
            bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)

            modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                              ("SEPMIN", "4"),
                                              ("CLUSTER_RADIUS", "2")
                                              ])
            run_xds(wdir=wdir, show_progress=show_progress)

            if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
                print >>decilog, "  .. not solved. Give up."
                revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
        else:
            print >>decilog, "  Now OK."
            remove_backups(backup_needed, bk_prefix, wdir=wdir)
            modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
                                              ])

    # If Cell hint exists, try to use it..
    if known_sgnum > 0:
        flag_try_cell_hint = False
        xparm = os.path.join(wdir, "XPARM.XDS")
        if not os.path.isfile(xparm):
            flag_try_cell_hint = True
        else:
            xsxds = XPARM(xparm).crystal_symmetry()

            xsref = crystal.symmetry(known_cell, known_sgnum)
            cosets = reindex.reindexing_operators(xsref, xsxds,
                                                  tol_length, tol_angle)

            if cosets.double_cosets is None: flag_try_cell_hint = True

        if flag_try_cell_hint:
            print >>decilog, " Worth trying to use prior cell for indexing."
            modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                              ("UNIT_CELL_CONSTANTS",
                                               " ".join(map(lambda x: "%.3f"%x, known_cell))),
                                              ("SPACE_GROUP_NUMBER", "%d"%known_sgnum),
                                              ])
            run_xds(wdir=wdir, show_progress=False)
            modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
                                              ])           
Пример #18
0
def xds_sequence(root, params):
    print
    print os.path.relpath(root, params.topdir)

    xparm = os.path.join(root, "XPARM.XDS")
    gxparm = os.path.join(root, "GXPARM.XDS")
    defpix_lp = os.path.join(root, "DEFPIX.LP")
    correct_lp = os.path.join(root, "CORRECT.LP")
    integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
    xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
    integrate_lp = os.path.join(root, "INTEGRATE.LP")
    xdsinp = os.path.join(root, "XDS.INP")

    assert os.path.isfile(xdsinp)

    decilog = multi_out()
    decilog.register("log", open(os.path.join(root, "decision.log"), "a"), atexit_send_to=None)

    print >>decilog, "xds_sequence started at %s in %s\n" % (time.strftime("%Y-%m-%d %H:%M:%S"), root)
    
    if params.show_progress:
        decilog.register("stdout", sys.stdout)

    if params.mode=="initial" and params.resume and os.path.isfile(correct_lp):
        print " Already processed."
        return

    if params.mode == "recycle" and not os.path.isfile(gxparm):
        print "GXPARM.XDS not found. Cannot do recycle."
        return

    if params.fast_delphi and (params.nproc is None or params.nproc > 1):
        delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
        print " Setting delphi to ", delphi
        modify_xdsinp(xdsinp, inp_params=[("DELPHI", str(delphi)),
                                          ])

    if params.nproc is not None and params.nproc > 1:
        modify_xdsinp(xdsinp, inp_params=[("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
                                          ])

    if params.mode == "initial":
        # To Indexing
        modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF")])
        run_xds(wdir=root, show_progress=params.show_progress)
        print # indexing stats like indexed percentage here.

        if params.tryhard:
            try_indexing_hard(root, params.show_progress, decilog,
                              known_sgnum=params.cell_prior.sgnum,
                              known_cell=params.cell_prior.cell,
                              tol_length=params.cell_prior.tol_length,
                              tol_angle=params.cell_prior.tol_angle)

        if not os.path.isfile(xparm):
            print >>decilog, " Indexing failed."
            return

        if params.cell_prior.check and params.cell_prior.sgnum > 0:
            xsxds = XPARM(xparm).crystal_symmetry()
            xsref = crystal.symmetry(params.cell_prior.cell, params.cell_prior.sgnum)
            cosets = reindex.reindexing_operators(xsref, xsxds,
                                                  params.cell_prior.tol_length, params.cell_prior.tol_angle)
            if cosets.double_cosets is None:
                print >>decilog, " Incompatible cell. Indexing failed."
                return

    elif params.mode == "recycle":
        print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
        for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS", "plot_integrate.log"):
            util.rotate_file(os.path.join(root, f), copy=True)
        shutil.copyfile(gxparm+".1", xparm)
    else:
        raise "Unknown mode (%s)" % params.mode

    # To Integration
    modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),
                                      ("INCLUDE_RESOLUTION_RANGE", "50 0")])
    run_xds(wdir=root, show_progress=params.show_progress)
    if os.path.isfile(integrate_lp):
        xds_plot_integrate.run(integrate_lp, os.path.join(root, "plot_integrate.log"))
    if not os.path.isfile(integrate_hkl):
        print >>decilog, " Integration failed."
        return


    # Make _noscale.HKL if needed
    if params.no_scaling:
        bk_prefix = make_backup(("XDS.INP",), wdir=root, quiet=True)
        xparm_obj = XPARM(xparm)
        modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                          ("CORRECTIONS", ""),
                                          ("NBATCH", "1"),
                                          ("MINIMUM_I/SIGMA", "50"),
                                          ("REFINE(CORRECT)", ""),
                                          ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x:"%.3f"%x, xparm_obj.unit_cell))),
                                          ("SPACE_GROUP_NUMBER", "%d"%xparm_obj.spacegroup),])
        print >>decilog, " running CORRECT without empirical scaling"
        run_xds(wdir=root, show_progress=params.show_progress)
        for f in xds_files.generated_by_CORRECT + ("XDS.INP",):
            ff = os.path.join(root, f)
            if not os.path.isfile(ff): continue
            if ff.endswith(".cbf"):
                os.remove(ff)
            else:
                os.rename(ff, ff+"_noscale")

        revert_files(("XDS.INP",), bk_prefix, wdir=root, quiet=True)

    # Run pointless
    symm_by_integrate = None
    if params.use_pointless:
        worker = Pointless()
        result = worker.run_for_symm(xdsin=integrate_hkl, 
                                     logout=os.path.join(root, "pointless_integrate.log"))
        if "symm" in result:
            symm = result["symm"]
            print >>decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info()
            sgnum = symm.space_group_info().type().number()
            cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
            modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "%d"%sgnum),
                                              ("UNIT_CELL_CONSTANTS", cell)])
            symm_by_integrate = symm
        else:
            print >>decilog, " pointless failed."

    # Do Scaling
    modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),])

    run_xds(wdir=root, show_progress=params.show_progress)

    if not os.path.isfile(gxparm):
        print >>decilog, " Scaling failed."
        return

    print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)

    ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
    if params.cut_resolution:
        if ret is not None and ret[0] is not None:
            d_min = ret[0]
            modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                              ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
            print >>decilog, " Re-scale at %.2f A" % d_min
            os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
            os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
            run_xds(wdir=root, show_progress=params.show_progress)
            print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
            print >>decilog, " (Original files are saved as *_fullres.*)"
        else:
            print >>decilog, "error: Can't decide resolution."

    last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)

    # Run pointless and (if result is different from INTEGRATE) re-scale.
    if params.use_pointless:
        worker = Pointless()
        result = worker.run_for_symm(xdsin=xac_hkl,
                                     logout=os.path.join(root, "pointless_correct.log"))
        if "symm" in result:
            symm = result["symm"]
            need_rescale = False

            if symm_by_integrate is not None:
                if not xtal.is_same_laue_symmetry(symm_by_integrate.space_group(), symm.space_group()):
                    print >>decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (symm.space_group_info(), symm_by_integrate.space_group_info())
                    need_rescale = True
            else:
                print >>decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info()
                need_rescale = True

            if need_rescale:
                # make backup, and do correct and compare ISa
                # if ISa got worse, revert the result.
                backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL","CORRECT_fullres.LP",
                                 "merging_stats.pkl","merging_stats.log")
                backup_needed += xds_files.generated_by_CORRECT
                bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)

                sgnum = symm.space_group_info().type().number()
                cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
                modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                                  ("SPACE_GROUP_NUMBER", "%d"%sgnum),
                                                  ("UNIT_CELL_CONSTANTS", cell),
                                                  ("INCLUDE_RESOLUTION_RANGE", "50 0")])

                run_xds(wdir=root, show_progress=params.show_progress)

                ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
                
                if params.cut_resolution:
                    if ret is not None and ret[0] is not None:
                        d_min = ret[0]
                        modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                                          ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
                        print >>decilog, " Re-scale at %.2f A" % d_min
                        os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
                        os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
                        run_xds(wdir=root, show_progress=params.show_progress)
                        print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
                        print >>decilog, " (Original files are saved as *_fullres.*)"
                    else:
                        print >>decilog, "error: Can't decide resolution."
                        for f in ("CORRECT_fullres.LP", "XDS_ASCII_fullres.HKL"):
                            if os.path.isfile(os.path.join(root, f)):
                                print >>decilog, "removing", f
                                os.remove(os.path.join(root, f))

                ISa = correctlp.get_ISa(correct_lp, check_valid=True)

                if ISa >= last_ISa or last_ISa!=last_ISa: # if improved or last_ISa is nan
                    print >>decilog, "ISa improved= %.2f" % ISa
                    remove_backups(backup_needed, bk_prefix, wdir=root)
                else:
                    print >>decilog, "ISa got worse= %.2f" % ISa
                    for f in backup_needed:
                        if os.path.isfile(os.path.join(root, f)): os.remove(os.path.join(root, f))

                    revert_files(backup_needed, bk_prefix, wdir=root, quiet=True)

    run_xdsstat(wdir=root)
    print
    if params.make_report: html_report.make_individual_report(root, root)
    print >>decilog, "xds_sequence finished at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
    decilog.close()
Пример #19
0
def rescale_with_specified_symm_worker(sym_wd_wdr,
                                       topdir,
                                       log_out,
                                       reference_symm,
                                       sgnum,
                                       sgnum_laue,
                                       prep_dials_files=False):
    # XXX Unsafe if multiple processes run this function for the same target directory at the same time

    sym, wd, wdr = sym_wd_wdr
    out = StringIO()
    print >> out, os.path.relpath(wd, topdir),

    # Find appropriate data # XXX not works for DIALS data!!
    xac_file = util.return_first_found_file(
        ("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
         "XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
         "XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
        wd=wd)
    if xac_file is None:
        print >> out, "Can't find XDS_ASCII file in %s" % wd
        log_out.write(out.getvalue())
        log_out.flush()
        return (wd, None)

    xac = XDS_ASCII(xac_file, read_data=False)
    print >> out, "%s %s (%s)" % (os.path.basename(xac_file),
                                  xac.symm.space_group_info(), ",".join(
                                      map(lambda x: "%.2f" % x,
                                          xac.symm.unit_cell().parameters())))

    if xac.symm.reflection_intensity_symmetry(
            False).space_group_info().type().number() == sgnum_laue:
        if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1,
                                              10):
            print >> out, "  Already scaled with specified symmetry"
            log_out.write(out.getvalue())
            log_out.flush()

            if wd != wdr: shutil.copy2(xac_file, wdr)

            if prep_dials_files: prepare_dials_files(wd, out, moveto=wdr)
            return (wdr, (numpy.array(xac.symm.unit_cell().parameters()),
                          os.path.join(wdr, os.path.basename(xac_file))))

    xdsinp = os.path.join(wd, "XDS.INP")
    cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)

    if len(cosets.combined_cb_ops()) == 0:
        print >> out, "Can't find operator:"
        sym.show_summary(out, " ")
        reference_symm.show_summary(out, " ")
        log_out.write(out.getvalue())
        log_out.flush()
        return (wdr, None)

    newcell = reference_symm.space_group().average_unit_cell(
        xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
    newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
    print >> out, "Scaling with transformed cell:", newcell

    #for f in xds_files.generated_by_CORRECT:
    #    util.rotate_file(os.path.join(wd, f))
    bk_prefix = make_backup(xds_files.generated_by_CORRECT,
                            wdir=wd,
                            quiet=True)

    modify_xdsinp(
        xdsinp,
        inp_params=[
            ("JOB", "CORRECT"),
            ("SPACE_GROUP_NUMBER", "%d" % sgnum),
            ("UNIT_CELL_CONSTANTS", newcell),
            ("INCLUDE_RESOLUTION_RANGE", "50 0"),
            ("CORRECTIONS", ""),
            ("NBATCH", "1"),
            ("MINIMUM_I/SIGMA", None),  # use default
            ("REFINE(CORRECT)", None),  # use default
        ])
    run_xds(wd)
    for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
        if os.path.exists(os.path.join(wd, f)):
            shutil.copyfile(os.path.join(wd, f),
                            os.path.join(wdr, f + "_rescale"))

    revert_files(xds_files.generated_by_CORRECT,
                 bk_prefix,
                 wdir=wd,
                 quiet=True)

    new_xac = os.path.join(wdr, "XDS_ASCII.HKL_rescale")

    if prep_dials_files:
        prepare_dials_files(wd,
                            out,
                            space_group=reference_symm.space_group(),
                            reindex_op=cosets.combined_cb_ops()[0],
                            moveto=wdr)

    ret = None
    if os.path.isfile(new_xac):
        ret = (XDS_ASCII(new_xac,
                         read_data=False).symm.unit_cell().parameters(),
               new_xac)
        print >> out, " OK:", ret[0]
    else:
        print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"

    return (wd, ret)
Пример #20
0
def xds_sequence(img_in, topdir, data_root_dir, params):
    relpath = os.path.relpath(os.path.dirname(img_in), data_root_dir)
    workdir = os.path.abspath(os.path.join(topdir, relpath, os.path.splitext(os.path.basename(img_in))[0]))
    print workdir
    frame_num = None

    if not os.path.exists(img_in):
        if "<>" in img_in:
            img_in, num = img_in.split("<>")
            frame_num = int(num)
            if not os.path.exists(img_in):
                print "Error: Not found!!", img_in
                return
            workdir += "_%.6d" % frame_num
            assert img_in.endswith(".h5")
        else:
            for ext in (".bz2", ".gz", ".xz"):
                if os.path.exists(img_in + ext):
                    img_in += ext
                    break

    if params.tmpdir is not None:
        tmpdir = tempfile.mkdtemp(prefix="xds", dir=params.tmpdir)
    else:
        tmpdir = workdir

    if not os.path.exists(tmpdir):
        os.makedirs(tmpdir)

    xparm = os.path.join(tmpdir, "XPARM.XDS")
    xdsinp = os.path.join(tmpdir, "XDS.INP")
    integrate_hkl = os.path.join(tmpdir, "INTEGRATE.HKL")
    decilog = open(os.path.join(tmpdir, "decision.log"), "w")

    try:
        print >> decilog, "Paramters:"
        libtbx.phil.parse(master_params_str).format(params).show(out=decilog, prefix=" ")

        print >> decilog, "\nStarting at %s" % time.strftime("%Y-%m-%d %H:%M:%S")

        # Prepare XDS.INP
        if params.sfx.cheetah_mpccd:
            xdsinp_str = sfx_xds_inp(img_in, xdsinp, params)
        else:
            if frame_num is not None:  # Eiger
                from yamtbx.dataproc import eiger

                img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000.cbf")
                eiger.extract_to_minicbf(img_in, frame_num, img_in_work)
            else:
                ext = os.path.splitext(img_in)[1]
                img_in_work = os.path.join(os.path.dirname(xdsinp), "data_10000" + ext)
                os.symlink(img_in, img_in_work)
            xdsinp_str = xds_inp.generate_xds_inp(
                img_files=[img_in_work],
                inp_dir=tmpdir,
                reverse_phi=True,
                anomalous=params.anomalous,
                spot_range=None,
                minimum=False,
                crystal_symmetry=None,
                integrate_nimages=None,
                osc_range=params.osc_range,
                orgx=params.orgx,
                orgy=params.orgy,
                rotation_axis=params.rotation_axis,
                distance=params.distance,
            )

        ofs = open(xdsinp, "w")
        ofs.write(xdsinp_str)
        ofs.close()
        # Start processing
        modify_xdsinp(
            xdsinp,
            inp_params=[
                ("JOB", "XYCORR INIT COLSPOT IDXREF"),
                ("MAXIMUM_NUMBER_OF_PROCESSORS", "1"),
                ("MINPK", "%.2f" % params.minpk),
                ("PROFILE_FITTING", bool_to_str(params.profile_fitting)),
                ("STRONG_PIXEL", "%.2f" % params.strong_pixel),
                ("MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT", "%d" % params.minimum_number_of_pixels_in_a_spot),
                ("SEPMIN", "%.2f" % params.sepmin),
                ("CLUSTER_RADIUS", "%.2f" % params.cluster_radius),
                ("INDEX_ERROR", "%.4f" % params.index_error),
                ("INDEX_MAGNITUDE", "%d" % params.index_magnitude),
                ("INDEX_QUALITY", "%.2f" % params.index_quality),
                ("REFINE(IDXREF)", " ".join(params.refine_idxref)),
                ("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.idxref_d_min)),
                (
                    "VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS",
                    "%.1f %.1f" % tuple(params.value_range_for_trusted_detector_pixels),
                ),
            ],
        )

        if len(params.extra_inp_str) > 0:
            ofs = open(xdsinp, "a")
            ofs.write("\n%s\n" % "\n".join(params.extra_inp_str))
            ofs.close()

        run_xds(wdir=tmpdir, show_progress=False)

        if params.tryhard:
            try_indexing_hard(tmpdir, show_progress=True, decilog=decilog)

            # If Cell hint exists, try to use it..
            if params.sgnum > 0:
                flag_try_cell_hint = False
                if not os.path.isfile(xparm):
                    flag_try_cell_hint = True
                else:
                    xsxds = XPARM(xparm).crystal_symmetry()
                    cosets = check_cell(params, xsxds)
                    if cosets.double_cosets is None:
                        flag_try_cell_hint = True

                if flag_try_cell_hint:
                    print >> decilog, " Worth trying to use prior cell for indexing."
                    modify_xdsinp(
                        xdsinp,
                        inp_params=[
                            ("JOB", "IDXREF"),
                            ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))),
                            ("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
                        ],
                    )
                    run_xds(wdir=tmpdir, show_progress=False)
                    modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0")])

        if not os.path.isfile(xparm):
            raise ProcFailed("Indexing failed")

        if params.checkcell.check and params.sgnum > 0:
            xsxds = XPARM(xparm).crystal_symmetry()
            cosets = check_cell(params, xsxds)
            if cosets.double_cosets is None:
                raise ProcFailed("Incompatible cell. Indexing failed.")

            if not cosets.combined_cb_ops()[0].is_identity_op():
                print >> decilog, "Re-idxref to match reference symmetry."
                xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0])  # Really OK??
                modify_xdsinp(
                    xdsinp,
                    inp_params=[
                        ("JOB", "IDXREF"),
                        ("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
                        ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))),
                    ],
                )
                run_xds(wdir=tmpdir, show_progress=False)

        modify_xdsinp(xdsinp, inp_params=[("INCLUDE_RESOLUTION_RANGE", "%.2f %.2f" % (params.d_max, params.d_min))])

        # To Integration
        modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE")])
        run_xds(wdir=tmpdir, show_progress=False)

        if not os.path.isfile(integrate_hkl):
            raise ProcFailed("Integration failed.")

        # Determine unit cell in CORRECT
        if params.refine_correct:
            tmp = [
                ("REFINE(CORRECT)", "ALL"),
                ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, params.cell))),
            ]
        else:
            # XXX What if CELL is refined in INTEGRATE?
            xsxds = XPARM(xparm).crystal_symmetry()
            cosets = check_cell(params, xsxds)
            if cosets.double_cosets is None:
                raise ProcFailed(" Incompatible cell. Failed before CORRECT.")

            xsxds_cb = xsxds.change_basis(cosets.combined_cb_ops()[0])  # Really OK??
            tmp = [
                ("REFINE(CORRECT)", ""),
                ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x: "%.3f" % x, xsxds_cb.unit_cell().parameters()))),
            ]

        # PEAK-corrected INTEGRATE.HKL
        ihk = os.path.join(tmpdir, "INTEGRATE.HKL")
        ihk_full = os.path.join(tmpdir, "INTEGRATE_full.HKL")
        ihk_part = os.path.join(tmpdir, "INTEGRATE_part.HKL")
        inhkl = integrate_hkl_as_flex.reader(ihk, [], False)
        inhkl.write_peak_corrected(ihk_part)
        os.rename(ihk, ihk_full)

        modify_xdsinp(
            xdsinp,
            inp_params=[
                ("JOB", "CORRECT"),
                ("DATA_RANGE", "1 20000"),
                ("CORRECTIONS", ""),
                ("NBATCH", "1"),
                ("SPACE_GROUP_NUMBER", "%d" % params.sgnum),
            ]
            + tmp,
        )

        xac = os.path.join(tmpdir, "XDS_ASCII.HKL")
        xac_full = os.path.join(tmpdir, "XDS_ASCII_full.HKL")
        xac_part = os.path.join(tmpdir, "XDS_ASCII_part.HKL")

        # CORRECT for full
        os.symlink(ihk_full, ihk)
        run_xds(wdir=tmpdir, comm="xds", show_progress=False)
        if os.path.isfile(xac):
            os.rename(xac, xac_full)
            os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_full.LP"))
        os.remove(ihk)

        # CORRECT for part
        os.symlink(ihk_part, ihk)
        run_xds(wdir=tmpdir, comm="xds", show_progress=False)
        if os.path.isfile(xac):
            os.rename(xac, xac_part)
            os.rename(os.path.join(tmpdir, "CORRECT.LP"), os.path.join(tmpdir, "CORRECT_part.LP"))
        os.remove(ihk)

        if params.pickle_hkl:
            for f in filter(lambda x: os.path.isfile(x), (xac_part, xac_full)):
                print >> decilog, "Pickling %s" % os.path.basename(f)
                x = xds_ascii.XDS_ASCII(f, log_out=decilog)
                if params.light_pickle:
                    x.xd, x.yd, x.zd, x.rlp, x.corr = None, None, None, None, None  # To make reading faster
                pickle.dump(x, open(f + ".pkl", "w"), -1)
        if params.pickle_hkl:
            for f in filter(lambda x: os.path.isfile(x), (ihk_part, ihk_full)):
                print >> decilog, "Pickling %s" % os.path.basename(f)
                inhkl = integrate_hkl_as_flex.reader(
                    f, read_columns=["IOBS", "SIGMA", "XCAL", "YCAL", "RLP", "PEAK", "MAXC"]
                )
                pickle.dump(inhkl, open(f + ".pkl", "w"), -1)

    except ProcFailed, e:
        print >> decilog, "Processing failed: %s" % e.message