Beispiel #1
0
def get_colspot_result(frame_ranges, wdir):
    assert all(map(lambda x: len(x) == 2 and x[0] <= x[1], frame_ranges))

    # Check all needed files exist
    if not check_needed_files(xds_files.needed_by_COLSPOT, wdir):
        return

    xdsinp = os.path.join(wdir, "XDS.INP")

    backup_needed = xds_files.generated_by_COLSPOT + ("XDS.INP",)

    # 1. Backup XDS.INP, etc. (Make copies; not renaming)
    bk_prefix = make_backup(backup_needed, wdir=wdir)

    try:
        # 2. Modify XDS.INP
        spot_ranges = map(lambda x: ("SPOT_RANGE", "%d %d" % tuple(x)), frame_ranges)
        data_range = "%d %d" % (min(map(lambda x: min(x), frame_ranges)), max(map(lambda x: max(x), frame_ranges)))
        modify_xdsinp(xdsinp, [("JOB", "COLSPOT"), ("DATA_RANGE", data_range)] + spot_ranges)

        # 3. Run xds
        p = subprocess.Popen("xds", cwd=wdir)
        p.wait()

        spotxds = SpotXds(os.path.join(wdir, "SPOT.XDS"))

    finally:
        # 6. Revert XDS.INP, etc.
        revert_files(backup_needed, bk_prefix, wdir=wdir)

    return spotxds
Beispiel #2
0
def run(params):
    xdsinp = "XDS.INP"
    kwds = dict(get_xdsinp_keyword(xdsinp))
    orgx_org, orgy_org = map(float, (kwds["ORGX"], kwds["ORGY"]))

    dx, dy = params.dx, params.dy
    if params.unit == "mm":
        assert "QX" in kwds
        assert "QY" in kwds
        dx /= float(kwds["QX"])
        dy /= float(kwds["QY"])

    backup_needed = files.generated_by_IDXREF + ("XDS.INP",)
    bk_prefix = make_backup(backup_needed)
    try:
        results = []
        for i in xrange(-params.nx, params.nx+1):
            for j in xrange(-params.ny, params.ny+1):
                work_name = "bs_x%+.2d_y%+.2d" % (i, j)
                orgx = orgx_org + i * dx
                orgy = orgy_org + j * dy
                print "Trying", orgx, orgy

                modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                                  ("ORGX", orgx),
                                                  ("ORGY", orgy),
                                                  ])
                call("xds")
                make_backup(backup_needed, work_name+"_")

                results.append([work_name, orgx, orgy])

        for ret in results:
                print ret,
                analyze_result(ret[0]+"_IDXREF.LP")
                

    finally:
        revert_files(backup_needed, bk_prefix)
Beispiel #3
0
def rescale_with_specified_symm_worker(sym_wd_wdr,
                                       topdir,
                                       log_out,
                                       reference_symm,
                                       sgnum,
                                       sgnum_laue,
                                       prep_dials_files=False):
    # XXX Unsafe if multiple processes run this function for the same target directory at the same time

    sym, wd, wdr = sym_wd_wdr
    out = StringIO()
    print >> out, os.path.relpath(wd, topdir),

    # Find appropriate data # XXX not works for DIALS data!!
    xac_file = util.return_first_found_file(
        ("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
         "XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
         "XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
        wd=wd)
    if xac_file is None:
        print >> out, "Can't find XDS_ASCII file in %s" % wd
        log_out.write(out.getvalue())
        log_out.flush()
        return (wd, None)

    xac = XDS_ASCII(xac_file, read_data=False)
    print >> out, "%s %s (%s)" % (os.path.basename(xac_file),
                                  xac.symm.space_group_info(), ",".join(
                                      map(lambda x: "%.2f" % x,
                                          xac.symm.unit_cell().parameters())))

    if xac.symm.reflection_intensity_symmetry(
            False).space_group_info().type().number() == sgnum_laue:
        if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1,
                                              10):
            print >> out, "  Already scaled with specified symmetry"
            log_out.write(out.getvalue())
            log_out.flush()

            if wd != wdr: shutil.copy2(xac_file, wdr)

            if prep_dials_files: prepare_dials_files(wd, out, moveto=wdr)
            return (wdr, (numpy.array(xac.symm.unit_cell().parameters()),
                          os.path.join(wdr, os.path.basename(xac_file))))

    xdsinp = os.path.join(wd, "XDS.INP")
    cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)

    if len(cosets.combined_cb_ops()) == 0:
        print >> out, "Can't find operator:"
        sym.show_summary(out, " ")
        reference_symm.show_summary(out, " ")
        log_out.write(out.getvalue())
        log_out.flush()
        return (wdr, None)

    newcell = reference_symm.space_group().average_unit_cell(
        xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
    newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
    print >> out, "Scaling with transformed cell:", newcell

    #for f in xds_files.generated_by_CORRECT:
    #    util.rotate_file(os.path.join(wd, f))
    bk_prefix = make_backup(xds_files.generated_by_CORRECT,
                            wdir=wd,
                            quiet=True)

    modify_xdsinp(
        xdsinp,
        inp_params=[
            ("JOB", "CORRECT"),
            ("SPACE_GROUP_NUMBER", "%d" % sgnum),
            ("UNIT_CELL_CONSTANTS", newcell),
            ("INCLUDE_RESOLUTION_RANGE", "50 0"),
            ("CORRECTIONS", ""),
            ("NBATCH", "1"),
            ("MINIMUM_I/SIGMA", None),  # use default
            ("REFINE(CORRECT)", None),  # use default
        ])
    run_xds(wd)
    for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
        if os.path.exists(os.path.join(wd, f)):
            shutil.copyfile(os.path.join(wd, f),
                            os.path.join(wdr, f + "_rescale"))

    revert_files(xds_files.generated_by_CORRECT,
                 bk_prefix,
                 wdir=wd,
                 quiet=True)

    new_xac = os.path.join(wdr, "XDS_ASCII.HKL_rescale")

    if prep_dials_files:
        prepare_dials_files(wd,
                            out,
                            space_group=reference_symm.space_group(),
                            reindex_op=cosets.combined_cb_ops()[0],
                            moveto=wdr)

    ret = None
    if os.path.isfile(new_xac):
        ret = (XDS_ASCII(new_xac,
                         read_data=False).symm.unit_cell().parameters(),
               new_xac)
        print >> out, " OK:", ret[0]
    else:
        print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"

    return (wd, ret)
def xds_sequence(root, params):
    print
    print os.path.relpath(root, params.topdir)

    xparm = os.path.join(root, "XPARM.XDS")
    gxparm = os.path.join(root, "GXPARM.XDS")
    defpix_lp = os.path.join(root, "DEFPIX.LP")
    correct_lp = os.path.join(root, "CORRECT.LP")
    integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
    xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
    integrate_lp = os.path.join(root, "INTEGRATE.LP")
    spot_xds = os.path.join(root, "SPOT.XDS")
    xdsinp = os.path.join(root, "XDS.INP")

    assert os.path.isfile(xdsinp)

    xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))

    decilog = multi_out()
    decilog.register("log",
                     open(os.path.join(root, "decision.log"), "a"),
                     atexit_send_to=None)

    print >> decilog, "xds_sequence started at %s in %s\n" % (
        time.strftime("%Y-%m-%d %H:%M:%S"), root)

    if params.show_progress:
        decilog.register("stdout", sys.stdout)

    if params.mode == "initial" and params.resume and os.path.isfile(
            correct_lp):
        print " Already processed."
        return

    if params.mode == "recycle" and not os.path.isfile(gxparm):
        print "GXPARM.XDS not found. Cannot do recycle."
        return

    if params.fast_delphi and (params.nproc is None or params.nproc > 1):
        delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
        print " Setting delphi to ", delphi
        modify_xdsinp(xdsinp, inp_params=[
            ("DELPHI", str(delphi)),
        ])

    if params.nproc is not None and params.nproc > 1:
        modify_xdsinp(xdsinp,
                      inp_params=[
                          ("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
                      ])

    if params.mode == "initial":
        # Peak search
        modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT")])
        run_xds(wdir=root, show_progress=params.show_progress)
        if params.auto_frame_exclude_spot_based:
            sx = idxreflp.SpotXds(spot_xds)
            sx.set_xdsinp(xdsinp)
            spots = filter(lambda x: 5 < x[-1] < 30,
                           sx.collected_spots())  # low-res (5 A)
            frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1, spots))
            data_range = map(int, xdsinp_dict["DATA_RANGE"].split())
            # XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
            h = numpy.histogram(frame_numbers,
                                bins=numpy.arange(data_range[0],
                                                  data_range[1] + 2,
                                                  step=1))
            q14 = numpy.percentile(h[0], [25, 75])
            iqr = q14[1] - q14[0]
            cutoff = max(h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5  # magic number
            print "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (iqr, q14,
                                                                 cutoff)
            cut_frames = h[1][h[0] < cutoff]
            keep_frames = h[1][h[0] >= cutoff]
            print "DEBUG:: keep_frames=", keep_frames
            print "DEBUG::  cut_frames=", cut_frames

            if len(cut_frames) > 0:
                cut_ranges = [
                    [cut_frames[0], cut_frames[0]],
                ]
                for fn in cut_frames:
                    if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
                    else: cut_ranges.append([fn, fn])

                # Edit XDS.INP
                cut_inp_str = "".join(
                    map(lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(x),
                        cut_ranges))
                open(xdsinp, "a").write("\n" + cut_inp_str)

                # Edit SPOT.XDS
                shutil.copyfile(spot_xds, spot_xds + ".org")
                sx.write(open(spot_xds, "w"), frame_selection=set(keep_frames))

        # Indexing
        modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])
        run_xds(wdir=root, show_progress=params.show_progress)
        print  # indexing stats like indexed percentage here.

        if params.tryhard:
            try_indexing_hard(root,
                              params.show_progress,
                              decilog,
                              known_sgnum=params.cell_prior.sgnum,
                              known_cell=params.cell_prior.cell,
                              tol_length=params.cell_prior.tol_length,
                              tol_angle=params.cell_prior.tol_angle)

        if not os.path.isfile(xparm):
            print >> decilog, " Indexing failed."
            return

        if params.cell_prior.check and params.cell_prior.sgnum > 0:
            xsxds = XPARM(xparm).crystal_symmetry()
            xsref = crystal.symmetry(params.cell_prior.cell,
                                     params.cell_prior.sgnum)
            cosets = reindex.reindexing_operators(xsref, xsxds,
                                                  params.cell_prior.tol_length,
                                                  params.cell_prior.tol_angle)
            if cosets.double_cosets is None:
                print >> decilog, " Incompatible cell. Indexing failed."
                return

    elif params.mode == "recycle":
        print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
            correct_lp, check_valid=True)
        for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
                                                     "plot_integrate.log"):
            util.rotate_file(os.path.join(root, f), copy=True)
        shutil.copyfile(gxparm + ".1", xparm)
    else:
        raise "Unknown mode (%s)" % params.mode

    # To Integration
    modify_xdsinp(xdsinp,
                  inp_params=[("JOB", "DEFPIX INTEGRATE"),
                              ("INCLUDE_RESOLUTION_RANGE", "50 0")])
    run_xds(wdir=root, show_progress=params.show_progress)
    if os.path.isfile(integrate_lp):
        xds_plot_integrate.run(integrate_lp,
                               os.path.join(root, "plot_integrate.log"))
    if not os.path.isfile(integrate_hkl):
        print >> decilog, " Integration failed."
        return

    # Make _noscale.HKL if needed
    if params.no_scaling:
        bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
        xparm_obj = XPARM(xparm)
        modify_xdsinp(xdsinp,
                      inp_params=[
                          ("JOB", "CORRECT"),
                          ("CORRECTIONS", ""),
                          ("NBATCH", "1"),
                          ("MINIMUM_I/SIGMA", "50"),
                          ("REFINE(CORRECT)", ""),
                          ("UNIT_CELL_CONSTANTS", " ".join(
                              map(lambda x: "%.3f" % x, xparm_obj.unit_cell))),
                          ("SPACE_GROUP_NUMBER", "%d" % xparm_obj.spacegroup),
                      ])
        print >> decilog, " running CORRECT without empirical scaling"
        run_xds(wdir=root, show_progress=params.show_progress)
        for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
            ff = os.path.join(root, f)
            if not os.path.isfile(ff): continue
            if ff.endswith(".cbf"):
                os.remove(ff)
            else:
                os.rename(ff, ff + "_noscale")

        revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)

    # Run pointless
    symm_by_integrate = None
    if params.use_pointless:
        worker = Pointless()
        result = worker.run_for_symm(xdsin=integrate_hkl,
                                     logout=os.path.join(
                                         root, "pointless_integrate.log"))
        if "symm" in result:
            symm = result["symm"]
            print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
            )
            sgnum = symm.space_group_info().type().number()
            cell = " ".join(
                map(lambda x: "%.2f" % x,
                    symm.unit_cell().parameters()))
            modify_xdsinp(xdsinp,
                          inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
                                      ("UNIT_CELL_CONSTANTS", cell)])
            symm_by_integrate = symm
        else:
            print >> decilog, " pointless failed."

    # Do Scaling
    modify_xdsinp(xdsinp, inp_params=[
        ("JOB", "CORRECT"),
    ])

    run_xds(wdir=root, show_progress=params.show_progress)

    if not os.path.isfile(gxparm):
        print >> decilog, " Scaling failed."
        return

    print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp,
                                                           check_valid=True)

    ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
    if params.cut_resolution:
        if ret is not None and ret[0] is not None:
            d_min = ret[0]
            modify_xdsinp(xdsinp,
                          inp_params=[("JOB", "CORRECT"),
                                      ("INCLUDE_RESOLUTION_RANGE",
                                       "50 %.2f" % d_min)])
            print >> decilog, " Re-scale at %.2f A" % d_min
            os.rename(os.path.join(root, "CORRECT.LP"),
                      os.path.join(root, "CORRECT_fullres.LP"))
            os.rename(os.path.join(root, "XDS_ASCII.HKL"),
                      os.path.join(root, "XDS_ASCII_fullres.HKL"))
            run_xds(wdir=root, show_progress=params.show_progress)
            print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
                correct_lp, check_valid=True)
            print >> decilog, " (Original files are saved as *_fullres.*)"
        else:
            print >> decilog, "error: Can't decide resolution."

    last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)

    # Run pointless and (if result is different from INTEGRATE) re-scale.
    if params.use_pointless:
        worker = Pointless()
        result = worker.run_for_symm(xdsin=xac_hkl,
                                     logout=os.path.join(
                                         root, "pointless_correct.log"))
        if "symm" in result:
            symm = result["symm"]
            need_rescale = False

            if symm_by_integrate is not None:
                if not xtal.is_same_laue_symmetry(
                        symm_by_integrate.space_group(), symm.space_group()):
                    print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
                        symm.space_group_info(),
                        symm_by_integrate.space_group_info())
                    need_rescale = True
            else:
                print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
                )
                need_rescale = True

            if need_rescale:
                # make backup, and do correct and compare ISa
                # if ISa got worse, revert the result.
                backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL",
                                 "CORRECT_fullres.LP", "merging_stats.pkl",
                                 "merging_stats.log")
                backup_needed += xds_files.generated_by_CORRECT
                bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)

                sgnum = symm.space_group_info().type().number()
                cell = " ".join(
                    map(lambda x: "%.2f" % x,
                        symm.unit_cell().parameters()))
                modify_xdsinp(xdsinp,
                              inp_params=[("JOB", "CORRECT"),
                                          ("SPACE_GROUP_NUMBER", "%d" % sgnum),
                                          ("UNIT_CELL_CONSTANTS", cell),
                                          ("INCLUDE_RESOLUTION_RANGE", "50 0")
                                          ])

                run_xds(wdir=root, show_progress=params.show_progress)

                ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))

                if params.cut_resolution:
                    if ret is not None and ret[0] is not None:
                        d_min = ret[0]
                        modify_xdsinp(xdsinp,
                                      inp_params=[("JOB", "CORRECT"),
                                                  ("INCLUDE_RESOLUTION_RANGE",
                                                   "50 %.2f" % d_min)])
                        print >> decilog, " Re-scale at %.2f A" % d_min
                        os.rename(os.path.join(root, "CORRECT.LP"),
                                  os.path.join(root, "CORRECT_fullres.LP"))
                        os.rename(os.path.join(root, "XDS_ASCII.HKL"),
                                  os.path.join(root, "XDS_ASCII_fullres.HKL"))
                        run_xds(wdir=root, show_progress=params.show_progress)
                        print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
                            correct_lp, check_valid=True)
                        print >> decilog, " (Original files are saved as *_fullres.*)"
                    else:
                        print >> decilog, "error: Can't decide resolution."
                        for f in ("CORRECT_fullres.LP",
                                  "XDS_ASCII_fullres.HKL"):
                            if os.path.isfile(os.path.join(root, f)):
                                print >> decilog, "removing", f
                                os.remove(os.path.join(root, f))

                ISa = correctlp.get_ISa(correct_lp, check_valid=True)

                if ISa >= last_ISa or last_ISa != last_ISa:  # if improved or last_ISa is nan
                    print >> decilog, "ISa improved= %.2f" % ISa
                    remove_backups(backup_needed, bk_prefix, wdir=root)
                else:
                    print >> decilog, "ISa got worse= %.2f" % ISa
                    for f in backup_needed:
                        if os.path.isfile(os.path.join(root, f)):
                            os.remove(os.path.join(root, f))

                    revert_files(backup_needed,
                                 bk_prefix,
                                 wdir=root,
                                 quiet=True)

    run_xdsstat(wdir=root)
    print
    if params.make_report: html_report.make_individual_report(root, root)
    print >> decilog, "xds_sequence finished at %s\n" % time.strftime(
        "%Y-%m-%d %H:%M:%S")
    decilog.close()
def try_indexing_hard(wdir,
                      show_progress,
                      decilog,
                      known_sgnum=None,
                      known_cell=None,
                      tol_length=None,
                      tol_angle=None):
    idxref_lp = os.path.join(wdir, "IDXREF.LP")
    xdsinp = os.path.join(wdir, "XDS.INP")

    lp_org = idxreflp.IdxrefLp(idxref_lp)

    if lp_org.is_cell_maybe_half():
        backup_needed = ("XDS.INP", ) + xds_files.generated_by_IDXREF

        print >> decilog, " !! Cell may be halved. Trying doubled cell."
        bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)

        cell = lp_org.deduce_correct_cell_based_on_integerness()
        cell = " ".join(map(lambda x: "%.2f" % x, cell.parameters()))
        modify_xdsinp(xdsinp,
                      inp_params=[("JOB", "IDXREF"),
                                  ("SPACE_GROUP_NUMBER", "1"),
                                  ("UNIT_CELL_CONSTANTS", cell)])
        run_xds(wdir=wdir, show_progress=show_progress)

        if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
            revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)

            print >> decilog, "  .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
            bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)

            modify_xdsinp(xdsinp,
                          inp_params=[("JOB", "IDXREF"), ("SEPMIN", "4"),
                                      ("CLUSTER_RADIUS", "2")])
            run_xds(wdir=wdir, show_progress=show_progress)

            if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
                print >> decilog, "  .. not solved. Give up."
                revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
        else:
            print >> decilog, "  Now OK."
            remove_backups(backup_needed, bk_prefix, wdir=wdir)
            modify_xdsinp(xdsinp, inp_params=[
                ("SPACE_GROUP_NUMBER", "0"),
            ])

    # If Cell hint exists, try to use it..
    if known_sgnum > 0:
        flag_try_cell_hint = False
        xparm = os.path.join(wdir, "XPARM.XDS")
        if not os.path.isfile(xparm):
            flag_try_cell_hint = True
        else:
            xsxds = XPARM(xparm).crystal_symmetry()

            xsref = crystal.symmetry(known_cell, known_sgnum)
            cosets = reindex.reindexing_operators(xsref, xsxds, tol_length,
                                                  tol_angle)

            if cosets.double_cosets is None: flag_try_cell_hint = True

        if flag_try_cell_hint:
            print >> decilog, " Worth trying to use prior cell for indexing."
            modify_xdsinp(xdsinp,
                          inp_params=[
                              ("JOB", "IDXREF"),
                              ("UNIT_CELL_CONSTANTS",
                               " ".join(map(lambda x: "%.3f" % x,
                                            known_cell))),
                              ("SPACE_GROUP_NUMBER", "%d" % known_sgnum),
                          ])
            run_xds(wdir=wdir, show_progress=False)
            modify_xdsinp(xdsinp, inp_params=[
                ("SPACE_GROUP_NUMBER", "0"),
            ])
Beispiel #6
0
def run(params, out):
    print >> out, "Frames:", params.frames
    backup_needed = xds_files.generated_by_DEFPIX + (
        "XDS.INP",
        "BKGINIT.cbf",
    )
    bk_prefix = make_backup(backup_needed, wdir=params.xdsdir)

    ret = {}  # {frame: [matches, spots, predicted]}

    try:
        # run DEFPIX to limit resolution.
        modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"),
                      [("JOB", "DEFPIX"),
                       ("INCLUDE_RESOLUTION_RANGE", "50 %.2f" % params.d_min)])

        p = subprocess.Popen("xds", cwd=params.xdsdir)
        p.wait()

        # copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT)
        shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"),
                        os.path.join(params.xdsdir, "BKGINIT.cbf"))

        for frame in params.frames:
            print >> out, "Frame %d" % frame
            print >> out, "====================\n"
            # search spots
            if params.spotfinder == "xds":
                spotxds = get_colspot_result(frame_ranges=[
                    [frame, frame],
                ],
                                             wdir=params.xdsdir)
                spots = map(lambda x: x[:2],
                            spotxds.collected_spots(with_resolution=False))
            else:
                raise "Sorry!"

            # run INTEGRATE to collect predicted coords
            integrate_results = xds_predict_mitai.run(
                param_source=os.path.join(params.xdsdir, "XPARM.XDS"),
                frame_num=frame,
                wdir=params.xdsdir,
                need_adx=False,
                sigmar=params.sigmar,
                sigmab=params.sigmab)

            # read predicted coords
            tmp = filter(lambda x: x.endswith(".HKL"), integrate_results)
            if len(tmp) == 0:
                print >> out, "Integration failed!"
                ret[frame] = (0, len(spots), 0)
                continue

            integrate_hkl = tmp[0]
            cols = integrate_hkl_as_flex.reader(integrate_hkl, [],
                                                False).get_column_names()
            i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL")
            predicted = []

            for l in open(integrate_hkl):
                if l.startswith("!"): continue
                sp = l.split()
                predicted.append(map(float, (sp[i_xcal], sp[i_ycal])))

            # compare them
            nmatch = calc_matches(
                spots, predicted, params.distance_limit_in_px,
                open(
                    os.path.join(params.xdsdir,
                                 "matched_predicted_%.4d.adx" % frame), "w"))
            #nmatch = calc_matches(predicted, spots, params.distance_limit_in_px,
            #                      open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w"))

            ret[frame] = (nmatch, len(spots), len(predicted))

    finally:
        revert_files(backup_needed, bk_prefix, wdir=params.xdsdir)

    print >> out
    for frame in sorted(ret):
        nmatch, nspots, npredicted = ret[frame]
        print >> out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (
            frame, nmatch, npredicted, 100. * float(nmatch) /
            npredicted if npredicted > 0 else float("nan"))
        print >> out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (
            frame, nmatch, nspots,
            100. * float(nmatch) / nspots if nspots > 0 else float("nan"))
        print >> out

    return ret
def xds_sequence(root, params):
    print
    print os.path.relpath(root, params.topdir)

    init_lp = os.path.join(root, "INIT.LP")
    xparm = os.path.join(root, "XPARM.XDS")
    gxparm = os.path.join(root, "GXPARM.XDS")
    defpix_lp = os.path.join(root, "DEFPIX.LP")
    correct_lp = os.path.join(root, "CORRECT.LP")
    integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
    xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
    integrate_lp = os.path.join(root, "INTEGRATE.LP")
    spot_xds = os.path.join(root, "SPOT.XDS")
    xdsinp = os.path.join(root, "XDS.INP")

    assert os.path.isfile(xdsinp)
    if params.cell_prior.force: assert params.cell_prior.check

    xdsinp_dict = dict(get_xdsinp_keyword(xdsinp))

    if params.cell_prior.sgnum > 0:
        xs_prior = crystal.symmetry(params.cell_prior.cell,
                                    params.cell_prior.sgnum)
    else:
        xs_prior = None

    decilog = multi_out()
    decilog.register("log",
                     open(os.path.join(root, "decision.log"), "a"),
                     atexit_send_to=None)
    try:
        print >> decilog, "xds_sequence started at %s in %s\n" % (
            time.strftime("%Y-%m-%d %H:%M:%S"), root)

        if not kamo_test_installation.tst_xds():
            print >> decilog, "XDS is not installed or expired!!"
            return

        if params.show_progress:
            decilog.register("stdout", sys.stdout)

        if params.mode == "initial" and params.resume and os.path.isfile(
                correct_lp):
            print >> decilog, " Already processed."
            return

        if params.mode == "recycle" and not os.path.isfile(gxparm):
            print >> decilog, "GXPARM.XDS not found. Cannot do recycle."
            return

        if params.fast_delphi and (params.nproc is None or params.nproc > 1):
            delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
            print >> decilog, " Setting delphi to ", delphi
            modify_xdsinp(xdsinp, inp_params=[
                ("DELPHI", str(delphi)),
            ])

        if params.nproc is not None and params.nproc > 1:
            modify_xdsinp(xdsinp,
                          inp_params=[
                              ("MAXIMUM_NUMBER_OF_PROCESSORS",
                               str(params.nproc)),
                          ])

        if params.mode == "initial":
            modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT")])
            run_xds(wdir=root, show_progress=params.show_progress)
            initlp = InitLp(init_lp)
            first_bad = initlp.check_bad_first_frames()
            if first_bad:
                print >> decilog, " first frames look bad (too weak) exposure:", first_bad
                new_data_range = map(
                    int,
                    dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
                new_data_range[0] = first_bad[-1] + 1
                print >> decilog, " changing DATA_RANGE= to", new_data_range
                modify_xdsinp(xdsinp,
                              inp_params=[("JOB", "INIT"),
                                          ("DATA_RANGE",
                                           "%d %d" % tuple(new_data_range))])
                for f in xds_files.generated_by_INIT:
                    util.rotate_file(os.path.join(root, f), copy=False)
                run_xds(wdir=root, show_progress=params.show_progress)

            # Peak search
            modify_xdsinp(xdsinp, inp_params=[("JOB", "COLSPOT")])
            run_xds(wdir=root, show_progress=params.show_progress)
            if params.auto_frame_exclude_spot_based:
                sx = idxreflp.SpotXds(spot_xds)
                sx.set_xdsinp(xdsinp)
                spots = filter(lambda x: 5 < x[-1] < 30,
                               sx.collected_spots())  # low-res (5 A)
                frame_numbers = numpy.array(map(lambda x: int(x[2]) + 1,
                                                spots))
                data_range = map(
                    int,
                    dict(get_xdsinp_keyword(xdsinp))["DATA_RANGE"].split())
                # XXX this assumes SPOT_RANGE equals to DATA_RANGE. Is this guaranteed?
                h = numpy.histogram(frame_numbers,
                                    bins=numpy.arange(data_range[0],
                                                      data_range[1] + 2,
                                                      step=1))
                q14 = numpy.percentile(h[0], [25, 75])
                iqr = q14[1] - q14[0]
                cutoff = max(
                    h[0][h[0] <= iqr * 1.5 + q14[1]]) / 5  # magic number
                print >> decilog, "DEBUG:: IQR= %.2f, Q1/4= %s, cutoff= %.2f" % (
                    iqr, q14, cutoff)
                cut_frames = h[1][h[0] < cutoff]
                keep_frames = h[1][h[0] >= cutoff]
                print >> decilog, "DEBUG:: keep_frames=", keep_frames
                print >> decilog, "DEBUG::  cut_frames=", cut_frames

                if len(cut_frames) > 0:
                    cut_ranges = [
                        [cut_frames[0], cut_frames[0]],
                    ]
                    for fn in cut_frames:
                        if fn - cut_ranges[-1][1] <= 1: cut_ranges[-1][1] = fn
                        else: cut_ranges.append([fn, fn])

                    # Edit XDS.INP
                    cut_inp_str = "".join(
                        map(
                            lambda x: "EXCLUDE_DATA_RANGE= %6d %6d\n" % tuple(
                                x), cut_ranges))
                    open(xdsinp, "a").write("\n" + cut_inp_str)

                    # Edit SPOT.XDS
                    shutil.copyfile(spot_xds, spot_xds + ".org")
                    sx.write(open(spot_xds, "w"),
                             frame_selection=set(keep_frames))

            # Indexing
            if params.cell_prior.method == "use_first":
                modify_xdsinp(xdsinp,
                              inp_params=[
                                  ("JOB", "IDXREF"),
                                  ("UNIT_CELL_CONSTANTS", " ".join(
                                      map(lambda x: "%.3f" % x,
                                          params.cell_prior.cell))),
                                  ("SPACE_GROUP_NUMBER",
                                   "%d" % params.cell_prior.sgnum),
                              ])
            else:
                modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF")])

            run_xds(wdir=root, show_progress=params.show_progress)
            print >> decilog, ""  # TODO indexing stats like indexed percentage here.

            if params.tryhard:
                try_indexing_hard(root,
                                  params.show_progress,
                                  decilog,
                                  known_sgnum=params.cell_prior.sgnum,
                                  known_cell=params.cell_prior.cell,
                                  tol_length=params.cell_prior.tol_length,
                                  tol_angle=params.cell_prior.tol_angle)

            if not os.path.isfile(xparm):
                print >> decilog, " Indexing failed."
                return

            if params.cell_prior.sgnum > 0:
                # Check anyway
                xsxds = XPARM(xparm).crystal_symmetry()
                cosets = reindex.reindexing_operators(
                    xs_prior, xsxds, params.cell_prior.tol_length,
                    params.cell_prior.tol_angle)
                if cosets.double_cosets is None:
                    if params.cell_prior.check:
                        print >> decilog, " Incompatible cell. Indexing failed."
                        return
                    else:
                        print >> decilog, " Warning: Incompatible cell."

                elif params.cell_prior.method == "symm_constraint_only":
                    cell = xsxds.unit_cell().change_basis(
                        cosets.combined_cb_ops()[0])
                    print >> decilog, " Trying symmetry-constrained cell parameter:", cell
                    modify_xdsinp(xdsinp,
                                  inp_params=[
                                      ("JOB", "IDXREF"),
                                      ("UNIT_CELL_CONSTANTS", " ".join(
                                          map(lambda x: "%.3f" % x,
                                              cell.parameters()))),
                                      ("SPACE_GROUP_NUMBER",
                                       "%d" % params.cell_prior.sgnum),
                                  ])
                    for f in xds_files.generated_by_IDXREF:
                        util.rotate_file(os.path.join(root, f),
                                         copy=(f == "SPOT.XDS"))

                    run_xds(wdir=root, show_progress=params.show_progress)

                    if not os.path.isfile(xparm):
                        print >> decilog, " Indexing failed."
                        return

                    # Check again
                    xsxds = XPARM(xparm).crystal_symmetry()
                    if not xsxds.unit_cell().is_similar_to(
                            xs_prior.unit_cell(), params.cell_prior.tol_length,
                            params.cell_prior.tol_angle):
                        print >> decilog, "  Resulted in different cell. Indexing failed."
                        return

        elif params.mode == "recycle":
            print >> decilog, " Start recycle. original ISa= %.2f" % correctlp.get_ISa(
                correct_lp, check_valid=True)
            for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS",
                                                         "plot_integrate.log"):
                util.rotate_file(os.path.join(root, f), copy=True)
            shutil.copyfile(gxparm + ".1", xparm)
        else:
            raise "Unknown mode (%s)" % params.mode

        # To Integration
        modify_xdsinp(xdsinp,
                      inp_params=[("JOB", "DEFPIX INTEGRATE"),
                                  ("INCLUDE_RESOLUTION_RANGE", "50 0")])
        run_xds(wdir=root, show_progress=params.show_progress)
        if os.path.isfile(integrate_lp):
            xds_plot_integrate.run(integrate_lp,
                                   os.path.join(root, "plot_integrate.log"))
        if not os.path.isfile(integrate_hkl):
            print >> decilog, " Integration failed."
            return

        # Make _noscale.HKL if needed
        if params.no_scaling:
            bk_prefix = make_backup(("XDS.INP", ), wdir=root, quiet=True)
            xparm_obj = XPARM(xparm)
            modify_xdsinp(xdsinp,
                          inp_params=[
                              ("JOB", "CORRECT"),
                              ("CORRECTIONS", ""),
                              ("NBATCH", "1"),
                              ("MINIMUM_I/SIGMA", "50"),
                              ("REFINE(CORRECT)", ""),
                              ("UNIT_CELL_CONSTANTS", " ".join(
                                  map(lambda x: "%.3f" % x,
                                      xparm_obj.unit_cell))),
                              ("SPACE_GROUP_NUMBER",
                               "%d" % xparm_obj.spacegroup),
                          ])
            print >> decilog, " running CORRECT without empirical scaling"
            run_xds(wdir=root, show_progress=params.show_progress)
            for f in xds_files.generated_by_CORRECT + ("XDS.INP", ):
                ff = os.path.join(root, f)
                if not os.path.isfile(ff): continue
                if ff.endswith(".cbf"):
                    os.remove(ff)
                else:
                    os.rename(ff, ff + "_noscale")

            revert_files(("XDS.INP", ), bk_prefix, wdir=root, quiet=True)

        # Run pointless
        pointless_integrate = {}
        if params.use_pointless:
            worker = Pointless()
            pointless_integrate = worker.run_for_symm(
                xdsin=integrate_hkl,
                logout=os.path.join(root, "pointless_integrate.log"))
            if "symm" in pointless_integrate:
                symm = pointless_integrate["symm"]
                print >> decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info(
                )
                if xs_prior:
                    if xtal.is_same_space_group_ignoring_enantiomorph(
                            symm.space_group(), xs_prior.space_group()):
                        print >> decilog, " which is consistent with given symmetry."
                    elif xtal.is_same_laue_symmetry(symm.space_group(),
                                                    xs_prior.space_group()):
                        print >> decilog, " which has consistent Laue symmetry with given symmetry."
                    else:
                        print >> decilog, " which is inconsistent with given symmetry."

                sgnum = symm.space_group_info().type().number()
                cell = " ".join(
                    map(lambda x: "%.2f" % x,
                        symm.unit_cell().parameters()))
                modify_xdsinp(xdsinp,
                              inp_params=[("SPACE_GROUP_NUMBER", "%d" % sgnum),
                                          ("UNIT_CELL_CONSTANTS", cell)])
            else:
                print >> decilog, " pointless failed."

        flag_do_not_change_symm = False

        if xs_prior and params.cell_prior.force:
            modify_xdsinp(xdsinp,
                          inp_params=[("UNIT_CELL_CONSTANTS", " ".join(
                              map(lambda x: "%.3f" % x,
                                  params.cell_prior.cell))),
                                      ("SPACE_GROUP_NUMBER",
                                       "%d" % params.cell_prior.sgnum)])
            flag_do_not_change_symm = True
        elif params.cell_prior.method == "correct_only":
            xsxds = XPARM(xparm).crystal_symmetry()
            cosets = reindex.reindexing_operators(xs_prior, xsxds,
                                                  params.cell_prior.tol_length,
                                                  params.cell_prior.tol_angle)
            if cosets.double_cosets is not None:
                cell = xsxds.unit_cell().change_basis(
                    cosets.combined_cb_ops()[0])
                print >> decilog, " Using given symmetry in CORRECT with symmetry constraints:", cell
                modify_xdsinp(xdsinp,
                              inp_params=[
                                  ("UNIT_CELL_CONSTANTS", " ".join(
                                      map(lambda x: "%.3f" % x,
                                          cell.parameters()))),
                                  ("SPACE_GROUP_NUMBER",
                                   "%d" % params.cell_prior.sgnum),
                              ])
                flag_do_not_change_symm = True
            else:
                print >> decilog, " Tried to use given symmetry in CORRECT, but cell in integration is incompatible."

        # Do Scaling
        modify_xdsinp(xdsinp, inp_params=[
            ("JOB", "CORRECT"),
        ])

        run_xds(wdir=root, show_progress=params.show_progress)

        if not os.path.isfile(xac_hkl):
            print >> decilog, " CORRECT failed."
            return

        if not os.path.isfile(gxparm):
            print >> decilog, " Refinement in CORRECT failed."

        print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
            correct_lp, check_valid=True)

        ret = calc_merging_stats(xac_hkl)
        if params.cut_resolution:
            if ret is not None and ret[0] is not None:
                d_min = ret[0]
                modify_xdsinp(xdsinp,
                              inp_params=[("JOB", "CORRECT"),
                                          ("INCLUDE_RESOLUTION_RANGE",
                                           "50 %.2f" % d_min)])
                print >> decilog, " Re-scale at %.2f A" % d_min
                os.rename(os.path.join(root, "CORRECT.LP"),
                          os.path.join(root, "CORRECT_fullres.LP"))
                os.rename(xac_hkl, os.path.join(root, "XDS_ASCII_fullres.HKL"))
                run_xds(wdir=root, show_progress=params.show_progress)
                print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
                    correct_lp, check_valid=True)
                print >> decilog, " (Original files are saved as *_fullres.*)"
            else:
                print >> decilog, "error: Can't decide resolution."

        last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)

        # Run pointless and (if result is different from INTEGRATE) re-scale.
        if params.use_pointless:
            worker = Pointless()
            pointless_correct = worker.run_for_symm(
                xdsin=xac_hkl,
                logout=os.path.join(root, "pointless_correct.log"))
            pointless_best_symm = None

            if "symm" in pointless_correct:
                symm = pointless_correct["symm"]
                need_rescale = False

                if pointless_integrate.get("symm"):
                    symm_by_integrate = pointless_integrate["symm"]

                    if not xtal.is_same_laue_symmetry(
                            symm_by_integrate.space_group(),
                            symm.space_group()):
                        print >> decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (
                            symm.space_group_info(),
                            symm_by_integrate.space_group_info())
                        prob_integrate = pointless_integrate.get(
                            "laue_prob", float("nan"))
                        prob_correct = pointless_correct.get(
                            "laue_prob", float("nan"))

                        print >> decilog, " Prob(%s |INTEGRATE), Prob(%s |CORRECT) = %.4f, %.4f." % (
                            symm_by_integrate.space_group_info(),
                            symm.space_group_info(), prob_integrate,
                            prob_correct)
                        if prob_correct > prob_integrate:
                            need_rescale = True
                            pointless_best_symm = symm
                        else:
                            pointless_best_symm = symm_by_integrate
                else:
                    need_rescale = True
                    pointless_best_symm = symm
                    print >> decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info(
                    )
                    if xs_prior:
                        if xtal.is_same_space_group_ignoring_enantiomorph(
                                symm.space_group(), xs_prior.space_group()):
                            print >> decilog, " which is consistent with given symmetry."
                        elif xtal.is_same_laue_symmetry(
                                symm.space_group(), xs_prior.space_group()):
                            print >> decilog, " which has consistent Laue symmetry with given symmetry."
                        else:
                            print >> decilog, " which is inconsistent with given symmetry."

                if need_rescale and not flag_do_not_change_symm:
                    sgnum = symm.space_group_info().type().number()
                    cell = " ".join(
                        map(lambda x: "%.2f" % x,
                            symm.unit_cell().parameters()))
                    modify_xdsinp(xdsinp,
                                  inp_params=[
                                      ("JOB", "CORRECT"),
                                      ("SPACE_GROUP_NUMBER", "%d" % sgnum),
                                      ("UNIT_CELL_CONSTANTS", cell),
                                      ("INCLUDE_RESOLUTION_RANGE", "50 0")
                                  ])

                    run_xds(wdir=root, show_progress=params.show_progress)

                    ret = calc_merging_stats(xac_hkl)

                    if params.cut_resolution:
                        if ret is not None and ret[0] is not None:
                            d_min = ret[0]
                            modify_xdsinp(xdsinp,
                                          inp_params=[
                                              ("JOB", "CORRECT"),
                                              ("INCLUDE_RESOLUTION_RANGE",
                                               "50 %.2f" % d_min)
                                          ])
                            print >> decilog, " Re-scale at %.2f A" % d_min
                            os.rename(os.path.join(root, "CORRECT.LP"),
                                      os.path.join(root, "CORRECT_fullres.LP"))
                            os.rename(
                                xac_hkl,
                                os.path.join(root, "XDS_ASCII_fullres.HKL"))
                            run_xds(wdir=root,
                                    show_progress=params.show_progress)
                            print >> decilog, " OK. ISa= %.2f" % correctlp.get_ISa(
                                correct_lp, check_valid=True)
                            print >> decilog, " (Original files are saved as *_fullres.*)"
                        else:
                            print >> decilog, "error: Can't decide resolution."
                            for f in ("CORRECT_fullres.LP",
                                      "XDS_ASCII_fullres.HKL"):
                                if os.path.isfile(os.path.join(root, f)):
                                    print >> decilog, "removing", f
                                    os.remove(os.path.join(root, f))

                    ISa = correctlp.get_ISa(correct_lp, check_valid=True)

                    if ISa >= last_ISa or last_ISa != last_ISa:  # if improved or last_ISa is nan
                        print >> decilog, "ISa improved= %.2f" % ISa
                    else:
                        print >> decilog, "ISa got worse= %.2f" % ISa

            if pointless_best_symm:
                xac_symm = XDS_ASCII(xac_hkl, read_data=False).symm
                if not xtal.is_same_space_group_ignoring_enantiomorph(
                        xac_symm.space_group(),
                        pointless_best_symm.space_group()):
                    if xtal.is_same_laue_symmetry(
                            xac_symm.space_group(),
                            pointless_best_symm.space_group()):
                        tmp = "same Laue symmetry"
                    else:
                        tmp = "different Laue symmetry"
                    print >> decilog, "WARNING: symmetry in scaling is different from Pointless result (%s)." % tmp

        run_xdsstat(wdir=root)
        print
        if params.make_report: html_report.make_individual_report(root, root)
    except:
        print >> decilog, traceback.format_exc()
    finally:
        print >> decilog, "\nxds_sequence finished at %s" % time.strftime(
            "%Y-%m-%d %H:%M:%S")
        decilog.close()
Beispiel #8
0
def rescale_with_specified_symm(topdir,
                                dirs,
                                symms,
                                out,
                                sgnum=None,
                                reference_symm=None):
    assert (sgnum, reference_symm).count(None) == 1

    if sgnum is not None:
        sgnum_laue = sgtbx.space_group_info(sgnum).group(
        ).build_derived_reflection_intensity_group(False).type().number()

        matches = filter(
            lambda x: x.reflection_intensity_symmetry(False).space_group_info(
            ).type().number() == sgnum_laue, symms)
        matched_cells = numpy.array(
            map(lambda x: x.unit_cell().parameters(), matches))
        median_cell = map(lambda x: numpy.median(matched_cells[:, x]),
                          xrange(6))

        reference_symm = crystal.symmetry(median_cell, sgnum)
    else:
        sgnum = reference_symm.space_group_info().type().number()
        sgnum_laue = reference_symm.space_group(
        ).build_derived_reflection_intensity_group(False).type().number()

    print >> out
    print >> out, "Re-scaling with specified symmetry:", reference_symm.space_group_info(
    ).symbol_and_number()
    print >> out, " reference cell:", reference_symm.unit_cell()
    print >> out
    print >> out

    cells = {}  # cell and file
    for sym, wd in zip(symms, dirs):
        print >> out, os.path.relpath(wd, topdir),

        # Find appropriate data
        xac_file = util.return_first_found_file(
            ("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale",
             "XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
             "XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
            wd=wd)
        if xac_file is None:
            print >> out, "Can't find XDS_ASCII file in %s" % wd
            continue

        xac = XDS_ASCII(xac_file, read_data=False)
        print >> out, "%s %s (%s)" % (
            os.path.basename(xac_file), xac.symm.space_group_info(), ",".join(
                map(lambda x: "%.2f" % x,
                    xac.symm.unit_cell().parameters())))

        if xac.symm.reflection_intensity_symmetry(
                False).space_group_info().type().number() == sgnum_laue:
            if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(),
                                                  0.1, 10):
                print >> out, "  Already scaled with specified symmetry"
                cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()),
                             xac_file)
                continue

        xdsinp = os.path.join(wd, "XDS.INP")
        cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2,
                                              20)

        if len(cosets.combined_cb_ops()) == 0:
            print >> out, "Can't find operator:"
            sym.show_summary(out, " ")
            reference_symm.show_summary(out, " ")
            continue

        newcell = reference_symm.space_group().average_unit_cell(
            xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
        newcell = " ".join(map(lambda x: "%.3f" % x, newcell.parameters()))
        print >> out, "Scaling with transformed cell:", newcell

        #for f in xds_files.generated_by_CORRECT:
        #    util.rotate_file(os.path.join(wd, f))
        bk_prefix = make_backup(xds_files.generated_by_CORRECT,
                                wdir=wd,
                                quiet=True)

        modify_xdsinp(
            xdsinp,
            inp_params=[
                ("JOB", "CORRECT"),
                ("SPACE_GROUP_NUMBER", "%d" % sgnum),
                ("UNIT_CELL_CONSTANTS", newcell),
                ("INCLUDE_RESOLUTION_RANGE", "50 0"),
                ("CORRECTIONS", ""),
                ("NBATCH", "1"),
                ("MINIMUM_I/SIGMA", None),  # use default
                ("REFINE(CORRECT)", None),  # use default
            ])
        run_xds(wd)
        for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
            if os.path.exists(os.path.join(wd, f)):
                shutil.copyfile(os.path.join(wd, f),
                                os.path.join(wd, f + "_rescale"))

        revert_files(xds_files.generated_by_CORRECT,
                     bk_prefix,
                     wdir=wd,
                     quiet=True)

        new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
        new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
        if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
            cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
            print "OK:", cells[wd][0]
        else:
            print >> out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
            continue

    return cells, reference_symm
Beispiel #9
0
def rescale_with_specified_symm(topdir, dirs, symms, out, sgnum=None, reference_symm=None):
    assert (sgnum, reference_symm).count(None) == 1

    if sgnum is not None:
        sgnum_laue = sgtbx.space_group_info(sgnum).group().build_derived_reflection_intensity_group(False).type().number()

        matches = filter(lambda x:x.reflection_intensity_symmetry(False).space_group_info().type().number()==sgnum_laue, symms)
        matched_cells = numpy.array(map(lambda x: x.unit_cell().parameters(), matches))
        median_cell = map(lambda x: numpy.median(matched_cells[:,x]), xrange(6))

        reference_symm = crystal.symmetry(median_cell, sgnum)
    else:
        sgnum = reference_symm.space_group_info().type().number()
        sgnum_laue = reference_symm.space_group().build_derived_reflection_intensity_group(False).type().number()
    
    print >>out
    print >>out,  "Re-scaling with specified symmetry:", reference_symm.space_group_info().symbol_and_number()
    print >>out,  " reference cell:", reference_symm.unit_cell()
    print >>out
    print >>out

    cells = {} # cell and file
    for sym, wd in zip(symms, dirs):
        print >>out,  os.path.relpath(wd, topdir),

        # Find appropriate data
        xac_file = util.return_first_found_file(("XDS_ASCII.HKL_noscale.org", "XDS_ASCII.HKL_noscale", 
                                                 "XDS_ASCII_fullres.HKL.org", "XDS_ASCII_fullres.HKL",
                                                 "XDS_ASCII.HKL.org", "XDS_ASCII.HKL"),
                                                wd=wd)
        if xac_file is None:
            print >>out, "Can't find XDS_ASCII file in %s" % wd
            continue

        xac = XDS_ASCII(xac_file, read_data=False)
        print >>out, "%s %s (%s)" % (os.path.basename(xac_file), xac.symm.space_group_info(),
                                     ",".join(map(lambda x: "%.2f"%x, xac.symm.unit_cell().parameters())))

        if xac.symm.reflection_intensity_symmetry(False).space_group_info().type().number() == sgnum_laue:
            if xac.symm.unit_cell().is_similar_to(reference_symm.unit_cell(), 0.1, 10):
                print >>out,  "  Already scaled with specified symmetry"
                cells[wd] = (numpy.array(xac.symm.unit_cell().parameters()), xac_file)
                continue

        xdsinp = os.path.join(wd, "XDS.INP")
        cosets = reindex.reindexing_operators(reference_symm, xac.symm, 0.2, 20)

        if len(cosets.combined_cb_ops())==0:
            print >>out, "Can't find operator:"
            sym.show_summary(out, " ")
            reference_symm.show_summary(out, " ")
            continue

        newcell = reference_symm.space_group().average_unit_cell(xac.symm.change_basis(cosets.combined_cb_ops()[0]).unit_cell())
        newcell = " ".join(map(lambda x: "%.3f"%x, newcell.parameters()))
        print >>out,  "Scaling with transformed cell:", newcell

        #for f in xds_files.generated_by_CORRECT:
        #    util.rotate_file(os.path.join(wd, f))
        bk_prefix = make_backup(xds_files.generated_by_CORRECT, wdir=wd, quiet=True)

        modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                          ("SPACE_GROUP_NUMBER", "%d"%sgnum),
                                          ("UNIT_CELL_CONSTANTS", newcell),
                                          ("INCLUDE_RESOLUTION_RANGE", "50 0"),
                                          ("CORRECTIONS", ""),
                                          ("NBATCH", "1"),
                                          ("MINIMUM_I/SIGMA", None), # use default
                                          ("REFINE(CORRECT)", None), # use default
                                          ])
        run_xds(wd)
        for f in ("XDS.INP", "CORRECT.LP", "XDS_ASCII.HKL", "GXPARM.XDS"):
            if os.path.exists(os.path.join(wd, f)):
                shutil.copyfile(os.path.join(wd, f), os.path.join(wd, f+"_rescale"))

        revert_files(xds_files.generated_by_CORRECT, bk_prefix, wdir=wd, quiet=True)

        new_xac = os.path.join(wd, "XDS_ASCII.HKL_rescale")
        new_gxparm = os.path.join(wd, "GXPARM.XDS_rescale")
        if os.path.isfile(new_xac) and os.path.isfile(new_gxparm):
            cells[wd] = (XPARM(new_gxparm).unit_cell, new_xac)
            print "OK:", cells[wd][0]
        else:
            print >>out, "Error: rescaling failed (Can't find XDS_ASCII.HKL)"
            continue

    return cells, reference_symm
Beispiel #10
0
def xds_sequence(root, params):
    print
    print os.path.relpath(root, params.topdir)

    xparm = os.path.join(root, "XPARM.XDS")
    gxparm = os.path.join(root, "GXPARM.XDS")
    defpix_lp = os.path.join(root, "DEFPIX.LP")
    correct_lp = os.path.join(root, "CORRECT.LP")
    integrate_hkl = os.path.join(root, "INTEGRATE.HKL")
    xac_hkl = os.path.join(root, "XDS_ASCII.HKL")
    integrate_lp = os.path.join(root, "INTEGRATE.LP")
    xdsinp = os.path.join(root, "XDS.INP")

    assert os.path.isfile(xdsinp)

    decilog = multi_out()
    decilog.register("log", open(os.path.join(root, "decision.log"), "a"), atexit_send_to=None)

    print >>decilog, "xds_sequence started at %s in %s\n" % (time.strftime("%Y-%m-%d %H:%M:%S"), root)
    
    if params.show_progress:
        decilog.register("stdout", sys.stdout)

    if params.mode=="initial" and params.resume and os.path.isfile(correct_lp):
        print " Already processed."
        return

    if params.mode == "recycle" and not os.path.isfile(gxparm):
        print "GXPARM.XDS not found. Cannot do recycle."
        return

    if params.fast_delphi and (params.nproc is None or params.nproc > 1):
        delphi = optimal_delphi_by_nproc(xdsinp=xdsinp, nproc=params.nproc)
        print " Setting delphi to ", delphi
        modify_xdsinp(xdsinp, inp_params=[("DELPHI", str(delphi)),
                                          ])

    if params.nproc is not None and params.nproc > 1:
        modify_xdsinp(xdsinp, inp_params=[("MAXIMUM_NUMBER_OF_PROCESSORS", str(params.nproc)),
                                          ])

    if params.mode == "initial":
        # To Indexing
        modify_xdsinp(xdsinp, inp_params=[("JOB", "XYCORR INIT COLSPOT IDXREF")])
        run_xds(wdir=root, show_progress=params.show_progress)
        print # indexing stats like indexed percentage here.

        if params.tryhard:
            try_indexing_hard(root, params.show_progress, decilog,
                              known_sgnum=params.cell_prior.sgnum,
                              known_cell=params.cell_prior.cell,
                              tol_length=params.cell_prior.tol_length,
                              tol_angle=params.cell_prior.tol_angle)

        if not os.path.isfile(xparm):
            print >>decilog, " Indexing failed."
            return

        if params.cell_prior.check and params.cell_prior.sgnum > 0:
            xsxds = XPARM(xparm).crystal_symmetry()
            xsref = crystal.symmetry(params.cell_prior.cell, params.cell_prior.sgnum)
            cosets = reindex.reindexing_operators(xsref, xsxds,
                                                  params.cell_prior.tol_length, params.cell_prior.tol_angle)
            if cosets.double_cosets is None:
                print >>decilog, " Incompatible cell. Indexing failed."
                return

    elif params.mode == "recycle":
        print " Start recycle. original ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
        for f in xds_files.generated_after_DEFPIX + ("XPARM.XDS", "plot_integrate.log"):
            util.rotate_file(os.path.join(root, f), copy=True)
        shutil.copyfile(gxparm+".1", xparm)
    else:
        raise "Unknown mode (%s)" % params.mode

    # To Integration
    modify_xdsinp(xdsinp, inp_params=[("JOB", "DEFPIX INTEGRATE"),
                                      ("INCLUDE_RESOLUTION_RANGE", "50 0")])
    run_xds(wdir=root, show_progress=params.show_progress)
    if os.path.isfile(integrate_lp):
        xds_plot_integrate.run(integrate_lp, os.path.join(root, "plot_integrate.log"))
    if not os.path.isfile(integrate_hkl):
        print >>decilog, " Integration failed."
        return


    # Make _noscale.HKL if needed
    if params.no_scaling:
        bk_prefix = make_backup(("XDS.INP",), wdir=root, quiet=True)
        xparm_obj = XPARM(xparm)
        modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                          ("CORRECTIONS", ""),
                                          ("NBATCH", "1"),
                                          ("MINIMUM_I/SIGMA", "50"),
                                          ("REFINE(CORRECT)", ""),
                                          ("UNIT_CELL_CONSTANTS", " ".join(map(lambda x:"%.3f"%x, xparm_obj.unit_cell))),
                                          ("SPACE_GROUP_NUMBER", "%d"%xparm_obj.spacegroup),])
        print >>decilog, " running CORRECT without empirical scaling"
        run_xds(wdir=root, show_progress=params.show_progress)
        for f in xds_files.generated_by_CORRECT + ("XDS.INP",):
            ff = os.path.join(root, f)
            if not os.path.isfile(ff): continue
            if ff.endswith(".cbf"):
                os.remove(ff)
            else:
                os.rename(ff, ff+"_noscale")

        revert_files(("XDS.INP",), bk_prefix, wdir=root, quiet=True)

    # Run pointless
    symm_by_integrate = None
    if params.use_pointless:
        worker = Pointless()
        result = worker.run_for_symm(xdsin=integrate_hkl, 
                                     logout=os.path.join(root, "pointless_integrate.log"))
        if "symm" in result:
            symm = result["symm"]
            print >>decilog, " pointless using INTEGRATE.HKL suggested", symm.space_group_info()
            sgnum = symm.space_group_info().type().number()
            cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
            modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "%d"%sgnum),
                                              ("UNIT_CELL_CONSTANTS", cell)])
            symm_by_integrate = symm
        else:
            print >>decilog, " pointless failed."

    # Do Scaling
    modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),])

    run_xds(wdir=root, show_progress=params.show_progress)

    if not os.path.isfile(gxparm):
        print >>decilog, " Scaling failed."
        return

    print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)

    ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
    if params.cut_resolution:
        if ret is not None and ret[0] is not None:
            d_min = ret[0]
            modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                              ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
            print >>decilog, " Re-scale at %.2f A" % d_min
            os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
            os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
            run_xds(wdir=root, show_progress=params.show_progress)
            print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
            print >>decilog, " (Original files are saved as *_fullres.*)"
        else:
            print >>decilog, "error: Can't decide resolution."

    last_ISa = correctlp.get_ISa(correct_lp, check_valid=True)

    # Run pointless and (if result is different from INTEGRATE) re-scale.
    if params.use_pointless:
        worker = Pointless()
        result = worker.run_for_symm(xdsin=xac_hkl,
                                     logout=os.path.join(root, "pointless_correct.log"))
        if "symm" in result:
            symm = result["symm"]
            need_rescale = False

            if symm_by_integrate is not None:
                if not xtal.is_same_laue_symmetry(symm_by_integrate.space_group(), symm.space_group()):
                    print >>decilog, "pointless suggested %s, which is different Laue symmetry from INTEGRATE.HKL (%s)" % (symm.space_group_info(), symm_by_integrate.space_group_info())
                    need_rescale = True
            else:
                print >>decilog, "pointless using XDS_ASCII.HKL suggested %s" % symm.space_group_info()
                need_rescale = True

            if need_rescale:
                # make backup, and do correct and compare ISa
                # if ISa got worse, revert the result.
                backup_needed = ("XDS.INP", "XDS_ASCII_fullres.HKL","CORRECT_fullres.LP",
                                 "merging_stats.pkl","merging_stats.log")
                backup_needed += xds_files.generated_by_CORRECT
                bk_prefix = make_backup(backup_needed, wdir=root, quiet=True)

                sgnum = symm.space_group_info().type().number()
                cell = " ".join(map(lambda x:"%.2f"%x, symm.unit_cell().parameters()))
                modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                                  ("SPACE_GROUP_NUMBER", "%d"%sgnum),
                                                  ("UNIT_CELL_CONSTANTS", cell),
                                                  ("INCLUDE_RESOLUTION_RANGE", "50 0")])

                run_xds(wdir=root, show_progress=params.show_progress)

                ret = calc_merging_stats(os.path.join(root, "XDS_ASCII.HKL"))
                
                if params.cut_resolution:
                    if ret is not None and ret[0] is not None:
                        d_min = ret[0]
                        modify_xdsinp(xdsinp, inp_params=[("JOB", "CORRECT"),
                                                          ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%d_min)])
                        print >>decilog, " Re-scale at %.2f A" % d_min
                        os.rename(os.path.join(root, "CORRECT.LP"), os.path.join(root, "CORRECT_fullres.LP"))
                        os.rename(os.path.join(root, "XDS_ASCII.HKL"), os.path.join(root, "XDS_ASCII_fullres.HKL"))
                        run_xds(wdir=root, show_progress=params.show_progress)
                        print >>decilog, " OK. ISa= %.2f" % correctlp.get_ISa(correct_lp, check_valid=True)
                        print >>decilog, " (Original files are saved as *_fullres.*)"
                    else:
                        print >>decilog, "error: Can't decide resolution."
                        for f in ("CORRECT_fullres.LP", "XDS_ASCII_fullres.HKL"):
                            if os.path.isfile(os.path.join(root, f)):
                                print >>decilog, "removing", f
                                os.remove(os.path.join(root, f))

                ISa = correctlp.get_ISa(correct_lp, check_valid=True)

                if ISa >= last_ISa or last_ISa!=last_ISa: # if improved or last_ISa is nan
                    print >>decilog, "ISa improved= %.2f" % ISa
                    remove_backups(backup_needed, bk_prefix, wdir=root)
                else:
                    print >>decilog, "ISa got worse= %.2f" % ISa
                    for f in backup_needed:
                        if os.path.isfile(os.path.join(root, f)): os.remove(os.path.join(root, f))

                    revert_files(backup_needed, bk_prefix, wdir=root, quiet=True)

    run_xdsstat(wdir=root)
    print
    if params.make_report: html_report.make_individual_report(root, root)
    print >>decilog, "xds_sequence finished at %s\n" % time.strftime("%Y-%m-%d %H:%M:%S")
    decilog.close()
Beispiel #11
0
def try_indexing_hard(wdir, show_progress, decilog, 
                      known_sgnum=None, known_cell=None, tol_length=None, tol_angle=None):
    idxref_lp = os.path.join(wdir, "IDXREF.LP")
    xdsinp = os.path.join(wdir, "XDS.INP")

    lp_org = idxreflp.IdxrefLp(idxref_lp)

    if lp_org.is_cell_maybe_half():
        backup_needed = ("XDS.INP",) + xds_files.generated_by_IDXREF

        print >>decilog, " !! Cell may be halved. Trying doubled cell."
        bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)

        cell = lp_org.deduce_correct_cell_based_on_integerness()
        cell = " ".join(map(lambda x:"%.2f"%x, cell.parameters()))
        modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                          ("SPACE_GROUP_NUMBER", "1"),
                                          ("UNIT_CELL_CONSTANTS", cell)
                                          ])
        run_xds(wdir=wdir, show_progress=show_progress)

        if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
            revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)

            print >>decilog, "  .. not solved. Next, try decreasing SEPMIN= and CLUSTER_RADIUS=."
            bk_prefix = make_backup(backup_needed, wdir=wdir, quiet=True)

            modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                              ("SEPMIN", "4"),
                                              ("CLUSTER_RADIUS", "2")
                                              ])
            run_xds(wdir=wdir, show_progress=show_progress)

            if idxreflp.IdxrefLp(idxref_lp).is_cell_maybe_half():
                print >>decilog, "  .. not solved. Give up."
                revert_files(backup_needed, bk_prefix, wdir=wdir, quiet=True)
        else:
            print >>decilog, "  Now OK."
            remove_backups(backup_needed, bk_prefix, wdir=wdir)
            modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
                                              ])

    # If Cell hint exists, try to use it..
    if known_sgnum > 0:
        flag_try_cell_hint = False
        xparm = os.path.join(wdir, "XPARM.XDS")
        if not os.path.isfile(xparm):
            flag_try_cell_hint = True
        else:
            xsxds = XPARM(xparm).crystal_symmetry()

            xsref = crystal.symmetry(known_cell, known_sgnum)
            cosets = reindex.reindexing_operators(xsref, xsxds,
                                                  tol_length, tol_angle)

            if cosets.double_cosets is None: flag_try_cell_hint = True

        if flag_try_cell_hint:
            print >>decilog, " Worth trying to use prior cell for indexing."
            modify_xdsinp(xdsinp, inp_params=[("JOB", "IDXREF"),
                                              ("UNIT_CELL_CONSTANTS",
                                               " ".join(map(lambda x: "%.3f"%x, known_cell))),
                                              ("SPACE_GROUP_NUMBER", "%d"%known_sgnum),
                                              ])
            run_xds(wdir=wdir, show_progress=False)
            modify_xdsinp(xdsinp, inp_params=[("SPACE_GROUP_NUMBER", "0"),
                                              ])           
Beispiel #12
0
def run(params, out):
    print >>out, "Frames:", params.frames
    backup_needed = xds_files.generated_by_DEFPIX + ("XDS.INP","BKGINIT.cbf",)
    bk_prefix = make_backup(backup_needed, wdir=params.xdsdir)

    ret = {} # {frame: [matches, spots, predicted]}

    try:
        # run DEFPIX to limit resolution.
        modify_xdsinp(os.path.join(params.xdsdir, "XDS.INP"),
                      [("JOB", "DEFPIX"), ("INCLUDE_RESOLUTION_RANGE", "50 %.2f"%params.d_min)])

        p = subprocess.Popen("xds", cwd=params.xdsdir)
        p.wait()

        # copy BKGPIX.cbf -> BKGINIT.cbf (for COLSPOT)
        shutil.copyfile(os.path.join(params.xdsdir, "BKGPIX.cbf"),
                        os.path.join(params.xdsdir, "BKGINIT.cbf"))
        
        for frame in params.frames:
            print >>out, "Frame %d" % frame
            print >>out, "====================\n"
            # search spots
            if params.spotfinder == "xds":
                spotxds = get_colspot_result(frame_ranges=[[frame, frame],], wdir=params.xdsdir)
                spots = map(lambda x: x[:2], spotxds.collected_spots(with_resolution=False))
            else:
                raise "Sorry!"

            # run INTEGRATE to collect predicted coords
            integrate_results = xds_predict_mitai.run(param_source=os.path.join(params.xdsdir, "XPARM.XDS"), 
                                                      frame_num=frame,
                                                      wdir=params.xdsdir, need_adx=False,
                                                      sigmar=params.sigmar, sigmab=params.sigmab)

            # read predicted coords
            tmp = filter(lambda x:x.endswith(".HKL"), integrate_results)
            if len(tmp) == 0:
                print >>out, "Integration failed!"
                ret[frame] = (0, len(spots), 0)
                continue

            integrate_hkl = tmp[0]
            cols = integrate_hkl_as_flex.reader(integrate_hkl, [], False).get_column_names()
            i_xcal, i_ycal = cols.index("XCAL"), cols.index("YCAL")
            predicted = []

            for l in open(integrate_hkl):
                if l.startswith("!"): continue
                sp = l.split()
                predicted.append(map(float, (sp[i_xcal], sp[i_ycal])))

            # compare them
            nmatch = calc_matches(spots, predicted, params.distance_limit_in_px,
                                  open(os.path.join(params.xdsdir, "matched_predicted_%.4d.adx"%frame), "w"))
            #nmatch = calc_matches(predicted, spots, params.distance_limit_in_px,
            #                      open(os.path.join(params.xdsdir, "matched_located_%.4d.adx"%frame), "w"))

            ret[frame] = (nmatch, len(spots), len(predicted))

    finally:
        revert_files(backup_needed, bk_prefix, wdir=params.xdsdir)

    print >>out
    for frame in sorted(ret):
        nmatch, nspots, npredicted = ret[frame]
        print >>out, "Frame %4d Located/Predicted: %d/%d= %.2f%%" % (frame, nmatch, npredicted,
                                                                     100.*float(nmatch)/npredicted if npredicted>0 else float("nan"))
        print >>out, "Frame %4d Predicted/Located: %d/%d= %.2f%%" % (frame, nmatch, nspots,
                                                                     100.*float(nmatch)/nspots if nspots>0 else float("nan"))
        print >>out


    return ret