示例#1
0
文件: e2import.py 项目: kuixu/eman2
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """prog [options] files
	This program performs a variety of tasks for getting data or metadata from other programs into an EMAN2 project.

	import_particles - will simply copy a set of per-micrograph particle files into EMAN2.1's preferred HDF format in particles/
	import_boxes - will read EMAN1 '.box' files (text files containing coordinates) into appropriate info/*json files (see --box_type)
	import_tomos - imports subtomogams for a SPT project (see also --importation)
	import_eman1 - will convert a typical EMAN1 phase-flipped start.hed/img file into an EMAN2 project (converting files, fixing CTF, splitting, ...)
"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_pos_argument(
        name="import_files",
        help="List the files to import here.",
        default="",
        guitype='filebox',
        browser="EMBrowserWidget(withmodal=True,multiselect=True)",
        row=0,
        col=0,
        rowspan=1,
        colspan=2,
        nosharedb=True,
        mode='coords,parts,tomos,eman1')
    parser.add_header(name="filterheader",
                      help='Options below this label are specific to e2import',
                      title="### e2import options ###",
                      row=1,
                      col=0,
                      rowspan=1,
                      colspan=2,
                      mode='coords,parts,tomos')
    parser.add_argument("--import_particles",
                        action="store_true",
                        help="Import particles",
                        default=False,
                        guitype='boolbox',
                        row=2,
                        col=0,
                        rowspan=1,
                        colspan=1,
                        mode='parts[True]')
    parser.add_argument(
        "--import_eman1",
        action="store_true",
        help="This will import a phase-flipped particle stack from EMAN1",
        default=False,
        guitype='boolbox',
        row=2,
        col=0,
        rowspan=1,
        colspan=1,
        mode='eman1[True]')
    parser.add_argument("--import_tomos",
                        action="store_true",
                        help="Import tomograms",
                        default=False,
                        guitype='boolbox',
                        row=2,
                        col=0,
                        rowspan=1,
                        colspan=1,
                        mode='tomos[True]')
    parser.add_argument(
        "--shrink",
        type=int,
        help=
        "Shrink tomograms before importing. Dose not work while not copying.",
        default=1,
        guitype='intbox',
        row=3,
        col=0,
        rowspan=1,
        colspan=1,
        mode='tomos')
    parser.add_argument("--invert",
                        action="store_true",
                        help="Invert the contrast before importing tomograms",
                        default=False,
                        guitype='boolbox',
                        row=3,
                        col=1,
                        rowspan=1,
                        colspan=1,
                        mode='tomos')
    parser.add_argument(
        "--tomoseg_auto",
        action="store_true",
        help=
        "Default process for tomogram segmentation, including lowpass, highpass, normalize, clampminmax.",
        default=True,
        guitype='boolbox',
        row=3,
        col=2,
        rowspan=1,
        colspan=1,
        mode='tomos')
    parser.add_argument(
        "--importation",
        help="Specify mode move, copy or link, for importing tomograms only",
        default='copy',
        guitype='combobox',
        choicelist='["move","copy","link"]',
        row=2,
        col=1,
        rowspan=1,
        colspan=1,
        mode='tomos')
    parser.add_argument(
        "--preprocess",
        type=str,
        help=
        "Other pre-processing operation before importing tomograms. Dose not work while not copying.",
        default="",
        guitype='strbox',
        row=4,
        col=0,
        rowspan=1,
        colspan=2,
        mode='tomos')
    parser.add_argument("--import_boxes",
                        action="store_true",
                        help="Import boxes",
                        default=False,
                        guitype='boolbox',
                        row=2,
                        col=0,
                        rowspan=1,
                        colspan=1,
                        mode='coords[True]')
    parser.add_argument(
        "--extension",
        type=str,
        help="Extension of the micrographs that the boxes match",
        default='dm3')
    parser.add_argument(
        "--box_type",
        help=
        "Type of boxes to import, normally boxes, but for tilted data use tiltedboxes, and untiltedboxes for the tilted  particle partner",
        default="boxes",
        guitype='combobox',
        choicelist=
        '["boxes","coords","relion_star","tiltedboxes","untiltedboxes"]',
        row=2,
        col=1,
        rowspan=1,
        colspan=1,
        mode="coords['boxes']")
    parser.add_argument("--boxsize",
                        help="Specify the boxsize for each particle.",
                        type=int,
                        default=256)
    parser.add_argument(
        "--curdefocushint",
        action="store_true",
        help="Used with import_eman1, will use EMAN1 defocus as starting point",
        default=False,
        guitype='boolbox',
        row=4,
        col=0,
        rowspan=1,
        colspan=1,
        mode='eman1[True]')
    parser.add_argument(
        "--curdefocusfix",
        action="store_true",
        help=
        "Used with import_eman1, will use EMAN1 defocus unchanged (+-.001 um)",
        default=False,
        guitype='boolbox',
        row=4,
        col=1,
        rowspan=1,
        colspan=1,
        mode='eman1[False]')
    parser.add_argument(
        "--threads",
        default=1,
        type=int,
        help=
        "Number of threads to run in parallel on a single computer when multi-computer parallelism isn't useful",
        guitype='intbox',
        row=6,
        col=0,
        rowspan=1,
        colspan=1,
        mode='eman1[1]')
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higner number means higher level of verboseness")
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv, options.ppid)

    # Import EMAN1
    # will read start.hed/img, split by micrograph (based on defocus), and reprocess CTF in EMAN2 style
    if options.import_eman1:
        try:
            n = EMUtil.get_image_count(args[0])
        except:
            print("Error, couldn't read images from: ", args[0])
            sys.exit(1)

        try:
            img = EMData(args[0], 0)
            ctf = img["ctf"]
        except:
            print("Error, start.hed/img must be phase-flipped to import")
            sys.exit(1)

        db = js_open_dict("info/project.json")
        db["global.apix"] = ctf.apix
        db["global.cs"] = ctf.cs
        db["global.voltage"] = ctf.voltage

        try:
            os.mkdir("particles")
        except:
            pass

        imgnum = 0
        lastdf = -1.0
        for i in xrange(n):
            img = EMData(args[0], i)
            ctf = img["ctf"]
            img.del_attr("ctf")
            fft1 = img.do_fft()
            if ctf.defocus != lastdf:
                imgnum += 1
                if options.verbose > 0:
                    print("Defocus {:4.2f} particles{:03d}".format(
                        ctf.defocus, imgnum))
                db = js_open_dict(
                    "info/particles{:03d}_info.json".format(imgnum))
                ctf2 = EMAN2Ctf()
                ctf2.defocus = ctf.defocus
                ctf2.cs = ctf.cs
                ctf2.apix = ctf.apix
                ctf2.voltage = ctf.voltage
                ctf2.ampcont = ctf.ampcont
                ctf2.dfdiff = 0
                ctf2.dfang = 0
                db["ctf"] = [ctf2]
                db.close()

                flipim = fft1.copy()
                ctf2.compute_2d_complex(flipim, Ctf.CtfType.CTF_SIGN)

            lastdf = ctf.defocus

            # unflip the EMAN1 phases (hopefully accurate enough)
            fft1.mult(flipim)
            img = fft1.do_ift()
            img.write_image("particles/particles{:03d}.hdf".format(imgnum),
                            -1)  # append particle to stack

        if options.curdefocusfix:
            flag = "--curdefocusfix"
            rbysnr = " "
        elif options.curdefocushint:
            flag = "--curdefocushint"
            rbysnr = "--refinebysnr"
        else:
            flag = ""
            rbysnr = "--refinebysnr"

        # fill in the needed CTF info
        launch_childprocess(
            "e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {} --computesf"
            .format(flag, options.threads, ctf.voltage, ctf.cs, ctf.ampcont,
                    ctf.apix))
        launch_childprocess(
            "e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {}"
            .format(flag, options.threads, ctf.voltage, ctf.cs, ctf.ampcont,
                    ctf.apix))

        # reflip flip the phases, and make "proc" images
        launch_childprocess(
            "e2ctf.py {} --phaseflip --allparticles --phaseflipproc filter.highpass.gauss:cutoff_freq=0.005 --phaseflipproc2 filter.lowpass.gauss:cutoff_freq=0.08 --phaseflipproc3 math.meanshrink:n=2"
            .format(rbysnr))

        # build sets
        launch_childprocess("e2buildsets.py --allparticles --setname all")

    # Import boxes
    if options.import_boxes:
        # Check to make sure there are micrographs
        if not os.access("info", os.R_OK):
            os.mkdir("info")
        # Do imports
        # we add boxsize/2 to the coords since box files are stored with origin being the lower left side of the box, but in EMAN2 origin is in the center
        if options.box_type == 'boxes':
            micros = os.listdir("micrographs")
            for filename in args:
                boxlist = []
                fh = open(filename, 'r')
                for line in fh.readlines():
                    if line[0] == "#": continue
                    fields = line.split()
                    if len(fields) < 4: continue  # skip lines that don't work
                    boxlist.append([
                        float(fields[0]) + float(fields[3]) / 2,
                        float(fields[1]) + float(fields[3]) / 2, 'manual'
                    ])

                js = js_open_dict(info_name(filename, nodir=True))
                js["boxes"] = boxlist
                js.close()
                if not "{}.hdf".format(base_name(filename,
                                                 nodir=True)) in micros:
                    print(
                        "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist"
                        .format(base_name(filename), base_name(filename,
                                                               True)))

        elif options.box_type == 'coords':
            micros = os.listdir("micrographs")
            for filename in args:
                boxlist = []
                fh = open(filename, 'r')
                for line in fh.readlines():
                    if line[0] == "#": continue
                    fields = line.split()
                    if len(fields) < 2: continue  # skip lines that don't work
                    boxlist.append(
                        [float(fields[0]),
                         float(fields[1]), 'manual'])
                js_open_dict(info_name(filename,
                                       nodir=True))["boxes"] = boxlist
                if not "{}.hdf".format(base_name(filename,
                                                 nodir=True)) in micros:
                    print(
                        "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist"
                        .format(base_name(filename), base_name(filename,
                                                               True)))

        elif options.box_type == 'tiltedboxes':

            for filename in args:
                boxlist = []
                fh = open(filename, 'r')
                for line in fh.readlines():
                    if line[0] == "#": continue
                    fields = line.split()
                    if len(fields) < 4: continue  # skip lines that don't work
                    boxlist.append([
                        float(fields[0]) + float(fields[3]) / 2,
                        float(fields[1]) + float(fields[3]) / 2, 'tilted'
                    ])
                js_open_dict(info_name(filename,
                                       nodir=True))["boxes_rct"] = boxlist

        elif options.box_type == 'untiltedboxes':
            for filename in args:
                boxlist = []
                fh = open(filename, 'r')
                for line in fh.readlines():
                    if line[0] == "#": continue
                    fields = line.split()
                    if len(fields) < 4: continue  # skip lines that don't work
                    boxlist.append([
                        float(fields[0]) + float(fields[3]) / 2,
                        float(fields[1]) + float(fields[3]) / 2, 'untilted'
                    ])
                js_open_dict(info_name(filename,
                                       nodir=True))["boxes_rct"] = boxlist

        elif options.box_type == 'relion_star':
            bs = options.boxsize
            starfs = [f for f in args if '.star' in f]
            if len(starfs) < 1:
                print(
                    "You must specify at least one .star file containing particle coordinates"
                )
                exit(1)
            for filename in starfs:
                print(("Importing from {}.star".format(
                    base_name(filename, nodir=True))))
                sf = StarFile(filename)
                hdr = sf.keys()
                if len(hdr) < 3:
                    print(("Could not parse {}".format(filename)))
                    continue
                mk = "rlnMicrographName"
                yk = "rlnCoordinateY"
                xk = "rlnCoordinateX"
                project_micros = os.listdir('micrographs')
                if mk not in hdr or yk not in hdr or xk not in hdr:
                    possible = "{}.hdf".format(
                        base_name(filename.replace('_autopick.star', ''),
                                  nodir=True))
                    if possible in project_micros:
                        micros = [possible]
                    else:
                        print((
                            "{} does not follow the RELION header convention for single particle data. To use this program"
                            .format(filename)))
                        if mk not in hdr:
                            print(
                                "Micrograph names should be listed under _rlnMicrographName"
                            )
                        if yk not in hdr:
                            print(
                                "Y coordinates must be listed under _rlnCoordinateY"
                            )
                        if xk not in hdr:
                            print(
                                "X coordinates must be listed under _rlnCoordinateX"
                            )
                        continue
                else:
                    micros = [i.split('/')[-1] for i in np.unique(sf[mk])]
                if len(micros) == 1:
                    mg = micros[0]
                    boxlist = []
                    print(("Found {} boxes for {}".format(len(sf[xk]), mg)))
                    for x, y in zip(sf[xk], sf[yk]):
                        xc = int(x)
                        yc = int(y)
                        boxlist.append([
                            xc, yc, 'manual'
                        ])  # should probably be 'relion' or 'from_star'
                    js_open_dict(info_name(mg, nodir=True))["boxes"] = boxlist
                    if not "{}.hdf".format(base_name(
                            mg, nodir=True)) in project_micros:
                        print(
                            "Warning: Imported boxes for {}.hdf, but micrographs/{}.hdf does not exist"
                            .format(base_name(filename),
                                    base_name(mg, nodir=True)))
                elif len(micros) > 1:
                    for mg in project_micros:
                        boxlist = []
                        ptcls = []
                        for i, name in enumerate(sf[mk]):
                            mgname = name.split('/')[-1].split('.')[0]
                            #print(mgname,hdf_name,mg)
                            if mg[:-4] in mgname: ptcls.append(i)
                        print(("Found {} boxes for {}".format(len(ptcls), mg)))
                        for p in ptcls:
                            xc = int(sf[xk][p])
                            yc = int(sf[yk][p])
                            boxlist.append([xc, yc, 'manual'])
                        js_open_dict(info_name(mg,
                                               nodir=True))["boxes"] = boxlist
                        if not "{}.hdf".format(base_name(
                                mg, nodir=True)) in project_micros:
                            print(
                                "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist"
                                .format(base_name(mg), base_name(mg,
                                                                 nodir=True)))

        else:
            print("ERROR: Unknown box_type")

    # Import particles
    if options.import_particles:
        if not os.access("particles", os.R_OK):
            os.mkdir("particles")

        fset = set([base_name(i) for i in args])
        if len(fset) != len(args):
            print(
                "ERROR: You specified multiple files to import with the same base name, eg - a10/abc123.spi and a12/abc123.spi. If you have multiple images with the same \
name, you will need to modify your naming convention (perhaps by prefixing the date) before importing. If the input files are in IMAGIC format, so you have .hed and .img files \
with the same name, you should specify only the .hed files (no renaming is necessary)."
            )
            sys.exit(1)

        for i, fsp in enumerate(args):
            E2progress(logid, float(i) / len(args))
            if EMData(fsp, 0, True)["nz"] > 1:
                run("e2proc2d.py {} particles/{}.hdf --threed2twod --inplace".
                    format(fsp, base_name(fsp)))
            else:
                run("e2proc2d.py {} particles/{}.hdf --inplace".format(
                    fsp, base_name(fsp)))

    # Import tomograms
    if options.import_tomos:
        tomosdir = os.path.join(".", "rawtomograms")
        if not os.access(tomosdir, os.R_OK):
            os.mkdir("rawtomograms")
        for filename in args:
            if options.importation == "move":
                os.rename(filename,
                          os.path.join(tomosdir, os.path.basename(filename)))
            if options.importation == "copy":
                ### use hdf file as output

                if options.shrink > 1:
                    shrinkstr = "_bin{:d}".format(options.shrink)
                else:
                    shrinkstr = ""

                tpos = filename.rfind('.')
                if tpos > 0:
                    newname = os.path.join(
                        tomosdir,
                        os.path.basename(filename[:tpos] + shrinkstr + '.hdf'))
                else:
                    newname = os.path.join(tomosdir,
                                           os.path.basename(filename))
                cmd = "e2proc3d.py {} {} ".format(filename, newname)
                if options.shrink > 1:
                    cmd += " --meanshrink {:d} ".format(options.shrink)
                if options.invert:
                    cmd += " --mult -1 --process normalize "
                if options.tomoseg_auto:
                    cmd += " --process filter.lowpass.gauss:cutoff_abs=.25 --process filter.highpass.gauss:cutoff_pixels=5 --process normalize --process threshold.clampminmax.nsigma:nsigma=3 "
                cmd += options.preprocess
                run(cmd)
                print("Done.")
                #shutil.copy(filename,os.path.join(tomosdir,os.path.basename(filename)))
            if options.importation == "link":
                os.symlink(filename,
                           os.path.join(tomosdir, os.path.basename(filename)))
    E2end(logid)
示例#2
0
def main():
	progname = os.path.basename(sys.argv[0])
	usage = """prog [options] files
	This program performs a variety of tasks for getting data or metadata from other programs into an EMAN2 project.

	import_movies - imports DDD movie data for a cryoEM project (--importation copy recommended)

	import_particles - will simply copy a set of per-micrograph particle files into EMAN2.1's preferred HDF format in particles/
	import_boxes - will read EMAN1 '.box' files (text files containing coordinates) into appropriate info/*json files (see --box_type)
	import_eman1 - will convert a typical EMAN1 phase-flipped start.hed/img file into an EMAN2 project (converting files, fixing CTF, splitting, ...)

	import_tomos - flag to handle imported files as subtomogams for a SPT project (see also --importation)
	import_serialem - flag to handle imported files as SerialEM mdoc files
	import_fei - flag to handle imported files as FEI tomography metadata files
	import_ucsftomo - flag to handle imported files as UCSF tomo metadata files
	import_tiltseries - imports tilt series for a tomography project (--importation copy recommended)
	"""

	parser = EMArgumentParser(usage=usage,version=EMANVERSION)

	parser.add_pos_argument(name="files",help="List the files to import here.", default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=True)",  row=0, col=0, rowspan=1, colspan=3, nosharedb=True, mode='coords,parts,tomos,eman1,movies,rawtilts,meta,tiltseries')

	parser.add_header(name="filterheader", help='Options below this label are specific to e2import', title="### e2import options ###", row=2, col=0, rowspan=1, colspan=2, mode='coords,parts')

	# Type Flags
	parser.add_argument("--import_movies",action="store_true",help="Import DDD movies",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=1, mode='movies[True]')

	parser.add_argument("--darkrefs",help="Specify a comma separated list of dark refereence stacks/images to import. Files will be placed in movierefs_raw. See --importation for additional options.",default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=True)", row=4, col=0, rowspan=1, colspan=2, mode='movies')
	parser.add_argument("--gainrefs",help="Specify a comma separated list of gain refereence stacks/images to import. Files will be placed in movierefs_raw. See --importation for additional options.",default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=True)", row=5, col=0, rowspan=1, colspan=2, mode='movies')

	#parser.add_argument("--import_rawtilts",action="store_true",help="Import tilt images",default=False, guitype='boolbox', row=4, col=2, rowspan=1, colspan=1, mode='rawtilts[True]')
	parser.add_argument("--apix",help="Specify the apix of the tiltseries you are importing. If -1 (default), the apix in the header will not be changed.",type=float,default=-1,guitype='floatbox', row=5, col=1, rowspan=1, colspan=1,mode='tiltseries[-1]')

	parser.add_argument("--import_tiltseries",action="store_true",help="Import tiltseries",default=False, guitype='boolbox', row=5, col=2, rowspan=1, colspan=1, mode='tiltseries[True]')
	parser.add_argument("--import_tomos",action="store_true",help="Import tomograms for segmentation and/or subtomogram averaging",default=False, guitype='boolbox', row=4, col=2, rowspan=1, colspan=1, mode='tomos[True]')

	#parser.add_pos_argument(name="tilt_angles",help="Specify a file containing tilt angles corresponding to the input tilt images.", default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=True)",  row=0, col=0, rowspan=1, colspan=2, nosharedb=True, mode='rawtilts')

	#parser.add_argument(name="--rawtlt",help="List the text file containing tilt angles for the tiltseries to be imported.", default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=False)",  row=3, col=0, rowspan=1, colspan=3, nosharedb=True, mode='tiltseries')
	#parser.add_argument(name="--start",help="First tilt angle. Increment determined by number of tilts. Custom tilt angles can be specified by a tilt angles text file.", default="", guitype='floatbox', row=4, col=0, rowspan=1, colspan=3, nosharedb=True, mode='tiltseries')
	#parser.add_argument(name="--stop",help="Final tilt angle. Increment determined by number of tilts. Custom tilt angles can be specified by a tilt angles text file.", default="", guitype='floatbox', row=4, col=1, rowspan=1, colspan=3, nosharedb=True, mode='tiltseries')

	# parser.add_argument("--serialem_mdoc",action="store_true",help="Import metadata from corresponding SerialEM '.mdoc' files.",default=False, guitype='boolbox', row=1, col=0, rowspan=1, colspan=3, mode='meta')
	# parser.add_argument("--fei_tomo",action="store_true",help="Import metadata from corresponding FEI tomography files.",default=False, guitype='boolbox', row=2, col=0, rowspan=1, colspan=3, mode='meta')
	# parser.add_argument("--ucsf_tomo",action="store_true",help="Import metadata from corresponding UCSF Tomo files.",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=3, mode='meta')

	parser.add_argument("--import_particles",action="store_true",help="Import particles",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=1, mode='parts[True]')
	parser.add_argument("--import_eman1",action="store_true",help="This will import a phase-flipped particle stack from EMAN1",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=1, mode='eman1[True]')

	parser.add_argument("--importation",help="Specify import mode: move, copy or link",default='copy',guitype='combobox',choicelist='["move","copy","link"]',row=9,col=0,rowspan=1,colspan=2, mode='tomos["copy"],rawtilts["copy"],movies["move"],tiltseries["copy"]',choices=["move","copy","link"])

	parser.add_argument("--invert",action="store_true",help="Invert the contrast before importing tomograms",default=False, guitype='boolbox', row=5, col=0, rowspan=1, colspan=1, mode='tomos,rawtilts,tiltseries')
	#parser.add_argument("--tomoseg_auto",action="store_true",help="Default process for tomogram segmentation, including lowpass, highpass, normalize, clampminmax.",default=True, guitype='boolbox', row=4, col=1, rowspan=1, colspan=1, mode='tomos,rawtilts,tiltseries')
	parser.add_argument("--shrink",type=int,help="Shrink tomograms before importing. Does not work while not copying.",default=1, guitype='intbox', row=6, col=0, rowspan=1, colspan=1, mode='tomos')
	#parser.add_argument("--preprocess",type=str,help="Other pre-processing operation before importing tomograms. Dose not work while not copying.",default="", guitype='strbox', row=6, col=0, rowspan=1, colspan=2, mode='tomos,rawtilts,tiltseries')

	parser.add_argument("--import_boxes",action="store_true",help="Import boxes",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=1, mode='coords[True]')
	parser.add_argument("--extension",type=str,help="Extension of the micrographs that the boxes match", default='dm3')
	parser.add_argument("--box_type",help="Type of boxes to import, normally boxes, but for tilted data use tiltedboxes, and untiltedboxes for the tilted  particle partner",default="boxes",guitype='combobox',choicelist='["boxes","coords","relion_star","tiltedboxes","untiltedboxes"]',row=3,col=1,rowspan=1,colspan=1, mode="coords['boxes']")
	parser.add_argument("--boxsize",help="Specify the boxsize for each particle.",type=int,default=256)
	parser.add_argument("--curdefocushint",action="store_true",help="Used with import_eman1, will use EMAN1 defocus as starting point",default=False, guitype='boolbox', row=5, col=0, rowspan=1, colspan=1, mode='eman1[True]')
	parser.add_argument("--curdefocusfix",action="store_true",help="Used with import_eman1, will use EMAN1 defocus unchanged (+-.001 um)",default=False, guitype='boolbox', row=5, col=1, rowspan=1, colspan=1, mode='eman1[False]')

	parser.add_argument("--threads", default=1,type=int,help="Number of threads to run in parallel on a single computer when multi-computer parallelism isn't useful",guitype='intbox', row=7, col=0, rowspan=1, colspan=1, mode='eman1[1]')
	parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
	parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)

	(options, args) = parser.parse_args()
	logid=E2init(sys.argv,options.ppid)

	# Import EMAN1
	# will read start.hed/img, split by micrograph (based on defocus), and reprocess CTF in EMAN2 style
	if options.import_eman1 :
		try:
			n=EMUtil.get_image_count(args[0])
		except:
			print("Error, couldn't read images from: ",args[0])
			sys.exit(1)

		try:
			img=EMData(args[0],0)
			ctf=img["ctf"]
		except:
			print("Error, start.hed/img must be phase-flipped to import")
			sys.exit(1)

		db=js_open_dict("info/project.json")
		db["global.apix"]=ctf.apix
		db["global.cs"]=ctf.cs
		db["global.voltage"]=ctf.voltage

		try: os.mkdir("particles")
		except: pass

		imgnum=0
		lastdf=-1.0
		for i in range(n):
			img=EMData(args[0],i)
			ctf=img["ctf"]
			img.del_attr("ctf")
			fft1=img.do_fft()
			if ctf.defocus!=lastdf :
				imgnum+=1
				if options.verbose>0: print("Defocus {:4.2f} particles{:03d}".format(ctf.defocus,imgnum))
				db=js_open_dict("info/particles{:03d}_info.json".format(imgnum))
				ctf2=EMAN2Ctf()
				ctf2.defocus=ctf.defocus
				ctf2.cs=ctf.cs
				ctf2.apix=ctf.apix
				ctf2.voltage=ctf.voltage
				ctf2.ampcont=ctf.ampcont
				ctf2.dfdiff=0
				ctf2.dfang=0
				db["ctf"]=[ctf2]
				db.close()

				flipim=fft1.copy()
				ctf2.compute_2d_complex(flipim,Ctf.CtfType.CTF_SIGN)

			lastdf=ctf.defocus

			# unflip the EMAN1 phases (hopefully accurate enough)
			fft1.mult(flipim)
			img=fft1.do_ift()
			img.write_image("particles/particles{:03d}.hdf".format(imgnum),-1)		# append particle to stack

		if options.curdefocusfix:
			flag="--curdefocusfix"
			rbysnr=" "
		elif options.curdefocushint:
			flag="--curdefocushint"
			rbysnr="--refinebysnr"
		else:
			flag=""
			rbysnr="--refinebysnr"

		# fill in the needed CTF info
		launch_childprocess("e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {} --computesf".format(flag,options.threads,ctf.voltage,ctf.cs,ctf.ampcont,ctf.apix))
		launch_childprocess("e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {}".format(flag,options.threads,ctf.voltage,ctf.cs,ctf.ampcont,ctf.apix))

		# reflip flip the phases, and make "proc" images
		launch_childprocess("e2ctf.py {} --phaseflip --allparticles --phaseflipproc filter.highpass.gauss:cutoff_freq=0.005 --phaseflipproc2 filter.lowpass.gauss:cutoff_freq=0.08 --phaseflipproc3 math.meanshrink:n=2".format(rbysnr))

		# build sets
		launch_childprocess("e2buildsets.py --allparticles --setname all")

	# Import boxes
	if options.import_boxes:
		# Check to make sure there are micrographs
		if not os.access("info", os.R_OK):
			os.mkdir("info")
		# Do imports
		# we add boxsize/2 to the coords since box files are stored with origin being the lower left side of the box, but in EMAN2 origin is in the center
		if options.box_type == 'boxes':
			micros=os.listdir("micrographs")
			for filename in args:
				boxlist = []
				fh = open(filename, 'r')
				for line in fh.readlines():
					if line[0]=="#" : continue
					fields = line.split()
					if len(fields)<4 : continue		# skip lines that don't work
					boxlist.append([float(fields[0])+old_div(float(fields[3]),2), float(fields[1])+old_div(float(fields[3]),2), 'manual'])

				js=js_open_dict(info_name(filename,nodir=True))
				js["boxes"]=boxlist
				js.close()
				if not "{}.hdf".format(base_name(filename,nodir=True)) in micros:
					print("Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(base_name(filename),base_name(filename,True)))

		elif options.box_type == 'coords':
			micros=os.listdir("micrographs")
			for filename in args:
				boxlist = []
				fh = open(filename, 'r')
				for line in fh.readlines():
					if line[0]=="#" : continue
					fields = line.split()
					if len(fields)<2 : continue		# skip lines that don't work
					boxlist.append([float(fields[0]), float(fields[1]), 'manual'])
				js_open_dict(info_name(filename,nodir=True))["boxes"]=boxlist
				if not "{}.hdf".format(base_name(filename,nodir=True)) in micros:
					print("Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(base_name(filename),base_name(filename,True)))


		elif options.box_type == 'tiltedboxes':

			for filename in args:
				boxlist = []
				fh = open(filename, 'r')
				for line in fh.readlines():
					if line[0]=="#" : continue
					fields = line.split()
					if len(fields)<4 : continue		# skip lines that don't work
					boxlist.append([float(fields[0])+old_div(float(fields[3]),2), float(fields[1])+old_div(float(fields[3]),2), 'tilted'])
				js_open_dict(info_name(filename,nodir=True))["boxes_rct"]=boxlist

		elif options.box_type == 'untiltedboxes':
			for filename in args:
				boxlist = []
				fh = open(filename, 'r')
				for line in fh.readlines():
					if line[0]=="#" : continue
					fields = line.split()
					if len(fields)<4 : continue		# skip lines that don't work
					boxlist.append([float(fields[0])+old_div(float(fields[3]),2), float(fields[1])+old_div(float(fields[3]),2), 'untilted'])
				js_open_dict(info_name(filename,nodir=True))["boxes_rct"]=boxlist

		elif options.box_type == 'relion_star':
			bs = options.boxsize
			starfs = [f for f in args if '.star' in f]
			if len(starfs) < 1:
				print("You must specify at least one .star file containing particle coordinates")
				exit(1)
			for filename in starfs:
				print(("Importing from {}.star".format(base_name(filename,nodir=True))))
				sf = StarFile(filename)
				hdr = list(sf.keys())
				if len(hdr) < 3:
					print(("Could not parse {}".format(filename)))
					continue
				mk = "rlnMicrographName"
				yk = "rlnCoordinateY"
				xk = "rlnCoordinateX"
				project_micros = os.listdir('micrographs')
				if mk not in hdr or yk not in hdr or xk not in hdr:
					possible = "{}.hdf".format(base_name(filename.replace('_autopick.star',''),nodir=True))
					if possible in project_micros:
						micros = [possible]
					else:
						print(("{} does not follow the RELION header convention for single particle data. To use this program".format(filename)))
						if mk not in hdr: print("Micrograph names should be listed under _rlnMicrographName")
						if yk not in hdr: print("Y coordinates must be listed under _rlnCoordinateY")
						if xk not in hdr: print("X coordinates must be listed under _rlnCoordinateX")
						continue
				else: micros=[i.split('/')[-1] for i in np.unique(sf[mk])]
				if len(micros) == 1:
					mg = micros[0]
					boxlist = []
					print(("Found {} boxes for {}".format(len(sf[xk]),mg)))
					for x,y in zip(sf[xk],sf[yk]):
						xc = int(x)
						yc = int(y)
						boxlist.append([xc,yc,'manual']) # should probably be 'relion' or 'from_star'
					js_open_dict(info_name(mg,nodir=True))["boxes"]=boxlist
					if not "{}.hdf".format(base_name(mg,nodir=True)) in project_micros:
						print("Warning: Imported boxes for {}.hdf, but micrographs/{}.hdf does not exist".format(base_name(filename),base_name(mg,nodir=True)))
				elif len(micros) > 1:
					for mg in project_micros:
						boxlist = []
						ptcls = []
						for i,name in enumerate(sf[mk]):
							mgname = name.split('/')[-1].split('.')[0]
							#print(mgname,hdf_name,mg)
							if mg[:-4] in mgname: ptcls.append(i)
						print(("Found {} boxes for {}".format(len(ptcls),mg)))
						for p in ptcls:
							xc = int(sf[xk][p])
							yc = int(sf[yk][p])
							boxlist.append([xc,yc,'manual'])
						js_open_dict(info_name(mg,nodir=True))["boxes"]=boxlist
						if not "{}.hdf".format(base_name(mg,nodir=True)) in project_micros:
							print("Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(base_name(mg),base_name(mg,nodir=True)))

		else : print("ERROR: Unknown box_type")

	# Import particles
	if options.import_particles:
		if not os.access("particles", os.R_OK):
			os.mkdir("particles")

		fset=set([base_name(i) for i in args])
		if len(fset)!=len(args):
			print("ERROR: You specified multiple files to import with the same base name, eg - a10/abc123.spi and a12/abc123.spi. If you have multiple images with the same \
name, you will need to modify your naming convention (perhaps by prefixing the date) before importing. If the input files are in IMAGIC format, so you have .hed and .img files \
with the same name, you should specify only the .hed files (no renaming is necessary).")
			sys.exit(1)

		for i,fsp in enumerate(args):
			E2progress(logid,old_div(float(i),len(args)))
			if EMData(fsp,0,True)["nz"]>1 :
				run("e2proc2d.py {} particles/{}.hdf --threed2twod --inplace".format(fsp,base_name(fsp)))
			else: run("e2proc2d.py {} particles/{}.hdf --inplace".format(fsp,base_name(fsp)))

	if options.gainrefs != "" or options.darkrefs != "":
		refsdir = os.path.join(".","movierefs")
		if not os.access(refsdir, os.R_OK):
			os.mkdir(refsdir)

	if options.gainrefs != "":
		for ref in options.gainrefs.split(","):
			refname=os.path.join(refsdir,os.path.basename(ref))
			if refname[-4:] == ".mrc": refname+="s"
			if not os.path.isfile(refname):
				if options.importation == "move":
					os.rename(ref,refname)
				elif options.importation == "link":
					print("Movie references must be moved or copied. Linking is not supported.")
					sys.exit(1)
				elif options.importation == "copy":
					run("e2proc2d.py {} {} ".format(ref, refname))

	if options.darkrefs != "":
		for ref in options.darkrefs.split(","):
			refname=os.path.join(refsdir,os.path.basename(ref))
			if refname[-4:] == ".mrc": refname+="s"
			if not os.path.isfile(refname):
				if options.importation == "move":
					os.rename(ref,refname)
				elif options.importation == "link":
					print("Movie references must be moved or copied. Linking is not supported.")
					sys.exit(1)
				elif options.importation == "copy":
					run("e2proc2d.py {} {} ".format(ref, refname))


	if options.import_movies:

		moviesdir = os.path.join(".","movies")
		if not os.access(moviesdir, os.R_OK):
			os.mkdir(moviesdir)

		for filename in args:
			newname=os.path.join(moviesdir,os.path.basename(filename))
			if not os.path.isfile(newname):
				if newname[-4:] == ".mrc": newname+="s"
				if options.importation == "move":
					os.rename(filename,newname)
				if options.importation == "copy":
					run("e2proc2d.py {} {} ".format(filename, newname))
				if options.importation == "link":
					os.symlink(filename,newname)
		print("Done.")

	# Import tilts
	# if options.import_rawtilts:

	# 	stdir = os.path.join(".","raw_tilts")
	# 	if not os.access(stdir, os.R_OK):
	# 		os.mkdir("tilts")

	# 	for filename in args:
	# 		newname=os.path.join(stdir,os.path.basename(filename))
	# 		if options.importation == "move":
	# 			os.rename(filename,newname)
	# 		if options.importation == "copy":
	# 			tpos=filename.rfind('.')
	# 			if tpos>0: newname=os.path.join(stdir,os.path.basename(filename[:tpos]+'.hdf'))
	# 			else: newname=os.path.join(stdir,os.path.basename(filename))
	# 			cmd="e2proc2d.py {} {} ".format(filename, newname)
	# 			if options.invert: cmd+=" --mult -1 --process normalize "
	# 			#if options.tomoseg_auto:
	# 			#	cmd+=" --process filter.lowpass.gauss:cutoff_abs=.25 --process filter.highpass.gauss:cutoff_pixels=5 --process threshold.clampminmax.nsigma:nsigma=3 "
	# 			cmd+=options.preprocess
	# 			run(cmd)
	# 			print("Done.")
	# 		if options.importation == "link":
	# 			os.symlink(filename,newname)

	# Import tilt series
	if options.import_tiltseries:
		# try:
		# 	db=js_open_dict("info/project.json")
		# 	if options.apix == -1: 
		# 		options.apix = db["global.apix"]
		# 		print("Using global apix: {}".format(db["global.apix"]))
		# except: pass

		stdir = os.path.join(".","tiltseries")
		if not os.access(stdir, os.R_OK) : os.mkdir("tiltseries")

		for filename in args:
			newname=os.path.join(stdir,os.path.basename(filename))
			if options.importation == "move":
				os.rename(filename,newname)
			if options.importation == "copy":
				if os.path.isfile(newname): os.remove(newname)
				tpos=filename.rfind('.')
				if tpos>0: newname=os.path.join(stdir,os.path.basename(filename[:tpos]+'.hdf'))
				else: newname=os.path.join(stdir,os.path.basename(filename))
				cmd="e2proc2d.py {} {} --inplace ".format(filename, newname)
				if options.invert: cmd+=" --mult -1 --process normalize "
				if options.apix != -1: cmd += " --apix {} ".format(options.apix)
				#if options.tomoseg_auto:
				#	cmd+=" --process filter.lowpass.gauss:cutoff_abs=.25 --process filter.highpass.gauss:cutoff_pixels=5 --process threshold.clampminmax.nsigma:nsigma=3 "
				#cmd+=options.preprocess
				run(cmd)
				print("Done.")
			if options.importation == "link": os.symlink(filename,newname)
			if (options.importation == "link" or options.importation == "move") and options.apix != -1:
				run("e2proc3d.py {} {} --apix {} --threed2twod".format(newname, newname, options.apix))
	
	# Import tomograms
	if options.import_tomos:
		tomosdir = os.path.join(".","tomograms")
		if not os.access(tomosdir, os.R_OK):
			os.mkdir("tomograms")
		for filename in args:
			if options.importation == "move":
				os.rename(filename,os.path.join(tomosdir,os.path.basename(filename)))
			if options.importation == "copy":
				### use hdf file as output
				if options.shrink>1:
					shrinkstr="_bin{:d}".format(options.shrink)
				else:
					shrinkstr=""
				tpos=filename.rfind('.')
				if tpos>0:
					newname=os.path.join(tomosdir,os.path.basename(filename[:tpos]+shrinkstr+'.hdf'))
				else:
					newname=os.path.join(tomosdir,os.path.basename(filename))
				hdr=EMData(filename,0,True)
				cmd="e2proc3d.py {} {} ".format(filename, newname)
				
				# shrink, and clip from the origin to give us a good box size. Done from origin so segmentation results will be positioned well for scaling
				if options.shrink>1:
					nx=good_size(old_div(hdr["nx"],options.shrink))*options.shrink
					ny=good_size(old_div(hdr["ny"],options.shrink))*options.shrink
					nz=good_size(old_div(hdr["nz"],options.shrink))*options.shrink
#					cmd+="--clip {},{},{},{},{},{} --meanshrink {:d} ".format(nx,ny,nz,nx/2,ny/2,nz/2,options.shrink)
					cmd+="--clip {},{},{} --meanshrink {:d} ".format(nx,ny,nz,options.shrink)		# no origin shift. If result is scaled up and clipped to original size this may work better
				else:
					nx=good_size(hdr["nx"])
					ny=good_size(hdr["ny"])
					nz=good_size(hdr["nz"])
#					cmd+="--clip {},{},{},{},{},{} ".format(nx,ny,nz,nx/2,ny/2,nz/2,options.shrink)
					cmd+="--clip {},{},{} ".format(nx,ny,nz,options.shrink)
					
				if options.invert:
					cmd+=" --mult -1 --process normalize "
					
				cmd+=" --process normalize "
				#if options.tomoseg_auto:
				#	cmd+=" --process filter.lowpass.gauss:cutoff_abs=.25 --process filter.highpass.gauss:cutoff_pixels=5 --process normalize --process threshold.clampminmax.nsigma:nsigma=3 "
				#cmd+=options.preprocess
				run(cmd)
				print("Done.")
			#shutil.copy(filename,os.path.join(tomosdir,os.path.basename(filename)))
			if options.importation == "link":
				os.symlink(filename,os.path.join(tomosdir,os.path.basename(filename)))

	# # Import serialEM metadata
	# if options.serialem_mdoc:
	# 	for fn in args:
	# 		mdoc = read_mdoc(fn)
	# 		# check and correct project parameters from MDOC file contents
	# 		d = js_open_dict("info/project.json")
	# 		try: d.setval("global.apix",mdoc["PixelSpacing"],deferupdate=True)
	# 		except: pass
	# 		try: d.setval("global.microscope_voltage",mdoc["Voltage"],deferupdate=True)
	# 		except: pass
	# 		d.close()
	# 		# for each referenced image, append pertinent keys/values to corresponding info.json
	# 		for z in range(mdoc["zval"]+1):
	# 			tlt = mdoc[z]["SubFramePath"].rsplit("\\")[-1]+"_RawImages"
	# 			d = js_open_dict(info_name(tlt))
	# 			for k in mdoc[z].keys():
	# 				d.setval(k,mdoc[z][k],deferupdate=True)
	# 			d.close()
	#
	# # Import FEI metadata
	# if options.fei_tomo:
	# 	print("FEI tomography metadata not yet handled by this program.")
	# 	sys.exit(1)
	#
	# # Import UCSF tomo metadata
	# if options.ucsf_tomo:
	# 	print("UCSF tomography metadata not yet handled by this program.")
	# 	sys.exit(1)

	E2end(logid)
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """prog [options] <relion STAR file>

This program will take data from a Relion project and convert it into a basic EMAN2 project and generate a .lst file. Provide 
the name of the Relion STAR file associated with the raw particle data. An eman2 subdirectory will be
created, and the images, and available metadata will be copied into the new project. CTF parameters
will be extracted from the STAR file and will be automatically processed through EMAN1's CTF procedure
(this takes most of the time the script will require).

"""
    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_argument("--lst", type=str, default='', help='.lst file.')
    parser.add_argument('--boxsize', type=int, default=0, help='box size')
    parser.add_argument(
        '--allrelion',
        type=int,
        default=0,
        help='convert all relion parameters to lst file if set to 1')
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help=
        "verbose level [0-9], higner number means higher level of verboseness")
    parser.add_argument(
        "--ny",
        type=int,
        help="Particle NY size, so no need to read image to get this value",
        default=0)
    parser.add_argument(
        "--ppid",
        type=int,
        help="Set the PID of the parent process, used for cross platform PPID",
        default=-1)

    (options, args) = parser.parse_args()

    logid = E2init(sys.argv, options.ppid)

    inputStarFileName = args[0]
    if not os.path.islink(inputStarFileName):
        inputStarFileName = os.path.abspath(inputStarFileName)
    else:
        #print "%s is a link file"%(inputStarFileName)
        inputStarFileName = os.path.abspath(os.readlink(inputStarFileName))

    topdir = os.path.dirname(inputStarFileName)

    #	try: os.mkdir("particles")
    #	except: pass

    if options.verbose > 0:
        print "Parsing STAR file %s" % (inputStarFileName)

    star = StarFile(inputStarFileName)
    #print star

    if (options.lst):
        lstfile = options.lst
    else:
        lstfile = os.path.basename(inputStarFileName)
        lstfile = lstfile[0:lstfile.rfind('.')] + '.lst'

    oldname = ""
    olddf = -1.0
    micronum = 0  # number of micrograph
    fnum = 0  # image number in new file
    params = []
    for i in xrange(len(star["rlnImageName"])):
        name = star["rlnImageName"][i].split("@")[1]
        name = os.path.join(topdir, name)
        imgnum = int(star["rlnImageName"][i].split("@")[0]) - 1

        if (star.has_key("rlnMagnificationCorrection")
            ):  #for old version relion star filr
            apix = 1e4 * float(star["rlnDetectorPixelSize"][i]) / (
                float(star["rlnMagnification"][i]) *
                float(star["rlnMagnificationCorrection"][i]))
        else:
            apix = 1e4 * float(star["rlnDetectorPixelSize"][i]) / (float(
                star["rlnMagnification"][i]))

        if name != oldname:
            #		hdr=EMData(name,0,True)
            #		nx=hdr["nx"]
            #		ny=hdr["ny"]
            nx = options.ny
            ny = options.ny
            oldname = name

        if i == 0 or star["rlnDefocusU"][i - 1] != star["rlnDefocusU"][i]:
            #this line will not run when i==0 (the 1st loop)
            if micronum > 0 and options.verbose > 1:
                print "Image {}: {} particles, df={}, size={}x{}".format(
                    micronum, fnum, ctf.defocus, nx, ny)

            micronum += 1
            fnum = 0  #when reaching a new micrograph, fnum need to start from 0 for write_image to a new good particle HDF stack
            microname = "particles/{}_{:04d}.hdf".format(
                base_name(name), micronum)

        boxsize = options.boxsize
        if boxsize <= 0:
            boxsize = ny

        # Make a "micrograph" CTF entry for each set of different defocuses to use when fitting
        # check ctffind3.f function CTF(CS,WL,WGH1,WGH2,DFMID1,DFMID2,ANGAST,THETATR, IX, IY)
        dfu = star["rlnDefocusU"][i] / 1e4
        dfv = star["rlnDefocusV"][i] / 1e4
        dfang = star["rlnDefocusAngle"][i]

        defocus = (dfu + dfv) / 2.
        dfdiff = math.fabs(dfu -
                           dfv) / 2.  # dfu can be larger or smaller than dfv
        if dfu > dfv:
            dfang = math.fmod(
                dfang + 360. + 90., 360.
            )  # largest defocus direction to smallest defocus direction
        else:
            pass  # already at smallest defocus direction

        ctf = EMAN1Ctf()
        ctf.from_dict({
            "defocus": defocus,
            "dfang": dfang,
            "dfdiff": dfdiff,
            "voltage": star["rlnVoltage"][i],
            "cs": star["rlnSphericalAberration"][i],
            "ampcont": star["rlnAmplitudeContrast"][i] * 100.0,
            "apix": apix,
            "amplitude": 1,
            "bfactor": 1e-5
        })

        # copy the image
        #	if name[-5:]==".mrcs":
        #		img=EMData(name,imgnum)		# read one slice from the MRC stack
        #	else:
        #		img=EMData(name,0,False,Region(0,0,imgnum,nx,ny,1))		# read one slice from the MRC stack

        #	img["ctf"] = ctf
        #	img['apix_x'] = apix
        #	img['apix_y'] = apix

        #	if options.verbose>2:
        #		print "Particle %d: %s" % (i, img["ctf"].to_dict())

        #	img.write_image(microname,fnum,EMUtil.ImageType.IMAGE_HDF,False) #microname is a .hdf file, False tells it to write both header and image data

        #print star["rlnAngleRot"][i], star["rlnAngleTilt"][i], star["rlnAnglePsi"][i]
        alt, az, phi, x, y = relion2EMAN2(star["rlnAngleRot"][i],
                                          star["rlnAngleTilt"][i],
                                          star["rlnAnglePsi"][i],
                                          star["rlnOriginX"][i],
                                          star["rlnOriginY"][i], boxsize)

        #all particles in eman2/particles folder are good particles, so the 1st column in .lst file should be 0,1,2,3,4,... in order
        new_file = os.path.relpath(microname, os.getcwd())
        param = [fnum, new_file, alt, az, phi, x, y, defocus, dfdiff, dfang]

        if (options.allrelion):
            for var in sorted(star.keys()):
                param.append(star[var][i])

        params.append(param)
        fnum += 1

    #generate .lst file corresponding to EMAN2 convention
    fp = open(lstfile, 'w')
    fp.write("#LST\n")
    for p in params:
        fp.write(
            "%d\t%s\teuler=%g,%g,%g\tcenter=%g,%g\tdefocus=%s\tdfdiff=%s\tdfang=%s"
            % (p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], p[9]))
        if options.allrelion:
            vars = sorted(star.keys())
            for vi, var in enumerate(vars):
                fp.write("\t%s=%s" % (var.split("rln")[-1], p[10 + vi]))
        fp.write("\n")
    fp.close()

    if options.verbose > 0:
        print "%d particles saved to %s" % (len(star["rlnImageName"]), lstfile)

    E2end(logid)
示例#4
0
文件: e2import.py 项目: cryoem/eman2
def main():
    progname = os.path.basename(sys.argv[0])
    usage = """prog [options] files
	This program performs a variety of tasks for getting data or metadata from other programs into an EMAN2 project.

	import_particles - will simply copy a set of per-micrograph particle files into EMAN2.1's preferred HDF format in particles/
	import_boxes - will read EMAN1 '.box' files (text files containing coordinates) into appropriate info/*json files (see --box_type)
	import_tomos - imports subtomogams for a SPT project (see also --importation)
	import_eman1 - will convert a typical EMAN1 phase-flipped start.hed/img file into an EMAN2 project (converting files, fixing CTF, splitting, ...)
"""

    parser = EMArgumentParser(usage=usage, version=EMANVERSION)

    parser.add_pos_argument(
        name="import_files",
        help="List the files to import here.",
        default="",
        guitype="filebox",
        browser="EMBrowserWidget(withmodal=True,multiselect=True)",
        row=0,
        col=0,
        rowspan=1,
        colspan=2,
        nosharedb=True,
        mode="coords,parts,tomos,eman1",
    )
    parser.add_header(
        name="filterheader",
        help="Options below this label are specific to e2import",
        title="### e2import options ###",
        row=1,
        col=0,
        rowspan=1,
        colspan=2,
        mode="coords,parts,tomos",
    )
    parser.add_argument(
        "--import_particles",
        action="store_true",
        help="Import particles",
        default=False,
        guitype="boolbox",
        row=2,
        col=0,
        rowspan=1,
        colspan=1,
        mode="parts[True]",
    )
    parser.add_argument(
        "--import_eman1",
        action="store_true",
        help="This will import a phase-flipped particle stack from EMAN1",
        default=False,
        guitype="boolbox",
        row=2,
        col=0,
        rowspan=1,
        colspan=1,
        mode="eman1[True]",
    )
    parser.add_argument(
        "--import_tomos",
        action="store_true",
        help="Import tomograms",
        default=False,
        guitype="boolbox",
        row=2,
        col=0,
        rowspan=1,
        colspan=1,
        mode="tomos[True]",
    )
    parser.add_argument(
        "--shrink",
        type=int,
        help="Shrink tomograms before importing. Dose not work while not copying.",
        default=1,
        guitype="intbox",
        row=3,
        col=0,
        rowspan=1,
        colspan=1,
        mode="tomos",
    )
    parser.add_argument(
        "--invert",
        action="store_true",
        help="Invert the contrast before importing tomograms",
        default=False,
        guitype="boolbox",
        row=3,
        col=1,
        rowspan=1,
        colspan=1,
        mode="tomos",
    )
    parser.add_argument(
        "--local_normalize",
        action="store_true",
        help="Apply a localized normalization before importing. Dose not work while not copying.",
        default=True,
        guitype="boolbox",
        row=3,
        col=2,
        rowspan=1,
        colspan=1,
        mode="tomos",
    )
    parser.add_argument(
        "--importation",
        help="Specify mode move, copy or link, for importing tomograms only",
        default="copy",
        guitype="combobox",
        choicelist='["move","copy","link"]',
        row=2,
        col=1,
        rowspan=1,
        colspan=1,
        mode="tomos",
    )
    parser.add_argument(
        "--preprocess",
        type=str,
        help="Other pre-processing operation before importing tomograms. Dose not work while not copying.",
        default="",
        guitype="strbox",
        row=4,
        col=0,
        rowspan=1,
        colspan=2,
        mode="tomos",
    )
    parser.add_argument(
        "--import_boxes",
        action="store_true",
        help="Import boxes",
        default=False,
        guitype="boolbox",
        row=2,
        col=0,
        rowspan=1,
        colspan=1,
        mode="coords[True]",
    )
    parser.add_argument(
        "--extension", type=str, help="Extension of the micrographs that the boxes match", default="dm3"
    )
    parser.add_argument(
        "--box_type",
        help="Type of boxes to import, normally boxes, but for tilted data use tiltedboxes, and untiltedboxes for the tilted  particle partner",
        default="boxes",
        guitype="combobox",
        choicelist='["boxes","coords","relion_star","tiltedboxes","untiltedboxes"]',
        row=2,
        col=1,
        rowspan=1,
        colspan=1,
        mode="coords['boxes']",
    )
    parser.add_argument("--boxsize", help="Specify the boxsize for each particle.", type=int, default=256)
    parser.add_argument(
        "--curdefocushint",
        action="store_true",
        help="Used with import_eman1, will use EMAN1 defocus as starting point",
        default=False,
        guitype="boolbox",
        row=4,
        col=0,
        rowspan=1,
        colspan=1,
        mode="eman1[True]",
    )
    parser.add_argument(
        "--curdefocusfix",
        action="store_true",
        help="Used with import_eman1, will use EMAN1 defocus unchanged (+-.001 um)",
        default=False,
        guitype="boolbox",
        row=4,
        col=1,
        rowspan=1,
        colspan=1,
        mode="eman1[False]",
    )
    parser.add_argument(
        "--threads",
        default=1,
        type=int,
        help="Number of threads to run in parallel on a single computer when multi-computer parallelism isn't useful",
        guitype="intbox",
        row=6,
        col=0,
        rowspan=1,
        colspan=1,
        mode="eman1[1]",
    )
    parser.add_argument(
        "--verbose",
        "-v",
        dest="verbose",
        action="store",
        metavar="n",
        type=int,
        default=0,
        help="verbose level [0-9], higner number means higher level of verboseness",
    )
    parser.add_argument(
        "--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1
    )

    (options, args) = parser.parse_args()
    logid = E2init(sys.argv, options.ppid)

    # Import EMAN1
    # will read start.hed/img, split by micrograph (based on defocus), and reprocess CTF in EMAN2 style
    if options.import_eman1:
        try:
            n = EMUtil.get_image_count(args[0])
        except:
            print "Error, couldn't read images from: ", args[0]
            sys.exit(1)

        try:
            img = EMData(args[0], 0)
            ctf = img["ctf"]
        except:
            print "Error, start.hed/img must be phase-flipped to import"
            sys.exit(1)

        db = js_open_dict("info/project.json")
        db["global.apix"] = ctf.apix
        db["global.cs"] = ctf.cs
        db["global.voltage"] = ctf.voltage

        try:
            os.mkdir("particles")
        except:
            pass

        imgnum = 0
        lastdf = -1.0
        for i in xrange(n):
            img = EMData(args[0], i)
            ctf = img["ctf"]
            img.del_attr("ctf")
            fft1 = img.do_fft()
            if ctf.defocus != lastdf:
                imgnum += 1
                if options.verbose > 0:
                    print "Defocus {:4.2f} particles{:03d}".format(ctf.defocus, imgnum)
                db = js_open_dict("info/particles{:03d}_info.json".format(imgnum))
                ctf2 = EMAN2Ctf()
                ctf2.defocus = ctf.defocus
                ctf2.cs = ctf.cs
                ctf2.apix = ctf.apix
                ctf2.voltage = ctf.voltage
                ctf2.ampcont = ctf.ampcont
                ctf2.dfdiff = 0
                ctf2.dfang = 0
                db["ctf"] = [ctf2]
                db.close()

                flipim = fft1.copy()
                ctf2.compute_2d_complex(flipim, Ctf.CtfType.CTF_SIGN)

            lastdf = ctf.defocus

            # unflip the EMAN1 phases (hopefully accurate enough)
            fft1.mult(flipim)
            img = fft1.do_ift()
            img.write_image("particles/particles{:03d}.hdf".format(imgnum), -1)  # append particle to stack

        if options.curdefocusfix:
            flag = "--curdefocusfix"
            rbysnr = " "
        elif options.curdefocushint:
            flag = "--curdefocushint"
            rbysnr = "--refinebysnr"
        else:
            flag = ""
            rbysnr = "--refinebysnr"

            # fill in the needed CTF info
        launch_childprocess(
            "e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {} --computesf".format(
                flag, options.threads, ctf.voltage, ctf.cs, ctf.ampcont, ctf.apix
            )
        )
        launch_childprocess(
            "e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {}".format(
                flag, options.threads, ctf.voltage, ctf.cs, ctf.ampcont, ctf.apix
            )
        )

        # reflip flip the phases, and make "proc" images
        launch_childprocess(
            "e2ctf.py {} --phaseflip --allparticles --phaseflipproc filter.highpass.gauss:cutoff_freq=0.005 --phaseflipproc2 filter.lowpass.gauss:cutoff_freq=0.08 --phaseflipproc3 math.meanshrink:n=2".format(
                rbysnr
            )
        )

        # build sets
        launch_childprocess("e2buildsets.py --allparticles --setname all")

        # Import boxes
    if options.import_boxes:
        # Check to make sure there are micrographs
        if not os.access("info", os.R_OK):
            os.mkdir("info")
            # Do imports
            # we add boxsize/2 to the coords since box files are stored with origin being the lower left side of the box, but in EMAN2 origin is in the center
        if options.box_type == "boxes":
            micros = os.listdir("micrographs")
            for filename in args:
                boxlist = []
                fh = open(filename, "r")
                for line in fh.readlines():
                    if line[0] == "#":
                        continue
                    fields = line.split()
                    if len(fields) < 4:
                        continue  # skip lines that don't work
                    boxlist.append(
                        [float(fields[0]) + float(fields[3]) / 2, float(fields[1]) + float(fields[3]) / 2, "manual"]
                    )

                js = js_open_dict(info_name(filename, nodir=True))
                js["boxes"] = boxlist
                js.close()
                if not "{}.hdf".format(base_name(filename, nodir=True)) in micros:
                    print "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(
                        base_name(filename), base_name(filename, True)
                    )

        elif options.box_type == "coords":
            micros = os.listdir("micrographs")
            for filename in args:
                boxlist = []
                fh = open(filename, "r")
                for line in fh.readlines():
                    if line[0] == "#":
                        continue
                    fields = line.split()
                    if len(fields) < 2:
                        continue  # skip lines that don't work
                    boxlist.append([float(fields[0]), float(fields[1]), "manual"])
                js_open_dict(info_name(filename, nodir=True))["boxes"] = boxlist
                if not "{}.hdf".format(base_name(filename, nodir=True)) in micros:
                    print "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(
                        base_name(filename), base_name(filename, True)
                    )

        elif options.box_type == "tiltedboxes":

            for filename in args:
                boxlist = []
                fh = open(filename, "r")
                for line in fh.readlines():
                    if line[0] == "#":
                        continue
                    fields = line.split()
                    if len(fields) < 4:
                        continue  # skip lines that don't work
                    boxlist.append(
                        [float(fields[0]) + float(fields[3]) / 2, float(fields[1]) + float(fields[3]) / 2, "tilted"]
                    )
                js_open_dict(info_name(filename, nodir=True))["boxes_rct"] = boxlist

        elif options.box_type == "untiltedboxes":
            for filename in args:
                boxlist = []
                fh = open(filename, "r")
                for line in fh.readlines():
                    if line[0] == "#":
                        continue
                    fields = line.split()
                    if len(fields) < 4:
                        continue  # skip lines that don't work
                    boxlist.append(
                        [float(fields[0]) + float(fields[3]) / 2, float(fields[1]) + float(fields[3]) / 2, "untilted"]
                    )
                js_open_dict(info_name(filename, nodir=True))["boxes_rct"] = boxlist

        elif options.box_type == "relion_star":
            bs = options.boxsize
            starfs = [f for f in args if ".star" in f]
            if len(starfs) < 1:
                print ("You must specify at least one .star file containing particle coordinates")
                exit(1)
            for filename in starfs:
                print ("Importing from {}.star".format(base_name(filename, nodir=True)))
                sf = StarFile(filename)
                hdr = sf.keys()
                if len(hdr) < 3:
                    print ("Could not parse {}".format(filename))
                    continue
                mk = "rlnMicrographName"
                yk = "rlnCoordinateY"
                xk = "rlnCoordinateX"
                project_micros = os.listdir("micrographs")
                if mk not in hdr or yk not in hdr or xk not in hdr:
                    possible = "{}.hdf".format(base_name(filename.replace("_autopick.star", ""), nodir=True))
                    if possible in project_micros:
                        micros = [possible]
                    else:
                        print (
                            "{} does not follow the RELION header convention for single particle data. To use this program".format(
                                filename
                            )
                        )
                        if mk not in hdr:
                            print ("Micrograph names should be listed under _rlnMicrographName")
                        if yk not in hdr:
                            print ("Y coordinates must be listed under _rlnCoordinateY")
                        if xk not in hdr:
                            print ("X coordinates must be listed under _rlnCoordinateX")
                        continue
                else:
                    micros = [i.split("/")[-1] for i in np.unique(sf[mk])]
                if len(micros) == 1:
                    mg = micros[0]
                    boxlist = []
                    print ("Found {} boxes for {}".format(len(sf[xk]), mg))
                    for x, y in zip(sf[xk], sf[yk]):
                        xc = int(x)
                        yc = int(y)
                        boxlist.append([xc, yc, "manual"])  # should probably be 'relion' or 'from_star'
                    js_open_dict(info_name(mg, nodir=True))["boxes"] = boxlist
                    if not "{}.hdf".format(base_name(mg, nodir=True)) in project_micros:
                        print "Warning: Imported boxes for {}.hdf, but micrographs/{}.hdf does not exist".format(
                            base_name(filename), base_name(mg, nodir=True)
                        )
                elif len(micros) > 1:
                    for mg in project_micros:
                        boxlist = []
                        ptcls = []
                        for i, name in enumerate(sf[mk]):
                            mgname = name.split("/")[-1].split(".")[0]
                            # print(mgname,hdf_name,mg)
                            if mg[:-4] in mgname:
                                ptcls.append(i)
                        print ("Found {} boxes for {}".format(len(ptcls), mg))
                        for p in ptcls:
                            xc = int(sf[xk][p])
                            yc = int(sf[yk][p])
                            boxlist.append([xc, yc, "manual"])
                        js_open_dict(info_name(mg, nodir=True))["boxes"] = boxlist
                        if not "{}.hdf".format(base_name(mg, nodir=True)) in project_micros:
                            print "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(
                                base_name(mg), base_name(mg, nodir=True)
                            )

        else:
            print "ERROR: Unknown box_type"

        # Import particles
    if options.import_particles:
        if not os.access("particles", os.R_OK):
            os.mkdir("particles")

        fset = set([base_name(i) for i in args])
        if len(fset) != len(args):
            print "ERROR: You specified multiple files to import with the same base name, eg - a10/abc123.spi and a12/abc123.spi. If you have multiple images with the same \
name, you will need to modify your naming convention (perhaps by prefixing the date) before importing. If the input files are in IMAGIC format, so you have .hed and .img files \
with the same name, you should specify only the .hed files (no renaming is necessary)."
            sys.exit(1)

        for i, fsp in enumerate(args):
            E2progress(logid, float(i) / len(args))
            if EMData(fsp, 0, True)["nz"] > 1:
                run("e2proc2d.py {} particles/{}.hdf --threed2twod --inplace".format(fsp, base_name(fsp)))
            else:
                run("e2proc2d.py {} particles/{}.hdf --inplace".format(fsp, base_name(fsp)))

            # Import tomograms
    if options.import_tomos:
        tomosdir = os.path.join(".", "rawtomograms")
        if not os.access(tomosdir, os.R_OK):
            os.mkdir("rawtomograms")
        for filename in args:
            if options.importation == "move":
                os.rename(filename, os.path.join(tomosdir, os.path.basename(filename)))
            if options.importation == "copy":
                ### use hdf file as output
                tpos = filename.rfind(".")
                if tpos > 0:
                    newname = os.path.join(tomosdir, os.path.basename(filename[:tpos] + ".hdf"))
                else:
                    newname = os.path.join(tomosdir, os.path.basename(filename))
                cmd = "e2proc3d.py {} {} ".format(filename, newname)
                if options.shrink > 1:
                    cmd += " --meanshrink {:d} ".format(options.shrink)
                if options.invert:
                    cmd += " --mult -1 --process normalize "
                if options.local_normalize:
                    cmd += " --process normalize.local:radius=64 "
                cmd += options.preprocess
                run(cmd)
                # shutil.copy(filename,os.path.join(tomosdir,os.path.basename(filename)))
            if options.importation == "link":
                os.symlink(filename, os.path.join(tomosdir, os.path.basename(filename)))
    E2end(logid)
示例#5
0
def main(options):
    """
    Projection subtraction program entry point.
    :param options: Command-line arguments parsed by ArgumentParser.parse_args()
    :return: Exit status
    """
    rchop = lambda x, y: x if not x.endswith(y) or len(y) == 0 else x[:-len(y)]
    options.output = rchop(options.output, ".star")
    options.suffix = rchop(options.suffix, ".mrc")
    options.suffix = rchop(options.suffix, ".mrcs")

    star = StarFile(options.input)
    npart = len(star['rlnImageName'])

    sub_dens = EMData(options.submap)

    if options.wholemap is not None:
        dens = EMData(options.wholemap)
    else:
        print "Reference map is required."
        return 1

    # Write star header for output.star.
    top_header = "\ndata_\n\nloop_\n"
    headings = star.keys()
    output_star = open("{0}.star".format(options.output), 'w')

    output_star.write(top_header)
    for i, heading in enumerate(headings):
        output_star.write("_{0} #{1}\n".format(heading, i + 1))

    if options.recenter:  # Compute difference vector between new and old mass centers.
        if options.wholemap is None:
            print "Reference map required for recentering."
            return 1

        new_dens = dens - sub_dens
        # Note the sign of the shift in coordinate frame is opposite the shift in the CoM.
        recenter = Vec3f(*dens.phase_cog()[:3]) - Vec3f(
            *new_dens.phase_cog()[:3])
    else:
        recenter = None

    pool = None
    if options.nproc > 1:  # Compute subtraction in parallel.
        pool = Pool(processes=options.nproc)
        results = pool.imap(
            lambda x: subtract(x,
                               dens,
                               sub_dens,
                               recenter=recenter,
                               no_frc=options.no_frc,
                               low_cutoff=options.low_cutoff,
                               high_cutoff=options.high_cutoff),
            particles(star),
            chunksize=min(npart / options.nproc, options.maxchunk))
    else:  # Use serial generator.
        results = (subtract(x,
                            dens,
                            sub_dens,
                            recenter=recenter,
                            no_frc=options.no_frc,
                            low_cutoff=options.low_cutoff,
                            high_cutoff=options.high_cutoff)
                   for x in particles(star))

    # Write subtraction results to .mrcs and .star files.
    i = 0
    nfile = 1
    starpath = None
    mrcs = None
    mrcs_orig = None
    for r in results:
        if i % options.maxpart == 0:
            mrcsuffix = options.suffix + "_%d" % nfile
            nfile += 1
            starpath = "{0}.mrcs".format(
                os.path.sep.join(
                    os.path.relpath(mrcsuffix,
                                    options.output).split(os.path.sep)[1:]))
            mrcs = "{0}.mrcs".format(mrcsuffix)
            mrcs_orig = "{0}_original.mrcs".format(mrcsuffix)
            if os.path.exists(mrcs):
                os.remove(mrcs)
            if os.path.exists(mrcs_orig):
                os.remove(mrcs_orig)

        r.ptcl_norm_sub.append_image(mrcs)

        if options.original:
            r.ptcl.append_image(mrcs_orig)

        if logger.getEffectiveLevel(
        ) == logging.DEBUG:  # Write additional debug output.
            ptcl_sub_img = r.ptcl.process("math.sub.optimal", {
                "ref": r.ctfproj,
                "actual": r.ctfproj_sub,
                "return_subim": True
            })
            ptcl_lowpass = r.ptcl.process("filter.lowpass.gauss", {
                "apix": 1.22,
                "cutoff_freq": 0.05
            })
            ptcl_sub_lowpass = r.ptcl_norm_sub.process("filter.lowpass.gauss",
                                                       {
                                                           "apix": 1.22,
                                                           "cutoff_freq": 0.05
                                                       })
            ptcl_sub_img.write_image("poreclass_subimg.mrcs", -1)
            ptcl_lowpass.write_image("poreclass_lowpass.mrcs", -1)
            ptcl_sub_lowpass.write_image("poreclass_sublowpass.mrcs", -1)
            r.ctfproj.write_image("poreclass_ctfproj.mrcs", -1)
            r.ctfproj_sub.write_image("poreclass_ctfprojsub.mrcs", -1)

        assert r.meta.i == i  # Assert particle order is preserved.
        star['rlnImageName'][i] = "{0:06d}@{1}".format(
            i % options.maxpart + 1, starpath)  # Set new image name.
        r.meta.update(star)  # Update StarFile with altered fields.
        line = '  '.join(str(star[key][i]) for key in headings)
        output_star.write("{0}\n".format(line))
        i += 1

    output_star.close()

    if pool is not None:
        pool.close()
        pool.join()

    return 0