def parse_relion(fp):
star_dict = StarFile(fp)
training_data_dict = {}
# Check that the STAR file contains necessary metadata for the particles
metadata_list = [
'rlnVoltage', 'rlnSphericalAberration', 'rlnAmplitudeContrast',
'rlnImagePixelSize'
]
for metadata in metadata_list:
if metadata not in star_dict:
warnings.warn('STAR file is missing metadata: %s' % metadata)
else:
training_data_dict[metadata] = star_dict[metadata][0]
# Check that the STAR file has the following necessary parameters for particles
parameter_list = ['rlnCoordinateX', 'rlnCoordinateY', 'rlnMicrographName']
for parameter in parameter_list:
if parameter in star_dict:
training_data_dict[parameter] = star_dict[parameter]
else:
raise Exception('STAR file is missing necessary parameter: %s' %
parameter)
assert (len(star_dict['rlnCoordinateX']) == len(
star_dict['rlnCoordinateY']) == len(star_dict['rlnMicrographName'])
) # sanity check
return training_data_dict
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <relion STAR file>
This program will take data from a Relion project and convert it into a basic EMAN2 project. Provide
the name of the Relion STAR file associated with the raw particle data. An eman2 subdirectory will be
created, and the images, and available metadata will be copied into the new project. CTF parameters
will be extracted from the STAR file and stored as frame CTF parameters. You will still need to run
CTF autoprocessing after importing is complete.
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
#options associated with e2refine.py
parser.add_header(name="text1", help='Important instructions', title="Use this to create an EMAN2.1 project from a Relion project:", row=0, col=0, rowspan=1, colspan=3)
parser.add_header(name="text2", help='Important instructions', title="* cd <folder with Relion STAR file>", row=1, col=0, rowspan=1, colspan=3)
parser.add_header(name="text3", help='Important instructions', title="* run e2projectmanager, and use this tool", row=2, col=0, rowspan=1, colspan=3)
parser.add_header(name="text4", help='Important instructions', title="* exit PM, cd eman2, run PM from new eman2 folder", row=3, col=0, rowspan=1, colspan=3)
parser.add_pos_argument(name="star_file",help="Select STAR file", default="", guitype='filebox', browser="EMParticlesEditTable(withmodal=True,multiselect=False)", row=6, col=0,rowspan=1, colspan=3)
parser.add_argument("--apix", default=0, type=float,help="The angstrom per pixel of the input particles.", guitype='floatbox', row=8, col=0, rowspan=1, colspan=1)
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
(options, args) = parser.parse_args()
logid=E2init(sys.argv,options.ppid)
try: os.mkdir("eman2")
except: pass
try: os.mkdir("eman2/particles")
except: pass
os.chdir("eman2") # many things need to happen with the project directory as a base
if options.verbose>0 : print("Parsing STAR file")
star=StarFile("../"+args[0])
if options.apix<=0 :
try:
options.apix=old_div(star["rlnDetectorPixelSize"][0],star["rlnMagnification"][0])*10000.0
print("Using {} A/pix from Relion file".format(options.apix))
except:
print("A/pix not specified and not found in STAR file")
sys.exit(1)
prj=js_open_dict("info/project.json")
try:
prj["global.apix"]=options.apix
prj["global.microscope_cs"]=star["rlnSphericalAberration"][0]
if prj["global.microscope_cs"]<=0.0 : prj["global.microscope_cs"]=0.001
prj["global.microscope_voltage"]=star["rlnVoltage"][0]
print("V={} Cs={}".format(prj["global.microscope_voltage"],prj["global.microscope_cs"]))
except:
print("Did not find Voltage and Cs in Relion file")
if "rlnMicrographName" in star : difkey="rlnMicrographName"
else: difkey="rlnDefocusU"
oldname=""
olddf=-1.0
micronum=0 # number of micrograph
fnum=0 # image number in new file
for i in range(len(star["rlnImageName"])):
name=star["rlnImageName"][i].split("@")[1]
imgnum=int(star["rlnImageName"][i].split("@")[0])-1
if name!=oldname:
hdr=EMData("../"+name,0,True)
nx=hdr["nx"]
ny=hdr["ny"]
oldname=name
if options.verbose>0 : print("Particle dimensions: {}x{}".format(nx,ny))
if i==0 or star[difkey][i-1]!=star[difkey][i]:
if micronum>0 and options.verbose>0 : print("Image {}: {} particles processed, df={}".format(micronum,fnum,ctf.defocus))
micronum+=1
fnum=0
microname="particles/{}_{:04d}.hdf".format(base_name(name),micronum)
jdb=js_open_dict(info_name(microname))
# Make a "micrograph" CTF entry for each set of different defocuses to use when fitting
dfu=star["rlnDefocusU"][i]
dfv=star["rlnDefocusV"][i]
dfang=star["rlnDefocusAngle"][i]
ctf=EMAN2Ctf()
ctf.from_dict({"defocus":old_div((dfu+dfv),20000.0),"dfang":dfang,"dfdiff":old_div((dfu-dfv),10000.0),"voltage":star["rlnVoltage"][i],"cs":max(star["rlnSphericalAberration"][i],0.0001),"ampcont":star["rlnAmplitudeContrast"][i]*100.0,"apix":options.apix})
jdb["ctf_frame"]=[512,ctf,(256,256),tuple(),5,1]
# copy the image
if name[-5:]==".mrcs" : img=EMData("../"+name,imgnum) # read one slice from the MRC stack
else: img=EMData("../"+name,0,False,Region(0,0,imgnum,nx,ny,1)) # read one slice from the MRC stack
img.write_image(microname,fnum)
fnum+=1
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] files
This program performs a variety of tasks for getting data or metadata from other programs into an EMAN2 project.
import_particles - will simply copy a set of per-micrograph particle files into EMAN2.1's preferred HDF format in particles/
import_boxes - will read EMAN1 '.box' files (text files containing coordinates) into appropriate info/*json files (see --box_type)
import_tomos - imports subtomogams for a SPT project (see also --importation)
import_eman1 - will convert a typical EMAN1 phase-flipped start.hed/img file into an EMAN2 project (converting files, fixing CTF, splitting, ...)
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_pos_argument(
name="import_files",
help="List the files to import here.",
default="",
guitype='filebox',
browser="EMBrowserWidget(withmodal=True,multiselect=True)",
row=0,
col=0,
rowspan=1,
colspan=2,
nosharedb=True,
mode='coords,parts,tomos,eman1')
parser.add_header(name="filterheader",
help='Options below this label are specific to e2import',
title="### e2import options ###",
row=1,
col=0,
rowspan=1,
colspan=2,
mode='coords,parts,tomos')
parser.add_argument("--import_particles",
action="store_true",
help="Import particles",
default=False,
guitype='boolbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode='parts[True]')
parser.add_argument(
"--import_eman1",
action="store_true",
help="This will import a phase-flipped particle stack from EMAN1",
default=False,
guitype='boolbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode='eman1[True]')
parser.add_argument("--import_tomos",
action="store_true",
help="Import tomograms",
default=False,
guitype='boolbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode='tomos[True]')
parser.add_argument(
"--shrink",
type=int,
help=
"Shrink tomograms before importing. Dose not work while not copying.",
default=1,
guitype='intbox',
row=3,
col=0,
rowspan=1,
colspan=1,
mode='tomos')
parser.add_argument("--invert",
action="store_true",
help="Invert the contrast before importing tomograms",
default=False,
guitype='boolbox',
row=3,
col=1,
rowspan=1,
colspan=1,
mode='tomos')
parser.add_argument(
"--tomoseg_auto",
action="store_true",
help=
"Default process for tomogram segmentation, including lowpass, highpass, normalize, clampminmax.",
default=True,
guitype='boolbox',
row=3,
col=2,
rowspan=1,
colspan=1,
mode='tomos')
parser.add_argument(
"--importation",
help="Specify mode move, copy or link, for importing tomograms only",
default='copy',
guitype='combobox',
choicelist='["move","copy","link"]',
row=2,
col=1,
rowspan=1,
colspan=1,
mode='tomos')
parser.add_argument(
"--preprocess",
type=str,
help=
"Other pre-processing operation before importing tomograms. Dose not work while not copying.",
default="",
guitype='strbox',
row=4,
col=0,
rowspan=1,
colspan=2,
mode='tomos')
parser.add_argument("--import_boxes",
action="store_true",
help="Import boxes",
default=False,
guitype='boolbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode='coords[True]')
parser.add_argument(
"--extension",
type=str,
help="Extension of the micrographs that the boxes match",
default='dm3')
parser.add_argument(
"--box_type",
help=
"Type of boxes to import, normally boxes, but for tilted data use tiltedboxes, and untiltedboxes for the tilted particle partner",
default="boxes",
guitype='combobox',
choicelist=
'["boxes","coords","relion_star","tiltedboxes","untiltedboxes"]',
row=2,
col=1,
rowspan=1,
colspan=1,
mode="coords['boxes']")
parser.add_argument("--boxsize",
help="Specify the boxsize for each particle.",
type=int,
default=256)
parser.add_argument(
"--curdefocushint",
action="store_true",
help="Used with import_eman1, will use EMAN1 defocus as starting point",
default=False,
guitype='boolbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode='eman1[True]')
parser.add_argument(
"--curdefocusfix",
action="store_true",
help=
"Used with import_eman1, will use EMAN1 defocus unchanged (+-.001 um)",
default=False,
guitype='boolbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode='eman1[False]')
parser.add_argument(
"--threads",
default=1,
type=int,
help=
"Number of threads to run in parallel on a single computer when multi-computer parallelism isn't useful",
guitype='intbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode='eman1[1]')
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
# Import EMAN1
# will read start.hed/img, split by micrograph (based on defocus), and reprocess CTF in EMAN2 style
if options.import_eman1:
try:
n = EMUtil.get_image_count(args[0])
except:
print("Error, couldn't read images from: ", args[0])
sys.exit(1)
try:
img = EMData(args[0], 0)
ctf = img["ctf"]
except:
print("Error, start.hed/img must be phase-flipped to import")
sys.exit(1)
db = js_open_dict("info/project.json")
db["global.apix"] = ctf.apix
db["global.cs"] = ctf.cs
db["global.voltage"] = ctf.voltage
try:
os.mkdir("particles")
except:
pass
imgnum = 0
lastdf = -1.0
for i in xrange(n):
img = EMData(args[0], i)
ctf = img["ctf"]
img.del_attr("ctf")
fft1 = img.do_fft()
if ctf.defocus != lastdf:
imgnum += 1
if options.verbose > 0:
print("Defocus {:4.2f} particles{:03d}".format(
ctf.defocus, imgnum))
db = js_open_dict(
"info/particles{:03d}_info.json".format(imgnum))
ctf2 = EMAN2Ctf()
ctf2.defocus = ctf.defocus
ctf2.cs = ctf.cs
ctf2.apix = ctf.apix
ctf2.voltage = ctf.voltage
ctf2.ampcont = ctf.ampcont
ctf2.dfdiff = 0
ctf2.dfang = 0
db["ctf"] = [ctf2]
db.close()
flipim = fft1.copy()
ctf2.compute_2d_complex(flipim, Ctf.CtfType.CTF_SIGN)
lastdf = ctf.defocus
# unflip the EMAN1 phases (hopefully accurate enough)
fft1.mult(flipim)
img = fft1.do_ift()
img.write_image("particles/particles{:03d}.hdf".format(imgnum),
-1) # append particle to stack
if options.curdefocusfix:
flag = "--curdefocusfix"
rbysnr = " "
elif options.curdefocushint:
flag = "--curdefocushint"
rbysnr = "--refinebysnr"
else:
flag = ""
rbysnr = "--refinebysnr"
# fill in the needed CTF info
launch_childprocess(
"e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {} --computesf"
.format(flag, options.threads, ctf.voltage, ctf.cs, ctf.ampcont,
ctf.apix))
launch_childprocess(
"e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {}"
.format(flag, options.threads, ctf.voltage, ctf.cs, ctf.ampcont,
ctf.apix))
# reflip flip the phases, and make "proc" images
launch_childprocess(
"e2ctf.py {} --phaseflip --allparticles --phaseflipproc filter.highpass.gauss:cutoff_freq=0.005 --phaseflipproc2 filter.lowpass.gauss:cutoff_freq=0.08 --phaseflipproc3 math.meanshrink:n=2"
.format(rbysnr))
# build sets
launch_childprocess("e2buildsets.py --allparticles --setname all")
# Import boxes
if options.import_boxes:
# Check to make sure there are micrographs
if not os.access("info", os.R_OK):
os.mkdir("info")
# Do imports
# we add boxsize/2 to the coords since box files are stored with origin being the lower left side of the box, but in EMAN2 origin is in the center
if options.box_type == 'boxes':
micros = os.listdir("micrographs")
for filename in args:
boxlist = []
fh = open(filename, 'r')
for line in fh.readlines():
if line[0] == "#": continue
fields = line.split()
if len(fields) < 4: continue # skip lines that don't work
boxlist.append([
float(fields[0]) + float(fields[3]) / 2,
float(fields[1]) + float(fields[3]) / 2, 'manual'
])
js = js_open_dict(info_name(filename, nodir=True))
js["boxes"] = boxlist
js.close()
if not "{}.hdf".format(base_name(filename,
nodir=True)) in micros:
print(
"Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist"
.format(base_name(filename), base_name(filename,
True)))
elif options.box_type == 'coords':
micros = os.listdir("micrographs")
for filename in args:
boxlist = []
fh = open(filename, 'r')
for line in fh.readlines():
if line[0] == "#": continue
fields = line.split()
if len(fields) < 2: continue # skip lines that don't work
boxlist.append(
[float(fields[0]),
float(fields[1]), 'manual'])
js_open_dict(info_name(filename,
nodir=True))["boxes"] = boxlist
if not "{}.hdf".format(base_name(filename,
nodir=True)) in micros:
print(
"Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist"
.format(base_name(filename), base_name(filename,
True)))
elif options.box_type == 'tiltedboxes':
for filename in args:
boxlist = []
fh = open(filename, 'r')
for line in fh.readlines():
if line[0] == "#": continue
fields = line.split()
if len(fields) < 4: continue # skip lines that don't work
boxlist.append([
float(fields[0]) + float(fields[3]) / 2,
float(fields[1]) + float(fields[3]) / 2, 'tilted'
])
js_open_dict(info_name(filename,
nodir=True))["boxes_rct"] = boxlist
elif options.box_type == 'untiltedboxes':
for filename in args:
boxlist = []
fh = open(filename, 'r')
for line in fh.readlines():
if line[0] == "#": continue
fields = line.split()
if len(fields) < 4: continue # skip lines that don't work
boxlist.append([
float(fields[0]) + float(fields[3]) / 2,
float(fields[1]) + float(fields[3]) / 2, 'untilted'
])
js_open_dict(info_name(filename,
nodir=True))["boxes_rct"] = boxlist
elif options.box_type == 'relion_star':
bs = options.boxsize
starfs = [f for f in args if '.star' in f]
if len(starfs) < 1:
print(
"You must specify at least one .star file containing particle coordinates"
)
exit(1)
for filename in starfs:
print(("Importing from {}.star".format(
base_name(filename, nodir=True))))
sf = StarFile(filename)
hdr = sf.keys()
if len(hdr) < 3:
print(("Could not parse {}".format(filename)))
continue
mk = "rlnMicrographName"
yk = "rlnCoordinateY"
xk = "rlnCoordinateX"
project_micros = os.listdir('micrographs')
if mk not in hdr or yk not in hdr or xk not in hdr:
possible = "{}.hdf".format(
base_name(filename.replace('_autopick.star', ''),
nodir=True))
if possible in project_micros:
micros = [possible]
else:
print((
"{} does not follow the RELION header convention for single particle data. To use this program"
.format(filename)))
if mk not in hdr:
print(
"Micrograph names should be listed under _rlnMicrographName"
)
if yk not in hdr:
print(
"Y coordinates must be listed under _rlnCoordinateY"
)
if xk not in hdr:
print(
"X coordinates must be listed under _rlnCoordinateX"
)
continue
else:
micros = [i.split('/')[-1] for i in np.unique(sf[mk])]
if len(micros) == 1:
mg = micros[0]
boxlist = []
print(("Found {} boxes for {}".format(len(sf[xk]), mg)))
for x, y in zip(sf[xk], sf[yk]):
xc = int(x)
yc = int(y)
boxlist.append([
xc, yc, 'manual'
]) # should probably be 'relion' or 'from_star'
js_open_dict(info_name(mg, nodir=True))["boxes"] = boxlist
if not "{}.hdf".format(base_name(
mg, nodir=True)) in project_micros:
print(
"Warning: Imported boxes for {}.hdf, but micrographs/{}.hdf does not exist"
.format(base_name(filename),
base_name(mg, nodir=True)))
elif len(micros) > 1:
for mg in project_micros:
boxlist = []
ptcls = []
for i, name in enumerate(sf[mk]):
mgname = name.split('/')[-1].split('.')[0]
#print(mgname,hdf_name,mg)
if mg[:-4] in mgname: ptcls.append(i)
print(("Found {} boxes for {}".format(len(ptcls), mg)))
for p in ptcls:
xc = int(sf[xk][p])
yc = int(sf[yk][p])
boxlist.append([xc, yc, 'manual'])
js_open_dict(info_name(mg,
nodir=True))["boxes"] = boxlist
if not "{}.hdf".format(base_name(
mg, nodir=True)) in project_micros:
print(
"Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist"
.format(base_name(mg), base_name(mg,
nodir=True)))
else:
print("ERROR: Unknown box_type")
# Import particles
if options.import_particles:
if not os.access("particles", os.R_OK):
os.mkdir("particles")
fset = set([base_name(i) for i in args])
if len(fset) != len(args):
print(
"ERROR: You specified multiple files to import with the same base name, eg - a10/abc123.spi and a12/abc123.spi. If you have multiple images with the same \
name, you will need to modify your naming convention (perhaps by prefixing the date) before importing. If the input files are in IMAGIC format, so you have .hed and .img files \
with the same name, you should specify only the .hed files (no renaming is necessary)."
)
sys.exit(1)
for i, fsp in enumerate(args):
E2progress(logid, float(i) / len(args))
if EMData(fsp, 0, True)["nz"] > 1:
run("e2proc2d.py {} particles/{}.hdf --threed2twod --inplace".
format(fsp, base_name(fsp)))
else:
run("e2proc2d.py {} particles/{}.hdf --inplace".format(
fsp, base_name(fsp)))
# Import tomograms
if options.import_tomos:
tomosdir = os.path.join(".", "rawtomograms")
if not os.access(tomosdir, os.R_OK):
os.mkdir("rawtomograms")
for filename in args:
if options.importation == "move":
os.rename(filename,
os.path.join(tomosdir, os.path.basename(filename)))
if options.importation == "copy":
### use hdf file as output
if options.shrink > 1:
shrinkstr = "_bin{:d}".format(options.shrink)
else:
shrinkstr = ""
tpos = filename.rfind('.')
if tpos > 0:
newname = os.path.join(
tomosdir,
os.path.basename(filename[:tpos] + shrinkstr + '.hdf'))
else:
newname = os.path.join(tomosdir,
os.path.basename(filename))
cmd = "e2proc3d.py {} {} ".format(filename, newname)
if options.shrink > 1:
cmd += " --meanshrink {:d} ".format(options.shrink)
if options.invert:
cmd += " --mult -1 --process normalize "
if options.tomoseg_auto:
cmd += " --process filter.lowpass.gauss:cutoff_abs=.25 --process filter.highpass.gauss:cutoff_pixels=5 --process normalize --process threshold.clampminmax.nsigma:nsigma=3 "
cmd += options.preprocess
run(cmd)
print("Done.")
#shutil.copy(filename,os.path.join(tomosdir,os.path.basename(filename)))
if options.importation == "link":
os.symlink(filename,
os.path.join(tomosdir, os.path.basename(filename)))
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] files
This program performs a variety of tasks for getting data or metadata from other programs into an EMAN2 project.
import_movies - imports DDD movie data for a cryoEM project (--importation copy recommended)
import_particles - will simply copy a set of per-micrograph particle files into EMAN2.1's preferred HDF format in particles/
import_boxes - will read EMAN1 '.box' files (text files containing coordinates) into appropriate info/*json files (see --box_type)
import_eman1 - will convert a typical EMAN1 phase-flipped start.hed/img file into an EMAN2 project (converting files, fixing CTF, splitting, ...)
import_tomos - flag to handle imported files as subtomogams for a SPT project (see also --importation)
import_serialem - flag to handle imported files as SerialEM mdoc files
import_fei - flag to handle imported files as FEI tomography metadata files
import_ucsftomo - flag to handle imported files as UCSF tomo metadata files
import_tiltseries - imports tilt series for a tomography project (--importation copy recommended)
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_pos_argument(name="files",help="List the files to import here.", default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=True)", row=0, col=0, rowspan=1, colspan=3, nosharedb=True, mode='coords,parts,tomos,eman1,movies,rawtilts,meta,tiltseries')
parser.add_header(name="filterheader", help='Options below this label are specific to e2import', title="### e2import options ###", row=2, col=0, rowspan=1, colspan=2, mode='coords,parts')
# Type Flags
parser.add_argument("--import_movies",action="store_true",help="Import DDD movies",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=1, mode='movies[True]')
parser.add_argument("--darkrefs",help="Specify a comma separated list of dark refereence stacks/images to import. Files will be placed in movierefs_raw. See --importation for additional options.",default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=True)", row=4, col=0, rowspan=1, colspan=2, mode='movies')
parser.add_argument("--gainrefs",help="Specify a comma separated list of gain refereence stacks/images to import. Files will be placed in movierefs_raw. See --importation for additional options.",default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=True)", row=5, col=0, rowspan=1, colspan=2, mode='movies')
#parser.add_argument("--import_rawtilts",action="store_true",help="Import tilt images",default=False, guitype='boolbox', row=4, col=2, rowspan=1, colspan=1, mode='rawtilts[True]')
parser.add_argument("--apix",help="Specify the apix of the tiltseries you are importing. If -1 (default), the apix in the header will not be changed.",type=float,default=-1,guitype='floatbox', row=5, col=1, rowspan=1, colspan=1,mode='tiltseries[-1]')
parser.add_argument("--import_tiltseries",action="store_true",help="Import tiltseries",default=False, guitype='boolbox', row=5, col=2, rowspan=1, colspan=1, mode='tiltseries[True]')
parser.add_argument("--import_tomos",action="store_true",help="Import tomograms for segmentation and/or subtomogram averaging",default=False, guitype='boolbox', row=4, col=2, rowspan=1, colspan=1, mode='tomos[True]')
#parser.add_pos_argument(name="tilt_angles",help="Specify a file containing tilt angles corresponding to the input tilt images.", default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=True)", row=0, col=0, rowspan=1, colspan=2, nosharedb=True, mode='rawtilts')
#parser.add_argument(name="--rawtlt",help="List the text file containing tilt angles for the tiltseries to be imported.", default="", guitype='filebox', browser="EMBrowserWidget(withmodal=True,multiselect=False)", row=3, col=0, rowspan=1, colspan=3, nosharedb=True, mode='tiltseries')
#parser.add_argument(name="--start",help="First tilt angle. Increment determined by number of tilts. Custom tilt angles can be specified by a tilt angles text file.", default="", guitype='floatbox', row=4, col=0, rowspan=1, colspan=3, nosharedb=True, mode='tiltseries')
#parser.add_argument(name="--stop",help="Final tilt angle. Increment determined by number of tilts. Custom tilt angles can be specified by a tilt angles text file.", default="", guitype='floatbox', row=4, col=1, rowspan=1, colspan=3, nosharedb=True, mode='tiltseries')
# parser.add_argument("--serialem_mdoc",action="store_true",help="Import metadata from corresponding SerialEM '.mdoc' files.",default=False, guitype='boolbox', row=1, col=0, rowspan=1, colspan=3, mode='meta')
# parser.add_argument("--fei_tomo",action="store_true",help="Import metadata from corresponding FEI tomography files.",default=False, guitype='boolbox', row=2, col=0, rowspan=1, colspan=3, mode='meta')
# parser.add_argument("--ucsf_tomo",action="store_true",help="Import metadata from corresponding UCSF Tomo files.",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=3, mode='meta')
parser.add_argument("--import_particles",action="store_true",help="Import particles",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=1, mode='parts[True]')
parser.add_argument("--import_eman1",action="store_true",help="This will import a phase-flipped particle stack from EMAN1",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=1, mode='eman1[True]')
parser.add_argument("--importation",help="Specify import mode: move, copy or link",default='copy',guitype='combobox',choicelist='["move","copy","link"]',row=9,col=0,rowspan=1,colspan=2, mode='tomos["copy"],rawtilts["copy"],movies["move"],tiltseries["copy"]',choices=["move","copy","link"])
parser.add_argument("--invert",action="store_true",help="Invert the contrast before importing tomograms",default=False, guitype='boolbox', row=5, col=0, rowspan=1, colspan=1, mode='tomos,rawtilts,tiltseries')
#parser.add_argument("--tomoseg_auto",action="store_true",help="Default process for tomogram segmentation, including lowpass, highpass, normalize, clampminmax.",default=True, guitype='boolbox', row=4, col=1, rowspan=1, colspan=1, mode='tomos,rawtilts,tiltseries')
parser.add_argument("--shrink",type=int,help="Shrink tomograms before importing. Does not work while not copying.",default=1, guitype='intbox', row=6, col=0, rowspan=1, colspan=1, mode='tomos')
#parser.add_argument("--preprocess",type=str,help="Other pre-processing operation before importing tomograms. Dose not work while not copying.",default="", guitype='strbox', row=6, col=0, rowspan=1, colspan=2, mode='tomos,rawtilts,tiltseries')
parser.add_argument("--import_boxes",action="store_true",help="Import boxes",default=False, guitype='boolbox', row=3, col=0, rowspan=1, colspan=1, mode='coords[True]')
parser.add_argument("--extension",type=str,help="Extension of the micrographs that the boxes match", default='dm3')
parser.add_argument("--box_type",help="Type of boxes to import, normally boxes, but for tilted data use tiltedboxes, and untiltedboxes for the tilted particle partner",default="boxes",guitype='combobox',choicelist='["boxes","coords","relion_star","tiltedboxes","untiltedboxes"]',row=3,col=1,rowspan=1,colspan=1, mode="coords['boxes']")
parser.add_argument("--boxsize",help="Specify the boxsize for each particle.",type=int,default=256)
parser.add_argument("--curdefocushint",action="store_true",help="Used with import_eman1, will use EMAN1 defocus as starting point",default=False, guitype='boolbox', row=5, col=0, rowspan=1, colspan=1, mode='eman1[True]')
parser.add_argument("--curdefocusfix",action="store_true",help="Used with import_eman1, will use EMAN1 defocus unchanged (+-.001 um)",default=False, guitype='boolbox', row=5, col=1, rowspan=1, colspan=1, mode='eman1[False]')
parser.add_argument("--threads", default=1,type=int,help="Number of threads to run in parallel on a single computer when multi-computer parallelism isn't useful",guitype='intbox', row=7, col=0, rowspan=1, colspan=1, mode='eman1[1]')
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
(options, args) = parser.parse_args()
logid=E2init(sys.argv,options.ppid)
# Import EMAN1
# will read start.hed/img, split by micrograph (based on defocus), and reprocess CTF in EMAN2 style
if options.import_eman1 :
try:
n=EMUtil.get_image_count(args[0])
except:
print("Error, couldn't read images from: ",args[0])
sys.exit(1)
try:
img=EMData(args[0],0)
ctf=img["ctf"]
except:
print("Error, start.hed/img must be phase-flipped to import")
sys.exit(1)
db=js_open_dict("info/project.json")
db["global.apix"]=ctf.apix
db["global.cs"]=ctf.cs
db["global.voltage"]=ctf.voltage
try: os.mkdir("particles")
except: pass
imgnum=0
lastdf=-1.0
for i in range(n):
img=EMData(args[0],i)
ctf=img["ctf"]
img.del_attr("ctf")
fft1=img.do_fft()
if ctf.defocus!=lastdf :
imgnum+=1
if options.verbose>0: print("Defocus {:4.2f} particles{:03d}".format(ctf.defocus,imgnum))
db=js_open_dict("info/particles{:03d}_info.json".format(imgnum))
ctf2=EMAN2Ctf()
ctf2.defocus=ctf.defocus
ctf2.cs=ctf.cs
ctf2.apix=ctf.apix
ctf2.voltage=ctf.voltage
ctf2.ampcont=ctf.ampcont
ctf2.dfdiff=0
ctf2.dfang=0
db["ctf"]=[ctf2]
db.close()
flipim=fft1.copy()
ctf2.compute_2d_complex(flipim,Ctf.CtfType.CTF_SIGN)
lastdf=ctf.defocus
# unflip the EMAN1 phases (hopefully accurate enough)
fft1.mult(flipim)
img=fft1.do_ift()
img.write_image("particles/particles{:03d}.hdf".format(imgnum),-1) # append particle to stack
if options.curdefocusfix:
flag="--curdefocusfix"
rbysnr=" "
elif options.curdefocushint:
flag="--curdefocushint"
rbysnr="--refinebysnr"
else:
flag=""
rbysnr="--refinebysnr"
# fill in the needed CTF info
launch_childprocess("e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {} --computesf".format(flag,options.threads,ctf.voltage,ctf.cs,ctf.ampcont,ctf.apix))
launch_childprocess("e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {}".format(flag,options.threads,ctf.voltage,ctf.cs,ctf.ampcont,ctf.apix))
# reflip flip the phases, and make "proc" images
launch_childprocess("e2ctf.py {} --phaseflip --allparticles --phaseflipproc filter.highpass.gauss:cutoff_freq=0.005 --phaseflipproc2 filter.lowpass.gauss:cutoff_freq=0.08 --phaseflipproc3 math.meanshrink:n=2".format(rbysnr))
# build sets
launch_childprocess("e2buildsets.py --allparticles --setname all")
# Import boxes
if options.import_boxes:
# Check to make sure there are micrographs
if not os.access("info", os.R_OK):
os.mkdir("info")
# Do imports
# we add boxsize/2 to the coords since box files are stored with origin being the lower left side of the box, but in EMAN2 origin is in the center
if options.box_type == 'boxes':
micros=os.listdir("micrographs")
for filename in args:
boxlist = []
fh = open(filename, 'r')
for line in fh.readlines():
if line[0]=="#" : continue
fields = line.split()
if len(fields)<4 : continue # skip lines that don't work
boxlist.append([float(fields[0])+old_div(float(fields[3]),2), float(fields[1])+old_div(float(fields[3]),2), 'manual'])
js=js_open_dict(info_name(filename,nodir=True))
js["boxes"]=boxlist
js.close()
if not "{}.hdf".format(base_name(filename,nodir=True)) in micros:
print("Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(base_name(filename),base_name(filename,True)))
elif options.box_type == 'coords':
micros=os.listdir("micrographs")
for filename in args:
boxlist = []
fh = open(filename, 'r')
for line in fh.readlines():
if line[0]=="#" : continue
fields = line.split()
if len(fields)<2 : continue # skip lines that don't work
boxlist.append([float(fields[0]), float(fields[1]), 'manual'])
js_open_dict(info_name(filename,nodir=True))["boxes"]=boxlist
if not "{}.hdf".format(base_name(filename,nodir=True)) in micros:
print("Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(base_name(filename),base_name(filename,True)))
elif options.box_type == 'tiltedboxes':
for filename in args:
boxlist = []
fh = open(filename, 'r')
for line in fh.readlines():
if line[0]=="#" : continue
fields = line.split()
if len(fields)<4 : continue # skip lines that don't work
boxlist.append([float(fields[0])+old_div(float(fields[3]),2), float(fields[1])+old_div(float(fields[3]),2), 'tilted'])
js_open_dict(info_name(filename,nodir=True))["boxes_rct"]=boxlist
elif options.box_type == 'untiltedboxes':
for filename in args:
boxlist = []
fh = open(filename, 'r')
for line in fh.readlines():
if line[0]=="#" : continue
fields = line.split()
if len(fields)<4 : continue # skip lines that don't work
boxlist.append([float(fields[0])+old_div(float(fields[3]),2), float(fields[1])+old_div(float(fields[3]),2), 'untilted'])
js_open_dict(info_name(filename,nodir=True))["boxes_rct"]=boxlist
elif options.box_type == 'relion_star':
bs = options.boxsize
starfs = [f for f in args if '.star' in f]
if len(starfs) < 1:
print("You must specify at least one .star file containing particle coordinates")
exit(1)
for filename in starfs:
print(("Importing from {}.star".format(base_name(filename,nodir=True))))
sf = StarFile(filename)
hdr = list(sf.keys())
if len(hdr) < 3:
print(("Could not parse {}".format(filename)))
continue
mk = "rlnMicrographName"
yk = "rlnCoordinateY"
xk = "rlnCoordinateX"
project_micros = os.listdir('micrographs')
if mk not in hdr or yk not in hdr or xk not in hdr:
possible = "{}.hdf".format(base_name(filename.replace('_autopick.star',''),nodir=True))
if possible in project_micros:
micros = [possible]
else:
print(("{} does not follow the RELION header convention for single particle data. To use this program".format(filename)))
if mk not in hdr: print("Micrograph names should be listed under _rlnMicrographName")
if yk not in hdr: print("Y coordinates must be listed under _rlnCoordinateY")
if xk not in hdr: print("X coordinates must be listed under _rlnCoordinateX")
continue
else: micros=[i.split('/')[-1] for i in np.unique(sf[mk])]
if len(micros) == 1:
mg = micros[0]
boxlist = []
print(("Found {} boxes for {}".format(len(sf[xk]),mg)))
for x,y in zip(sf[xk],sf[yk]):
xc = int(x)
yc = int(y)
boxlist.append([xc,yc,'manual']) # should probably be 'relion' or 'from_star'
js_open_dict(info_name(mg,nodir=True))["boxes"]=boxlist
if not "{}.hdf".format(base_name(mg,nodir=True)) in project_micros:
print("Warning: Imported boxes for {}.hdf, but micrographs/{}.hdf does not exist".format(base_name(filename),base_name(mg,nodir=True)))
elif len(micros) > 1:
for mg in project_micros:
boxlist = []
ptcls = []
for i,name in enumerate(sf[mk]):
mgname = name.split('/')[-1].split('.')[0]
#print(mgname,hdf_name,mg)
if mg[:-4] in mgname: ptcls.append(i)
print(("Found {} boxes for {}".format(len(ptcls),mg)))
for p in ptcls:
xc = int(sf[xk][p])
yc = int(sf[yk][p])
boxlist.append([xc,yc,'manual'])
js_open_dict(info_name(mg,nodir=True))["boxes"]=boxlist
if not "{}.hdf".format(base_name(mg,nodir=True)) in project_micros:
print("Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(base_name(mg),base_name(mg,nodir=True)))
else : print("ERROR: Unknown box_type")
# Import particles
if options.import_particles:
if not os.access("particles", os.R_OK):
os.mkdir("particles")
fset=set([base_name(i) for i in args])
if len(fset)!=len(args):
print("ERROR: You specified multiple files to import with the same base name, eg - a10/abc123.spi and a12/abc123.spi. If you have multiple images with the same \
name, you will need to modify your naming convention (perhaps by prefixing the date) before importing. If the input files are in IMAGIC format, so you have .hed and .img files \
with the same name, you should specify only the .hed files (no renaming is necessary).")
sys.exit(1)
for i,fsp in enumerate(args):
E2progress(logid,old_div(float(i),len(args)))
if EMData(fsp,0,True)["nz"]>1 :
run("e2proc2d.py {} particles/{}.hdf --threed2twod --inplace".format(fsp,base_name(fsp)))
else: run("e2proc2d.py {} particles/{}.hdf --inplace".format(fsp,base_name(fsp)))
if options.gainrefs != "" or options.darkrefs != "":
refsdir = os.path.join(".","movierefs")
if not os.access(refsdir, os.R_OK):
os.mkdir(refsdir)
if options.gainrefs != "":
for ref in options.gainrefs.split(","):
refname=os.path.join(refsdir,os.path.basename(ref))
if refname[-4:] == ".mrc": refname+="s"
if not os.path.isfile(refname):
if options.importation == "move":
os.rename(ref,refname)
elif options.importation == "link":
print("Movie references must be moved or copied. Linking is not supported.")
sys.exit(1)
elif options.importation == "copy":
run("e2proc2d.py {} {} ".format(ref, refname))
if options.darkrefs != "":
for ref in options.darkrefs.split(","):
refname=os.path.join(refsdir,os.path.basename(ref))
if refname[-4:] == ".mrc": refname+="s"
if not os.path.isfile(refname):
if options.importation == "move":
os.rename(ref,refname)
elif options.importation == "link":
print("Movie references must be moved or copied. Linking is not supported.")
sys.exit(1)
elif options.importation == "copy":
run("e2proc2d.py {} {} ".format(ref, refname))
if options.import_movies:
moviesdir = os.path.join(".","movies")
if not os.access(moviesdir, os.R_OK):
os.mkdir(moviesdir)
for filename in args:
newname=os.path.join(moviesdir,os.path.basename(filename))
if not os.path.isfile(newname):
if newname[-4:] == ".mrc": newname+="s"
if options.importation == "move":
os.rename(filename,newname)
if options.importation == "copy":
run("e2proc2d.py {} {} ".format(filename, newname))
if options.importation == "link":
os.symlink(filename,newname)
print("Done.")
# Import tilts
# if options.import_rawtilts:
# stdir = os.path.join(".","raw_tilts")
# if not os.access(stdir, os.R_OK):
# os.mkdir("tilts")
# for filename in args:
# newname=os.path.join(stdir,os.path.basename(filename))
# if options.importation == "move":
# os.rename(filename,newname)
# if options.importation == "copy":
# tpos=filename.rfind('.')
# if tpos>0: newname=os.path.join(stdir,os.path.basename(filename[:tpos]+'.hdf'))
# else: newname=os.path.join(stdir,os.path.basename(filename))
# cmd="e2proc2d.py {} {} ".format(filename, newname)
# if options.invert: cmd+=" --mult -1 --process normalize "
# #if options.tomoseg_auto:
# # cmd+=" --process filter.lowpass.gauss:cutoff_abs=.25 --process filter.highpass.gauss:cutoff_pixels=5 --process threshold.clampminmax.nsigma:nsigma=3 "
# cmd+=options.preprocess
# run(cmd)
# print("Done.")
# if options.importation == "link":
# os.symlink(filename,newname)
# Import tilt series
if options.import_tiltseries:
# try:
# db=js_open_dict("info/project.json")
# if options.apix == -1:
# options.apix = db["global.apix"]
# print("Using global apix: {}".format(db["global.apix"]))
# except: pass
stdir = os.path.join(".","tiltseries")
if not os.access(stdir, os.R_OK) : os.mkdir("tiltseries")
for filename in args:
newname=os.path.join(stdir,os.path.basename(filename))
if options.importation == "move":
os.rename(filename,newname)
if options.importation == "copy":
if os.path.isfile(newname): os.remove(newname)
tpos=filename.rfind('.')
if tpos>0: newname=os.path.join(stdir,os.path.basename(filename[:tpos]+'.hdf'))
else: newname=os.path.join(stdir,os.path.basename(filename))
cmd="e2proc2d.py {} {} --inplace ".format(filename, newname)
if options.invert: cmd+=" --mult -1 --process normalize "
if options.apix != -1: cmd += " --apix {} ".format(options.apix)
#if options.tomoseg_auto:
# cmd+=" --process filter.lowpass.gauss:cutoff_abs=.25 --process filter.highpass.gauss:cutoff_pixels=5 --process threshold.clampminmax.nsigma:nsigma=3 "
#cmd+=options.preprocess
run(cmd)
print("Done.")
if options.importation == "link": os.symlink(filename,newname)
if (options.importation == "link" or options.importation == "move") and options.apix != -1:
run("e2proc3d.py {} {} --apix {} --threed2twod".format(newname, newname, options.apix))
# Import tomograms
if options.import_tomos:
tomosdir = os.path.join(".","tomograms")
if not os.access(tomosdir, os.R_OK):
os.mkdir("tomograms")
for filename in args:
if options.importation == "move":
os.rename(filename,os.path.join(tomosdir,os.path.basename(filename)))
if options.importation == "copy":
### use hdf file as output
if options.shrink>1:
shrinkstr="_bin{:d}".format(options.shrink)
else:
shrinkstr=""
tpos=filename.rfind('.')
if tpos>0:
newname=os.path.join(tomosdir,os.path.basename(filename[:tpos]+shrinkstr+'.hdf'))
else:
newname=os.path.join(tomosdir,os.path.basename(filename))
hdr=EMData(filename,0,True)
cmd="e2proc3d.py {} {} ".format(filename, newname)
# shrink, and clip from the origin to give us a good box size. Done from origin so segmentation results will be positioned well for scaling
if options.shrink>1:
nx=good_size(old_div(hdr["nx"],options.shrink))*options.shrink
ny=good_size(old_div(hdr["ny"],options.shrink))*options.shrink
nz=good_size(old_div(hdr["nz"],options.shrink))*options.shrink
# cmd+="--clip {},{},{},{},{},{} --meanshrink {:d} ".format(nx,ny,nz,nx/2,ny/2,nz/2,options.shrink)
cmd+="--clip {},{},{} --meanshrink {:d} ".format(nx,ny,nz,options.shrink) # no origin shift. If result is scaled up and clipped to original size this may work better
else:
nx=good_size(hdr["nx"])
ny=good_size(hdr["ny"])
nz=good_size(hdr["nz"])
# cmd+="--clip {},{},{},{},{},{} ".format(nx,ny,nz,nx/2,ny/2,nz/2,options.shrink)
cmd+="--clip {},{},{} ".format(nx,ny,nz,options.shrink)
if options.invert:
cmd+=" --mult -1 --process normalize "
cmd+=" --process normalize "
#if options.tomoseg_auto:
# cmd+=" --process filter.lowpass.gauss:cutoff_abs=.25 --process filter.highpass.gauss:cutoff_pixels=5 --process normalize --process threshold.clampminmax.nsigma:nsigma=3 "
#cmd+=options.preprocess
run(cmd)
print("Done.")
#shutil.copy(filename,os.path.join(tomosdir,os.path.basename(filename)))
if options.importation == "link":
os.symlink(filename,os.path.join(tomosdir,os.path.basename(filename)))
# # Import serialEM metadata
# if options.serialem_mdoc:
# for fn in args:
# mdoc = read_mdoc(fn)
# # check and correct project parameters from MDOC file contents
# d = js_open_dict("info/project.json")
# try: d.setval("global.apix",mdoc["PixelSpacing"],deferupdate=True)
# except: pass
# try: d.setval("global.microscope_voltage",mdoc["Voltage"],deferupdate=True)
# except: pass
# d.close()
# # for each referenced image, append pertinent keys/values to corresponding info.json
# for z in range(mdoc["zval"]+1):
# tlt = mdoc[z]["SubFramePath"].rsplit("\\")[-1]+"_RawImages"
# d = js_open_dict(info_name(tlt))
# for k in mdoc[z].keys():
# d.setval(k,mdoc[z][k],deferupdate=True)
# d.close()
#
# # Import FEI metadata
# if options.fei_tomo:
# print("FEI tomography metadata not yet handled by this program.")
# sys.exit(1)
#
# # Import UCSF tomo metadata
# if options.ucsf_tomo:
# print("UCSF tomography metadata not yet handled by this program.")
# sys.exit(1)
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
For more information on ctffind3 please see: Mindell, JA, Grigorieff N. 2003. Accurate determination of local defocus and specimen tilt in electron microscopy. J Struct Biol. 142:334-47.
"""
parser = EMArgumentParser(usage=usage,version=EMANVERSION)
parser.add_header(name="ctffind3header", help='Options below this label are specific to e2ctffind3util.py', title="### e2ctffind3util options (requires that ctffind3 be installed)###", row=0, col=0, rowspan=1, colspan=2, mode="import,run")
#options associated with e2ctffind3.py
parser.add_argument("--apix", default=0.0, type=float,help="The angstrom per pixel of the micrographs", guitype='floatbox', row=3, col=0, rowspan=1, colspan=1, mode="import,run")
parser.add_argument("--cs", default=0.0, type=float,help="The spherical aberration of the microscope", guitype='floatbox', row=3, col=1, rowspan=1, colspan=1, mode="import,run")
parser.add_argument("--voltage", default=0.0, type=float,help="The voltage (in kV) of the microscope", guitype='floatbox', row=4, col=0, rowspan=1, colspan=1, mode="import,run")
parser.add_argument("--ac", default=0.0, type=float,help="The amplitude contrast of the micrographs", guitype='floatbox', row=4, col=1, rowspan=1, colspan=1, mode="import,run")
parser.add_argument("--verbose", "-v", dest="verbose", action="store", metavar="n", type=int, default=0, help="verbose level [0-9], higner number means higher level of verboseness", guitype='intbox', row=5, col=0, rowspan=1, colspan=1, mode="import,run")
parser.add_argument("--importrelionstar", default=False, action="store_true",help="Import CTFFIND3 data in Relion STAR format. Put STAR files in 'ctffind3' folder", guitype='boolbox', row=6, col=0, rowspan=1, colspan=1, mode='import[True]')
parser.add_argument("--importctffind3", default=False, action="store_true",help="Import ctffind3 data?", guitype='boolbox', row=6, col=0, rowspan=1, colspan=1, mode='import[True]')
parser.add_argument("--importctffind4", default=False, action="store_true",help="Import ctffind4 data?", guitype='boolbox', row=6, col=1, rowspan=1, colspan=1, mode='import[False]')
parser.add_argument("--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID",default=-1)
parser.add_pos_argument(name="micrographs",help="List the micrographs to run ctffind3 on here.", default="", guitype='filebox', browser="EMRawDataTable(withmodal=True,multiselect=True)", filecheck=False, row=1, col=0,rowspan=1, colspan=2, mode='run')
parser.add_argument("--allmicrographs", default=False, action="store_true",help="Run Ctffind3 on all micrographs in the micrographs directory?", guitype='boolbox', row=2, col=0, rowspan=1, colspan=1, mode="run")
parser.add_argument("--maxres", default=0.0, type=float,help="The highest resolution to be fitted (Angstroms)", guitype='floatbox', row=6, col=0, rowspan=1, colspan=1, mode="run")
parser.add_argument("--minres", default=0.0, type=float,help="The lowest resolution to be fitted (Angstroms)", guitype='floatbox', row=6, col=1, rowspan=1, colspan=1, mode="run")
parser.add_argument("--defocusmin", default=0.0, type=float,help="The starting defocus value for grid search (microns)", guitype='floatbox', row=7, col=0, rowspan=1, colspan=1, mode="run")
parser.add_argument("--defocusmax", default=0.0, type=float,help="The end defocus value for grid search (microns)", guitype='floatbox', row=7, col=1, rowspan=1, colspan=1, mode="run")
parser.add_argument("--defocusstep", default=0.0, type=float,help="The step width for grid search (microns)", guitype='floatbox', row=8, col=0, rowspan=1, colspan=1, mode="run")
parser.add_argument("--runctffind3", default=False, action="store_true",help="Run ctffind3 on the selected micrographs?", guitype='boolbox', row=9, col=0, rowspan=1, colspan=1, mode='run[True]')
parser.add_argument("--runctffind4", default=False, action="store_true",help="Run ctffind4 on the selected micrographs?", guitype='boolbox', row=9, col=1, rowspan=1, colspan=1, mode='run[False]')
parser.add_argument("--windowsize", default=0, type=int,help="The amplitude contrast of the micrographs", guitype='intbox', row=8, col=1, rowspan=1, colspan=1, mode="run")
(options, args) = parser.parse_args()
logid=E2init(sys.argv,options.ppid)
if options.importrelionstar :
fls=[i for i in os.listdir("ctffind3") if i[-5:]==".star"]
if len(fls)==0:
print "please create a folder called 'ctffind3' and put .star files in it before running this option"
sys.exit(1)
for f in fls:
star=StarFile("ctffind3/"+f)
if not star.has_key("rlnMicrographName") :
print "No rlnMicrographName in ",f
continue
print f
for i,imfsp in enumerate(star["rlnMicrographName"]):
im=info_name(imfsp) # should work regardless of extension
jdb=js_open_dict(im)
dfu=star["rlnDefocusU"][i]
dfv=star["rlnDefocusV"][i]
dfang=star["rlnDefocusAngle"][i]
ctf=EMAN2Ctf()
ctf.from_dict({"defocus":(dfu+dfv)/20000.0,"dfang":dfang,"dfdiff":(dfu-dfv)/10000.0,"voltage":star["rlnVoltage"][i],"cs":star["rlnSphericalAberration"][i],"ampcont":star["rlnAmplitudeContrast"][i]*100.0,"apix":options.apix})
jdb["ctf_frame"]=[512,ctf,(256,256),tuple(),5,1]
js_close_dict(im)
launch_childprocess("e2ctf.py --voltage {} --cs {} --ac {} --apix {} --allparticles --autofit --curdefocusfix --astigmatism".format(ctf.voltage,ctf.cs,ctf.ampcont,ctf.apix))
print "All done"
sys.exit(0)
if options.apix<= 0 :
print "Angstrom per pixel (apix) must be specified!"
exit(-1)
if options.cs<= 0 :
print "Spherical Aberration (cs) must be specified!"
exit(-2)
if options.voltage<= 0 :
print "Voltage must be specified!"
exit(-3)
if options.ac<= 0 :
print "Amplitude Contrast must be specified!"
exit(-4)
if options.allmicrographs:
args = []
for item in os.listdir("micrographs"):
try:
if item.split(".")[1] == "mrc" or item.split(".")[1] == "hdf":
args.append("micrographs/" + item)
except:
pass
if options.runctffind4:
version = "ctffind4"
run_ctffind(options.apix, args, options.cs, options.voltage, options.ac, options.windowsize, options.minres, options.maxres, options.defocusmin*10000, options.defocusmax*10000, options.defocusstep*10000, options.verbose,version)
elif options.runctffind3:
version = "ctffind3"
run_ctffind(options.apix, args, options.cs, options.voltage, options.ac, options.windowsize, options.minres, options.maxres, options.defocusmin*10000, options.defocusmax*10000, options.defocusstep*10000, options.verbose, version)
if options.importctffind4 or options.runctffind4:
version = "ctffind4"
else:
version = "ctffind3"
import_ctf(options.voltage, options.cs, options.ac, options.apix, options.verbose, version)
print "e2ctffind3util.py complete!"
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options]
For more information on ctffind3 please see: Mindell, JA, Grigorieff N. 2003. Accurate determination of local defocus and specimen tilt in electron microscopy. J Struct Biol. 142:334-47.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_header(
name="ctffind3header",
help='Options below this label are specific to e2ctffind3util.py',
title=
"### e2ctffind3util options (requires that ctffind3 be installed)###",
row=0,
col=0,
rowspan=1,
colspan=2,
mode="import,run")
#options associated with e2ctffind3.py
parser.add_argument("--apix",
default=0.0,
type=float,
help="The angstrom per pixel of the micrographs",
guitype='floatbox',
row=3,
col=0,
rowspan=1,
colspan=1,
mode="import,run")
parser.add_argument("--cs",
default=0.0,
type=float,
help="The spherical aberration of the microscope",
guitype='floatbox',
row=3,
col=1,
rowspan=1,
colspan=1,
mode="import,run")
parser.add_argument("--voltage",
default=0.0,
type=float,
help="The voltage (in kV) of the microscope",
guitype='floatbox',
row=4,
col=0,
rowspan=1,
colspan=1,
mode="import,run")
parser.add_argument("--ac",
default=0.0,
type=float,
help="The amplitude contrast of the micrographs",
guitype='floatbox',
row=4,
col=1,
rowspan=1,
colspan=1,
mode="import,run")
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness",
guitype='intbox',
row=5,
col=0,
rowspan=1,
colspan=1,
mode="import,run")
parser.add_argument(
"--importrelionstar",
default=False,
action="store_true",
help=
"Import CTFFIND3 data in Relion STAR format. Put STAR files in 'ctffind3' folder",
guitype='boolbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode='import[True]')
parser.add_argument("--importctffind3",
default=False,
action="store_true",
help="Import ctffind3 data?",
guitype='boolbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode='import[True]')
parser.add_argument("--importctffind4",
default=False,
action="store_true",
help="Import ctffind4 data?",
guitype='boolbox',
row=6,
col=1,
rowspan=1,
colspan=1,
mode='import[False]')
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
parser.add_pos_argument(
name="micrographs",
help="List the micrographs to run ctffind3 on here.",
default="",
guitype='filebox',
browser="EMRawDataTable(withmodal=True,multiselect=True)",
filecheck=False,
row=1,
col=0,
rowspan=1,
colspan=2,
mode='run')
parser.add_argument(
"--allmicrographs",
default=False,
action="store_true",
help="Run Ctffind3 on all micrographs in the micrographs directory?",
guitype='boolbox',
row=2,
col=0,
rowspan=1,
colspan=1,
mode="run")
parser.add_argument("--maxres",
default=0.0,
type=float,
help="The highest resolution to be fitted (Angstroms)",
guitype='floatbox',
row=6,
col=0,
rowspan=1,
colspan=1,
mode="run")
parser.add_argument("--minres",
default=0.0,
type=float,
help="The lowest resolution to be fitted (Angstroms)",
guitype='floatbox',
row=6,
col=1,
rowspan=1,
colspan=1,
mode="run")
parser.add_argument(
"--defocusmin",
default=0.0,
type=float,
help="The starting defocus value for grid search (microns)",
guitype='floatbox',
row=7,
col=0,
rowspan=1,
colspan=1,
mode="run")
parser.add_argument("--defocusmax",
default=0.0,
type=float,
help="The end defocus value for grid search (microns)",
guitype='floatbox',
row=7,
col=1,
rowspan=1,
colspan=1,
mode="run")
parser.add_argument("--defocusstep",
default=0.0,
type=float,
help="The step width for grid search (microns)",
guitype='floatbox',
row=8,
col=0,
rowspan=1,
colspan=1,
mode="run")
parser.add_argument("--runctffind3",
default=False,
action="store_true",
help="Run ctffind3 on the selected micrographs?",
guitype='boolbox',
row=9,
col=0,
rowspan=1,
colspan=1,
mode='run[True]')
parser.add_argument("--runctffind4",
default=False,
action="store_true",
help="Run ctffind4 on the selected micrographs?",
guitype='boolbox',
row=9,
col=1,
rowspan=1,
colspan=1,
mode='run[False]')
parser.add_argument("--windowsize",
default=0,
type=int,
help="The amplitude contrast of the micrographs",
guitype='intbox',
row=8,
col=1,
rowspan=1,
colspan=1,
mode="run")
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
if options.importrelionstar:
fls = [i for i in os.listdir("ctffind3") if i[-5:] == ".star"]
if len(fls) == 0:
print "please create a folder called 'ctffind3' and put .star files in it before running this option"
sys.exit(1)
for f in fls:
star = StarFile("ctffind3/" + f)
if not star.has_key("rlnMicrographName"):
print "No rlnMicrographName in ", f
continue
print f
for i, imfsp in enumerate(star["rlnMicrographName"]):
im = info_name(imfsp) # should work regardless of extension
jdb = js_open_dict(im)
dfu = star["rlnDefocusU"][i]
dfv = star["rlnDefocusV"][i]
dfang = star["rlnDefocusAngle"][i]
ctf = EMAN2Ctf()
ctf.from_dict({
"defocus": (dfu + dfv) / 20000.0,
"dfang": dfang,
"dfdiff": (dfu - dfv) / 10000.0,
"voltage": star["rlnVoltage"][i],
"cs": star["rlnSphericalAberration"][i],
"ampcont": star["rlnAmplitudeContrast"][i] * 100.0,
"apix": options.apix
})
jdb["ctf_frame"] = [512, ctf, (256, 256), tuple(), 5, 1]
js_close_dict(im)
launch_childprocess(
"e2ctf.py --voltage {} --cs {} --ac {} --apix {} --allparticles --autofit --curdefocusfix --astigmatism"
.format(ctf.voltage, ctf.cs, ctf.ampcont, ctf.apix))
print "All done"
sys.exit(0)
if options.apix <= 0:
print "Angstrom per pixel (apix) must be specified!"
exit(-1)
if options.cs <= 0:
print "Spherical Aberration (cs) must be specified!"
exit(-2)
if options.voltage <= 0:
print "Voltage must be specified!"
exit(-3)
if options.ac <= 0:
print "Amplitude Contrast must be specified!"
exit(-4)
if options.allmicrographs:
args = []
for item in os.listdir("micrographs"):
try:
if item.split(".")[1] == "mrc" or item.split(".")[1] == "hdf":
args.append("micrographs/" + item)
except:
pass
if options.runctffind4:
version = "ctffind4"
run_ctffind(options.apix, args, options.cs, options.voltage,
options.ac, options.windowsize, options.minres,
options.maxres, options.defocusmin * 10000,
options.defocusmax * 10000, options.defocusstep * 10000,
options.verbose, version)
elif options.runctffind3:
version = "ctffind3"
run_ctffind(options.apix, args, options.cs, options.voltage,
options.ac, options.windowsize, options.minres,
options.maxres, options.defocusmin * 10000,
options.defocusmax * 10000, options.defocusstep * 10000,
options.verbose, version)
if options.importctffind4 or options.runctffind4:
version = "ctffind4"
else:
version = "ctffind3"
import_ctf(options.voltage, options.cs, options.ac, options.apix,
options.verbose, version)
print "e2ctffind3util.py complete!"
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] files
This program performs a variety of tasks for getting data or metadata from other programs into an EMAN2 project.
import_particles - will simply copy a set of per-micrograph particle files into EMAN2.1's preferred HDF format in particles/
import_boxes - will read EMAN1 '.box' files (text files containing coordinates) into appropriate info/*json files (see --box_type)
import_tomos - imports subtomogams for a SPT project (see also --importation)
import_eman1 - will convert a typical EMAN1 phase-flipped start.hed/img file into an EMAN2 project (converting files, fixing CTF, splitting, ...)
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_pos_argument(
name="import_files",
help="List the files to import here.",
default="",
guitype="filebox",
browser="EMBrowserWidget(withmodal=True,multiselect=True)",
row=0,
col=0,
rowspan=1,
colspan=2,
nosharedb=True,
mode="coords,parts,tomos,eman1",
)
parser.add_header(
name="filterheader",
help="Options below this label are specific to e2import",
title="### e2import options ###",
row=1,
col=0,
rowspan=1,
colspan=2,
mode="coords,parts,tomos",
)
parser.add_argument(
"--import_particles",
action="store_true",
help="Import particles",
default=False,
guitype="boolbox",
row=2,
col=0,
rowspan=1,
colspan=1,
mode="parts[True]",
)
parser.add_argument(
"--import_eman1",
action="store_true",
help="This will import a phase-flipped particle stack from EMAN1",
default=False,
guitype="boolbox",
row=2,
col=0,
rowspan=1,
colspan=1,
mode="eman1[True]",
)
parser.add_argument(
"--import_tomos",
action="store_true",
help="Import tomograms",
default=False,
guitype="boolbox",
row=2,
col=0,
rowspan=1,
colspan=1,
mode="tomos[True]",
)
parser.add_argument(
"--shrink",
type=int,
help="Shrink tomograms before importing. Dose not work while not copying.",
default=1,
guitype="intbox",
row=3,
col=0,
rowspan=1,
colspan=1,
mode="tomos",
)
parser.add_argument(
"--invert",
action="store_true",
help="Invert the contrast before importing tomograms",
default=False,
guitype="boolbox",
row=3,
col=1,
rowspan=1,
colspan=1,
mode="tomos",
)
parser.add_argument(
"--local_normalize",
action="store_true",
help="Apply a localized normalization before importing. Dose not work while not copying.",
default=True,
guitype="boolbox",
row=3,
col=2,
rowspan=1,
colspan=1,
mode="tomos",
)
parser.add_argument(
"--importation",
help="Specify mode move, copy or link, for importing tomograms only",
default="copy",
guitype="combobox",
choicelist='["move","copy","link"]',
row=2,
col=1,
rowspan=1,
colspan=1,
mode="tomos",
)
parser.add_argument(
"--preprocess",
type=str,
help="Other pre-processing operation before importing tomograms. Dose not work while not copying.",
default="",
guitype="strbox",
row=4,
col=0,
rowspan=1,
colspan=2,
mode="tomos",
)
parser.add_argument(
"--import_boxes",
action="store_true",
help="Import boxes",
default=False,
guitype="boolbox",
row=2,
col=0,
rowspan=1,
colspan=1,
mode="coords[True]",
)
parser.add_argument(
"--extension", type=str, help="Extension of the micrographs that the boxes match", default="dm3"
)
parser.add_argument(
"--box_type",
help="Type of boxes to import, normally boxes, but for tilted data use tiltedboxes, and untiltedboxes for the tilted particle partner",
default="boxes",
guitype="combobox",
choicelist='["boxes","coords","relion_star","tiltedboxes","untiltedboxes"]',
row=2,
col=1,
rowspan=1,
colspan=1,
mode="coords['boxes']",
)
parser.add_argument("--boxsize", help="Specify the boxsize for each particle.", type=int, default=256)
parser.add_argument(
"--curdefocushint",
action="store_true",
help="Used with import_eman1, will use EMAN1 defocus as starting point",
default=False,
guitype="boolbox",
row=4,
col=0,
rowspan=1,
colspan=1,
mode="eman1[True]",
)
parser.add_argument(
"--curdefocusfix",
action="store_true",
help="Used with import_eman1, will use EMAN1 defocus unchanged (+-.001 um)",
default=False,
guitype="boolbox",
row=4,
col=1,
rowspan=1,
colspan=1,
mode="eman1[False]",
)
parser.add_argument(
"--threads",
default=1,
type=int,
help="Number of threads to run in parallel on a single computer when multi-computer parallelism isn't useful",
guitype="intbox",
row=6,
col=0,
rowspan=1,
colspan=1,
mode="eman1[1]",
)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help="verbose level [0-9], higner number means higher level of verboseness",
)
parser.add_argument(
"--ppid", type=int, help="Set the PID of the parent process, used for cross platform PPID", default=-1
)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
# Import EMAN1
# will read start.hed/img, split by micrograph (based on defocus), and reprocess CTF in EMAN2 style
if options.import_eman1:
try:
n = EMUtil.get_image_count(args[0])
except:
print "Error, couldn't read images from: ", args[0]
sys.exit(1)
try:
img = EMData(args[0], 0)
ctf = img["ctf"]
except:
print "Error, start.hed/img must be phase-flipped to import"
sys.exit(1)
db = js_open_dict("info/project.json")
db["global.apix"] = ctf.apix
db["global.cs"] = ctf.cs
db["global.voltage"] = ctf.voltage
try:
os.mkdir("particles")
except:
pass
imgnum = 0
lastdf = -1.0
for i in xrange(n):
img = EMData(args[0], i)
ctf = img["ctf"]
img.del_attr("ctf")
fft1 = img.do_fft()
if ctf.defocus != lastdf:
imgnum += 1
if options.verbose > 0:
print "Defocus {:4.2f} particles{:03d}".format(ctf.defocus, imgnum)
db = js_open_dict("info/particles{:03d}_info.json".format(imgnum))
ctf2 = EMAN2Ctf()
ctf2.defocus = ctf.defocus
ctf2.cs = ctf.cs
ctf2.apix = ctf.apix
ctf2.voltage = ctf.voltage
ctf2.ampcont = ctf.ampcont
ctf2.dfdiff = 0
ctf2.dfang = 0
db["ctf"] = [ctf2]
db.close()
flipim = fft1.copy()
ctf2.compute_2d_complex(flipim, Ctf.CtfType.CTF_SIGN)
lastdf = ctf.defocus
# unflip the EMAN1 phases (hopefully accurate enough)
fft1.mult(flipim)
img = fft1.do_ift()
img.write_image("particles/particles{:03d}.hdf".format(imgnum), -1) # append particle to stack
if options.curdefocusfix:
flag = "--curdefocusfix"
rbysnr = " "
elif options.curdefocushint:
flag = "--curdefocushint"
rbysnr = "--refinebysnr"
else:
flag = ""
rbysnr = "--refinebysnr"
# fill in the needed CTF info
launch_childprocess(
"e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {} --computesf".format(
flag, options.threads, ctf.voltage, ctf.cs, ctf.ampcont, ctf.apix
)
)
launch_childprocess(
"e2ctf.py --autofit {} --allparticles --threads {} --voltage {} --cs {} --ac {} --apix {}".format(
flag, options.threads, ctf.voltage, ctf.cs, ctf.ampcont, ctf.apix
)
)
# reflip flip the phases, and make "proc" images
launch_childprocess(
"e2ctf.py {} --phaseflip --allparticles --phaseflipproc filter.highpass.gauss:cutoff_freq=0.005 --phaseflipproc2 filter.lowpass.gauss:cutoff_freq=0.08 --phaseflipproc3 math.meanshrink:n=2".format(
rbysnr
)
)
# build sets
launch_childprocess("e2buildsets.py --allparticles --setname all")
# Import boxes
if options.import_boxes:
# Check to make sure there are micrographs
if not os.access("info", os.R_OK):
os.mkdir("info")
# Do imports
# we add boxsize/2 to the coords since box files are stored with origin being the lower left side of the box, but in EMAN2 origin is in the center
if options.box_type == "boxes":
micros = os.listdir("micrographs")
for filename in args:
boxlist = []
fh = open(filename, "r")
for line in fh.readlines():
if line[0] == "#":
continue
fields = line.split()
if len(fields) < 4:
continue # skip lines that don't work
boxlist.append(
[float(fields[0]) + float(fields[3]) / 2, float(fields[1]) + float(fields[3]) / 2, "manual"]
)
js = js_open_dict(info_name(filename, nodir=True))
js["boxes"] = boxlist
js.close()
if not "{}.hdf".format(base_name(filename, nodir=True)) in micros:
print "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(
base_name(filename), base_name(filename, True)
)
elif options.box_type == "coords":
micros = os.listdir("micrographs")
for filename in args:
boxlist = []
fh = open(filename, "r")
for line in fh.readlines():
if line[0] == "#":
continue
fields = line.split()
if len(fields) < 2:
continue # skip lines that don't work
boxlist.append([float(fields[0]), float(fields[1]), "manual"])
js_open_dict(info_name(filename, nodir=True))["boxes"] = boxlist
if not "{}.hdf".format(base_name(filename, nodir=True)) in micros:
print "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(
base_name(filename), base_name(filename, True)
)
elif options.box_type == "tiltedboxes":
for filename in args:
boxlist = []
fh = open(filename, "r")
for line in fh.readlines():
if line[0] == "#":
continue
fields = line.split()
if len(fields) < 4:
continue # skip lines that don't work
boxlist.append(
[float(fields[0]) + float(fields[3]) / 2, float(fields[1]) + float(fields[3]) / 2, "tilted"]
)
js_open_dict(info_name(filename, nodir=True))["boxes_rct"] = boxlist
elif options.box_type == "untiltedboxes":
for filename in args:
boxlist = []
fh = open(filename, "r")
for line in fh.readlines():
if line[0] == "#":
continue
fields = line.split()
if len(fields) < 4:
continue # skip lines that don't work
boxlist.append(
[float(fields[0]) + float(fields[3]) / 2, float(fields[1]) + float(fields[3]) / 2, "untilted"]
)
js_open_dict(info_name(filename, nodir=True))["boxes_rct"] = boxlist
elif options.box_type == "relion_star":
bs = options.boxsize
starfs = [f for f in args if ".star" in f]
if len(starfs) < 1:
print ("You must specify at least one .star file containing particle coordinates")
exit(1)
for filename in starfs:
print ("Importing from {}.star".format(base_name(filename, nodir=True)))
sf = StarFile(filename)
hdr = sf.keys()
if len(hdr) < 3:
print ("Could not parse {}".format(filename))
continue
mk = "rlnMicrographName"
yk = "rlnCoordinateY"
xk = "rlnCoordinateX"
project_micros = os.listdir("micrographs")
if mk not in hdr or yk not in hdr or xk not in hdr:
possible = "{}.hdf".format(base_name(filename.replace("_autopick.star", ""), nodir=True))
if possible in project_micros:
micros = [possible]
else:
print (
"{} does not follow the RELION header convention for single particle data. To use this program".format(
filename
)
)
if mk not in hdr:
print ("Micrograph names should be listed under _rlnMicrographName")
if yk not in hdr:
print ("Y coordinates must be listed under _rlnCoordinateY")
if xk not in hdr:
print ("X coordinates must be listed under _rlnCoordinateX")
continue
else:
micros = [i.split("/")[-1] for i in np.unique(sf[mk])]
if len(micros) == 1:
mg = micros[0]
boxlist = []
print ("Found {} boxes for {}".format(len(sf[xk]), mg))
for x, y in zip(sf[xk], sf[yk]):
xc = int(x)
yc = int(y)
boxlist.append([xc, yc, "manual"]) # should probably be 'relion' or 'from_star'
js_open_dict(info_name(mg, nodir=True))["boxes"] = boxlist
if not "{}.hdf".format(base_name(mg, nodir=True)) in project_micros:
print "Warning: Imported boxes for {}.hdf, but micrographs/{}.hdf does not exist".format(
base_name(filename), base_name(mg, nodir=True)
)
elif len(micros) > 1:
for mg in project_micros:
boxlist = []
ptcls = []
for i, name in enumerate(sf[mk]):
mgname = name.split("/")[-1].split(".")[0]
# print(mgname,hdf_name,mg)
if mg[:-4] in mgname:
ptcls.append(i)
print ("Found {} boxes for {}".format(len(ptcls), mg))
for p in ptcls:
xc = int(sf[xk][p])
yc = int(sf[yk][p])
boxlist.append([xc, yc, "manual"])
js_open_dict(info_name(mg, nodir=True))["boxes"] = boxlist
if not "{}.hdf".format(base_name(mg, nodir=True)) in project_micros:
print "Warning: Imported boxes for {}, but micrographs/{}.hdf does not exist".format(
base_name(mg), base_name(mg, nodir=True)
)
else:
print "ERROR: Unknown box_type"
# Import particles
if options.import_particles:
if not os.access("particles", os.R_OK):
os.mkdir("particles")
fset = set([base_name(i) for i in args])
if len(fset) != len(args):
print "ERROR: You specified multiple files to import with the same base name, eg - a10/abc123.spi and a12/abc123.spi. If you have multiple images with the same \
name, you will need to modify your naming convention (perhaps by prefixing the date) before importing. If the input files are in IMAGIC format, so you have .hed and .img files \
with the same name, you should specify only the .hed files (no renaming is necessary)."
sys.exit(1)
for i, fsp in enumerate(args):
E2progress(logid, float(i) / len(args))
if EMData(fsp, 0, True)["nz"] > 1:
run("e2proc2d.py {} particles/{}.hdf --threed2twod --inplace".format(fsp, base_name(fsp)))
else:
run("e2proc2d.py {} particles/{}.hdf --inplace".format(fsp, base_name(fsp)))
# Import tomograms
if options.import_tomos:
tomosdir = os.path.join(".", "rawtomograms")
if not os.access(tomosdir, os.R_OK):
os.mkdir("rawtomograms")
for filename in args:
if options.importation == "move":
os.rename(filename, os.path.join(tomosdir, os.path.basename(filename)))
if options.importation == "copy":
### use hdf file as output
tpos = filename.rfind(".")
if tpos > 0:
newname = os.path.join(tomosdir, os.path.basename(filename[:tpos] + ".hdf"))
else:
newname = os.path.join(tomosdir, os.path.basename(filename))
cmd = "e2proc3d.py {} {} ".format(filename, newname)
if options.shrink > 1:
cmd += " --meanshrink {:d} ".format(options.shrink)
if options.invert:
cmd += " --mult -1 --process normalize "
if options.local_normalize:
cmd += " --process normalize.local:radius=64 "
cmd += options.preprocess
run(cmd)
# shutil.copy(filename,os.path.join(tomosdir,os.path.basename(filename)))
if options.importation == "link":
os.symlink(filename, os.path.join(tomosdir, os.path.basename(filename)))
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <relion STAR file>
This program will take data from a Relion project and convert it into a basic EMAN2 project and generate a .lst file. Provide
the name of the Relion STAR file associated with the raw particle data. An eman2 subdirectory will be
created, and the images, and available metadata will be copied into the new project. CTF parameters
will be extracted from the STAR file and will be automatically processed through EMAN1's CTF procedure
(this takes most of the time the script will require).
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
parser.add_argument("--lst", type=str, default='', help='.lst file.')
parser.add_argument('--boxsize', type=int, default=0, help='box size')
parser.add_argument(
'--allrelion',
type=int,
default=0,
help='convert all relion parameters to lst file if set to 1')
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument(
"--ny",
type=int,
help="Particle NY size, so no need to read image to get this value",
default=0)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
logid = E2init(sys.argv, options.ppid)
inputStarFileName = args[0]
if not os.path.islink(inputStarFileName):
inputStarFileName = os.path.abspath(inputStarFileName)
else:
#print "%s is a link file"%(inputStarFileName)
inputStarFileName = os.path.abspath(os.readlink(inputStarFileName))
topdir = os.path.dirname(inputStarFileName)
# try: os.mkdir("particles")
# except: pass
if options.verbose > 0:
print "Parsing STAR file %s" % (inputStarFileName)
star = StarFile(inputStarFileName)
#print star
if (options.lst):
lstfile = options.lst
else:
lstfile = os.path.basename(inputStarFileName)
lstfile = lstfile[0:lstfile.rfind('.')] + '.lst'
oldname = ""
olddf = -1.0
micronum = 0 # number of micrograph
fnum = 0 # image number in new file
params = []
for i in xrange(len(star["rlnImageName"])):
name = star["rlnImageName"][i].split("@")[1]
name = os.path.join(topdir, name)
imgnum = int(star["rlnImageName"][i].split("@")[0]) - 1
if (star.has_key("rlnMagnificationCorrection")
): #for old version relion star filr
apix = 1e4 * float(star["rlnDetectorPixelSize"][i]) / (
float(star["rlnMagnification"][i]) *
float(star["rlnMagnificationCorrection"][i]))
else:
apix = 1e4 * float(star["rlnDetectorPixelSize"][i]) / (float(
star["rlnMagnification"][i]))
if name != oldname:
# hdr=EMData(name,0,True)
# nx=hdr["nx"]
# ny=hdr["ny"]
nx = options.ny
ny = options.ny
oldname = name
if i == 0 or star["rlnDefocusU"][i - 1] != star["rlnDefocusU"][i]:
#this line will not run when i==0 (the 1st loop)
if micronum > 0 and options.verbose > 1:
print "Image {}: {} particles, df={}, size={}x{}".format(
micronum, fnum, ctf.defocus, nx, ny)
micronum += 1
fnum = 0 #when reaching a new micrograph, fnum need to start from 0 for write_image to a new good particle HDF stack
microname = "particles/{}_{:04d}.hdf".format(
base_name(name), micronum)
boxsize = options.boxsize
if boxsize <= 0:
boxsize = ny
# Make a "micrograph" CTF entry for each set of different defocuses to use when fitting
# check ctffind3.f function CTF(CS,WL,WGH1,WGH2,DFMID1,DFMID2,ANGAST,THETATR, IX, IY)
dfu = star["rlnDefocusU"][i] / 1e4
dfv = star["rlnDefocusV"][i] / 1e4
dfang = star["rlnDefocusAngle"][i]
defocus = (dfu + dfv) / 2.
dfdiff = math.fabs(dfu -
dfv) / 2. # dfu can be larger or smaller than dfv
if dfu > dfv:
dfang = math.fmod(
dfang + 360. + 90., 360.
) # largest defocus direction to smallest defocus direction
else:
pass # already at smallest defocus direction
ctf = EMAN1Ctf()
ctf.from_dict({
"defocus": defocus,
"dfang": dfang,
"dfdiff": dfdiff,
"voltage": star["rlnVoltage"][i],
"cs": star["rlnSphericalAberration"][i],
"ampcont": star["rlnAmplitudeContrast"][i] * 100.0,
"apix": apix,
"amplitude": 1,
"bfactor": 1e-5
})
# copy the image
# if name[-5:]==".mrcs":
# img=EMData(name,imgnum) # read one slice from the MRC stack
# else:
# img=EMData(name,0,False,Region(0,0,imgnum,nx,ny,1)) # read one slice from the MRC stack
# img["ctf"] = ctf
# img['apix_x'] = apix
# img['apix_y'] = apix
# if options.verbose>2:
# print "Particle %d: %s" % (i, img["ctf"].to_dict())
# img.write_image(microname,fnum,EMUtil.ImageType.IMAGE_HDF,False) #microname is a .hdf file, False tells it to write both header and image data
#print star["rlnAngleRot"][i], star["rlnAngleTilt"][i], star["rlnAnglePsi"][i]
alt, az, phi, x, y = relion2EMAN2(star["rlnAngleRot"][i],
star["rlnAngleTilt"][i],
star["rlnAnglePsi"][i],
star["rlnOriginX"][i],
star["rlnOriginY"][i], boxsize)
#all particles in eman2/particles folder are good particles, so the 1st column in .lst file should be 0,1,2,3,4,... in order
new_file = os.path.relpath(microname, os.getcwd())
param = [fnum, new_file, alt, az, phi, x, y, defocus, dfdiff, dfang]
if (options.allrelion):
for var in sorted(star.keys()):
param.append(star[var][i])
params.append(param)
fnum += 1
#generate .lst file corresponding to EMAN2 convention
fp = open(lstfile, 'w')
fp.write("#LST\n")
for p in params:
fp.write(
"%d\t%s\teuler=%g,%g,%g\tcenter=%g,%g\tdefocus=%s\tdfdiff=%s\tdfang=%s"
% (p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], p[9]))
if options.allrelion:
vars = sorted(star.keys())
for vi, var in enumerate(vars):
fp.write("\t%s=%s" % (var.split("rln")[-1], p[10 + vi]))
fp.write("\n")
fp.close()
if options.verbose > 0:
print "%d particles saved to %s" % (len(star["rlnImageName"]), lstfile)
E2end(logid)
def main(options):
"""
Projection subtraction program entry point.
:param options: Command-line arguments parsed by ArgumentParser.parse_args()
:return: Exit status
"""
rchop = lambda x, y: x if not x.endswith(y) or len(y) == 0 else x[:-len(y)]
options.output = rchop(options.output, ".star")
options.suffix = rchop(options.suffix, ".mrc")
options.suffix = rchop(options.suffix, ".mrcs")
star = StarFile(options.input)
npart = len(star['rlnImageName'])
sub_dens = EMData(options.submap)
if options.wholemap is not None:
dens = EMData(options.wholemap)
else:
print "Reference map is required."
return 1
# Write star header for output.star.
top_header = "\ndata_\n\nloop_\n"
headings = star.keys()
output_star = open("{0}.star".format(options.output), 'w')
output_star.write(top_header)
for i, heading in enumerate(headings):
output_star.write("_{0} #{1}\n".format(heading, i + 1))
if options.recenter: # Compute difference vector between new and old mass centers.
if options.wholemap is None:
print "Reference map required for recentering."
return 1
new_dens = dens - sub_dens
# Note the sign of the shift in coordinate frame is opposite the shift in the CoM.
recenter = Vec3f(*dens.phase_cog()[:3]) - Vec3f(
*new_dens.phase_cog()[:3])
else:
recenter = None
pool = None
if options.nproc > 1: # Compute subtraction in parallel.
pool = Pool(processes=options.nproc)
results = pool.imap(
lambda x: subtract(x,
dens,
sub_dens,
recenter=recenter,
no_frc=options.no_frc,
low_cutoff=options.low_cutoff,
high_cutoff=options.high_cutoff),
particles(star),
chunksize=min(npart / options.nproc, options.maxchunk))
else: # Use serial generator.
results = (subtract(x,
dens,
sub_dens,
recenter=recenter,
no_frc=options.no_frc,
low_cutoff=options.low_cutoff,
high_cutoff=options.high_cutoff)
for x in particles(star))
# Write subtraction results to .mrcs and .star files.
i = 0
nfile = 1
starpath = None
mrcs = None
mrcs_orig = None
for r in results:
if i % options.maxpart == 0:
mrcsuffix = options.suffix + "_%d" % nfile
nfile += 1
starpath = "{0}.mrcs".format(
os.path.sep.join(
os.path.relpath(mrcsuffix,
options.output).split(os.path.sep)[1:]))
mrcs = "{0}.mrcs".format(mrcsuffix)
mrcs_orig = "{0}_original.mrcs".format(mrcsuffix)
if os.path.exists(mrcs):
os.remove(mrcs)
if os.path.exists(mrcs_orig):
os.remove(mrcs_orig)
r.ptcl_norm_sub.append_image(mrcs)
if options.original:
r.ptcl.append_image(mrcs_orig)
if logger.getEffectiveLevel(
) == logging.DEBUG: # Write additional debug output.
ptcl_sub_img = r.ptcl.process("math.sub.optimal", {
"ref": r.ctfproj,
"actual": r.ctfproj_sub,
"return_subim": True
})
ptcl_lowpass = r.ptcl.process("filter.lowpass.gauss", {
"apix": 1.22,
"cutoff_freq": 0.05
})
ptcl_sub_lowpass = r.ptcl_norm_sub.process("filter.lowpass.gauss",
{
"apix": 1.22,
"cutoff_freq": 0.05
})
ptcl_sub_img.write_image("poreclass_subimg.mrcs", -1)
ptcl_lowpass.write_image("poreclass_lowpass.mrcs", -1)
ptcl_sub_lowpass.write_image("poreclass_sublowpass.mrcs", -1)
r.ctfproj.write_image("poreclass_ctfproj.mrcs", -1)
r.ctfproj_sub.write_image("poreclass_ctfprojsub.mrcs", -1)
assert r.meta.i == i # Assert particle order is preserved.
star['rlnImageName'][i] = "{0:06d}@{1}".format(
i % options.maxpart + 1, starpath) # Set new image name.
r.meta.update(star) # Update StarFile with altered fields.
line = ' '.join(str(star[key][i]) for key in headings)
output_star.write("{0}\n".format(line))
i += 1
output_star.close()
if pool is not None:
pool.close()
pool.join()
return 0
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <relion STAR file> <output set name>
This program will take a STAR file representing a subset of data from a Relion project and convert it into an EMAN2
set, IF the EMAN2 project contains the same micrographs and exact same boxed out particles as the Relion project
which produced the STAR file. e2refinetoeman.py can be used to convert an entire Relion project into an EMAN2
project. This program can then be used in this established project to copy information about particle subsets
selected in Relion into EMAN2.
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
#options associated with e2refine.py
parser.add_pos_argument(
name="star_file",
help="Select STAR file",
default="",
guitype='filebox',
browser="EMParticlesEditTable(withmodal=True,multiselect=False)",
row=6,
col=0,
rowspan=1,
colspan=3)
parser.add_pos_argument(name="output_set_name",
help="Name of output set",
default="from_relion",
guitype='str',
row=7,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--origname",
action="store_true",
help="Adds the original STAR name as a comment on each image",
default=False)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higher number means higher level of verboseness")
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
if options.verbose > 0: print("Parsing STAR file")
star = StarFile(args[0])
logid = E2init(sys.argv, options.ppid)
if "rlnMicrographName" in star: difkey = "rlnMicrographName"
else: difkey = "rlnDefocusU"
if '.lst' in args[1]:
if not "sets/" in args[1]: args[1] = f"sets/{args[1]}"
else:
args[1] = f"sets/{args[1]}.lst"
try:
os.unlink(args[1])
except:
pass
out = LSXFile(args[1])
orig = None
for i in range(len(star["rlnImageName"])):
rlnname = star["rlnImageName"][i]
name = rlnname.split("@")[1]
imgnum = int(rlnname.split("@")[0]) - 1
emanname = os.path.split(name)[1]
emanname = os.path.join("particles",
os.path.splitext(emanname)[0] + ".hdf")
if not os.path.exists(emanname):
badname = emanname
# this won't always work right, but it's worth a try
emanname = os.path.split(name)[1]
emanname = os.path.join(
"particles",
name.split("/")[-2] + "-" + os.path.splitext(emanname)[0] +
".hdf")
if not os.path.exists(emanname):
raise Exception(
f"Cannot find:{emanname} or {badname} (from {rlnname})")
if options.origname: orig = "# " + rlnname
out.write(-1, imgnum, emanname, orig)
E2end(logid)
def main():
progname = os.path.basename(sys.argv[0])
usage = """prog [options] <relion STAR file>
This program will take data from a Relion project and convert it into a basic EMAN2 project. Provide
the name of the Relion STAR file associated with the raw particle data. An eman2 subdirectory will be
created, and the images, and available metadata will be copied into the new project. CTF parameters
will be extracted from the STAR file and will be automatically processed through EMAN2's CTF procedure
(this takes most of the time the script will require).
"""
parser = EMArgumentParser(usage=usage, version=EMANVERSION)
#options associated with e2refine.py
parser.add_header(
name="text1",
help='Important instructions',
title="Use this to create an EMAN2.1 project from a Relion project:",
row=0,
col=0,
rowspan=1,
colspan=3)
parser.add_header(name="text2",
help='Important instructions',
title="* cd <folder with Relion STAR file>",
row=1,
col=0,
rowspan=1,
colspan=3)
parser.add_header(name="text3",
help='Important instructions',
title="* run e2projectmanager, and use this tool",
row=2,
col=0,
rowspan=1,
colspan=3)
parser.add_header(
name="text4",
help='Important instructions',
title="* exit PM, cd eman2, run PM from new eman2 folder",
row=3,
col=0,
rowspan=1,
colspan=3)
parser.add_pos_argument(
name="star_file",
help="Select STAR file",
default="",
guitype='filebox',
browser="EMParticlesEditTable(withmodal=True,multiselect=False)",
row=6,
col=0,
rowspan=1,
colspan=3)
parser.add_argument("--apix",
default=0,
type=float,
help="The angstrom per pixel of the input particles.",
guitype='floatbox',
row=8,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--verbose",
"-v",
dest="verbose",
action="store",
metavar="n",
type=int,
default=0,
help=
"verbose level [0-9], higner number means higher level of verboseness")
parser.add_argument(
"--fixeddefocus",
action="store_true",
help="Defocus and astigmatism are used unchanged from STAR file",
default=False,
guitype='boolbox',
row=10,
col=0,
rowspan=1,
colspan=1)
parser.add_argument(
"--refinedefocus",
action="store_true",
help=
"Will refit defocus to +-0.1 micron then further optimize using SSNR",
default=False,
guitype='boolbox',
row=10,
col=2,
rowspan=1,
colspan=1,
mode="[True]")
parser.add_argument(
"--refitdefocus",
action="store_true",
help=
"Will use EMAN2 CTF fitting to refit the defocus values within +-0.1 micron, astigmatism unchanged",
default=False,
guitype='boolbox',
row=10,
col=1,
rowspan=1,
colspan=1)
parser.add_argument(
"--ppid",
type=int,
help="Set the PID of the parent process, used for cross platform PPID",
default=-1)
(options, args) = parser.parse_args()
if options.apix <= 0:
print "A/pix must be specified"
sys.exit(1)
logid = E2init(sys.argv, options.ppid)
try:
os.mkdir("eman2")
except:
pass
try:
os.mkdir("eman2/particles")
except:
pass
os.chdir(
"eman2"
) # many things need to happen with the project directory as a base
if options.verbose > 0: print "Parsing STAR file"
star = StarFile("../" + args[0])
oldname = ""
olddf = -1.0
micronum = 0 # number of micrograph
fnum = 0 # image number in new file
for i in xrange(len(star["rlnImageName"])):
name = star["rlnImageName"][i].split("@")[1]
imgnum = int(star["rlnImageName"][i].split("@")[0]) - 1
if name != oldname:
hdr = EMData("../" + name, 0, True)
nx = hdr["nx"]
ny = hdr["ny"]
oldname = name
if options.verbose > 0:
print "Particle dimensions: {}x{}".format(nx, ny)
if i == 0 or star["rlnDefocusU"][i - 1] != star["rlnDefocusU"][i]:
if micronum > 0 and options.verbose > 0:
print "Image {}: {} particles processed, df={}".format(
micronum, fnum, ctf.defocus)
micronum += 1
fnum = 0
microname = "particles/{}_{:04d}.hdf".format(
base_name(name), micronum)
jdb = js_open_dict(info_name(microname))
# Make a "micrograph" CTF entry for each set of different defocuses to use when fitting
dfu = star["rlnDefocusU"][i]
dfv = star["rlnDefocusV"][i]
dfang = star["rlnDefocusAngle"][i]
ctf = EMAN2Ctf()
ctf.from_dict({
"defocus": (dfu + dfv) / 20000.0,
"dfang": dfang,
"dfdiff": (dfu - dfv) / 10000.0,
"voltage": star["rlnVoltage"][i],
"cs": star["rlnSphericalAberration"][i],
"ampcont": star["rlnAmplitudeContrast"][i] * 100.0,
"apix": options.apix
})
jdb["ctf_frame"] = [512, ctf, (256, 256), tuple(), 5, 1]
# copy the image
if name[-5:] == ".mrcs":
img = EMData("../" + name,
imgnum) # read one slice from the MRC stack
else:
img = EMData("../" + name, 0, False,
Region(0, 0, imgnum, nx, ny,
1)) # read one slice from the MRC stack
img.write_image(microname, fnum)
fnum += 1
if options.refinedefocus:
dfopt = "--curdefocushint --refinebysnr"
if options.verbose > 0:
print "Defocus Refit and SSNR Refinement to +-0.1 micron"
elif options.refitdefocus:
dfopt = "--curdefocushint"
if options.verbose > 0:
print "Defocus Refit to +-0.1 micron from Relion values"
elif options.fixeddefocus:
dfopt = "--curdefocusfix"
if options.verbose > 0: print "Computing particle SNRs"
else:
dfopt = " "
if options.verbose > 0: print "Full refit of CTF parameters"
launch_childprocess(
"e2ctf.py --voltage {} --cs {} --ac {} --apix {} --allparticles --autofit {} -v {}"
.format(ctf.voltage, ctf.cs, ctf.ampcont, ctf.apix, dfopt,
options.verbose - 1))
E2end(logid)