def process_stack(stackfile,
phaseflip=None,
wiener=None,
edgenorm=True,
oversamp=1,
default_ctf=None,
invert=False):
"""Will phase-flip and/or Wiener filter particles in a file based on their stored CTF parameters.
phaseflip should be the path for writing the phase-flipped particles
wiener should be the path for writing the Wiener filtered (and possibly phase-flipped) particles
oversamp will oversample as part of the processing, ostensibly permitting phase-flipping on a wider range of defocus values
"""
im = EMData()
im.read_image(
stackfile,
0) # can't use the constructor if bdb terminology is being used
ys = im.get_ysize() * oversamp
ys2 = im.get_ysize()
n = EMUtil.get_image_count(stackfile)
lctf = None
for i in range(n):
im1 = EMData()
im1.read_image(stackfile, i)
try:
ctf = im1["ctf"]
except:
ctf = default_ctf
if type(ctf) == EMAN1Ctf:
ctf = default_ctf # EMAN1 ctf needs a structure factor for this to work
if edgenorm: im1.process_inplace("normalize.edgemean")
if oversamp > 1:
im1.clip_inplace(
Region(-(ys2 * (oversamp - 1) / 2),
-(ys2 * (oversamp - 1) / 2), ys, ys))
fft1 = im1.do_fft()
if phaseflip:
if not lctf or not lctf.equal(ctf):
flipim = fft1.copy()
ctf.compute_2d_complex(flipim, Ctf.CtfType.CTF_SIGN)
fft1.mult(flipim)
out = fft1.do_ift()
out["ctf"] = ctf
out.clip_inplace(
Region(int(ys2 * (oversamp - 1) / 2.0),
int(ys2 * (oversamp - 1) / 2.0), ys2, ys2))
if invert: out.mult(-1.0)
out.process("normalize.edgemean")
out.write_image(phaseflip, i)
if wiener:
if not lctf or not lctf.equal(ctf):
wienerim = fft1.copy()
ctf.compute_2d_complex(wienerim, Ctf.CtfType.CTF_WIENER_FILTER)
# sxprint wienerim.get_attr_dict()
# display(wienerim)
# sxprint ctf.to_string()
# plot(ctf.background)
# plot(ctf.snr)
# plot(ctf.compute_1d(ys,Ctf.CtfType.CTF_WIENER_FILTER))
fft1.mult(wienerim)
out = fft1.do_ift()
out["ctf"] = ctf
out.clip_inplace(
Region(int(ys2 * (oversamp - 1) / 2.0),
int(ys2 * (oversamp - 1) / 2.0), ys2, ys2))
if invert: out.mult(-1.0)
out.process("normalize.edgemean")
try:
out.write_image(wiener, i)
except:
sxprint(wiener, i)
try:
out.write_image(wiener, i)
except:
sxprint("!!! ", wiener, i)
out.write_image("error.hed", -1)
lctf = ctf
#if wiener and wiener[:4]=="bdb:" : db_close_dict(wiener)
#if phaseflip and phaseflip[:4]=="bdb:" : db_close_dict(phaseflip)
return
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + " stack outdir --phase=1 --ou=outer_radius|sxconsistency.py --phase=3 newlocal/main000 --ou=133 --thresherr=3.0 --params=paramsa outgrouparms"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option(
"--phase",
type="int",
default=1,
help=
"Phase =1 prepares resampled stacks, =2 analyzes consistency of orientation parameters"
)
parser.add_option("--ou",
type="int",
default=-1,
help="outer radius for calculation of pixel error")
parser.add_option("--sym",
type="string",
default="c1",
help="symmetry of the refined structure")
parser.add_option(
"--thresherr",
type="float",
default=1.0,
help="Threshold for accpetable orientation errors (in pixels)")
parser.add_option(
"--ndigits",
type="int",
default=1,
help="Accuracy for checking whether parameters are identical")
parser.add_option("--chunk",
type="string",
default="",
help="Root of of four chunk files with indeces")
parser.add_option(
"--params",
type="string",
default="",
help="Root of of six parameter file names with refinement results")
(options, args) = parser.parse_args(arglist[1:])
sp_global_def.BATCH = True
if options.phase == 1 and len(args) == 2:
inputbdb = args[0]
outdir = args[1]
nn = EMUtil.get_image_count(inputbdb)
t = list(range(nn))
shuffle(t)
chunks = []
for i in range(4):
# I use the MPI function here just to easily get the balanced load
j, k = MPI_start_end(nn, 4, i)
chunks.append(t[j:k])
chunks[i].sort()
write_text_file(chunks[i], os.path.join(outdir,
'chunk%01d.txt' % i))
del t
write_text_file([len(chunks[i]) for i in range(4)],
os.path.join(outdir, 'chunklengths.txt'))
"""
pt = [[None]]*6
ll=0
for i in xrange(3):
for j in xrange(i+1,4):
pt[ll] = chunks[i]+chunks[j]
ll+=1
for i in xrange(6):
listfile = os.path.join(outdir,'lili%01d.txt'%i)
write_text_file(pt[i],listfile)
outbdb = "bdb:"+ os.path.join(outdir,"X%01d"%i)
cmd = '{} {} {} {}'.format('e2bdb.py', inputbdb, '--makevstack='+outbdb, '--list='+listfile)
subprocess.call(cmd, shell=True)
"""
# Run 6 programs
elif options.phase == 2 and len(args) == 2:
outdir = args[0]
howmanythesame = args[1]
ndigits = options.ndigits #for chc5 1, for ribo 4#4.0
prms = []
for i in range(6):
prms.append(
read_text_row(
os.path.join(outdir, options.params + "%01d.txt" % i)))
for j in range(len(prms[-1])):
for k in range(5):
prms[-1][j][k] = round(prms[-1][j][k], ndigits)
nn = 2 * len(prms[0])
n4 = nn // 4
qt = [[[-1.0]] * nn for i in range(6)]
ll = 0
for i in range(3):
for j in range(i + 1, 4):
qt[ll][i * n4:(i + 1) * n4] = prms[ll][:n4]
qt[ll][j * n4:(j + 1) * n4] = prms[ll][n4:]
ll += 1
thesame = 0
for ll in range(len(qt[0])):
rw = []
for j in range(6):
if (len(qt[j][ll]) > 1): rw.append(qt[j][ll])
isame = True
for j in range(3):
if (rw[0][j] != rw[1][j]):
isame = False
#print ll,rw[0][j], rw[1][j]
break
if (rw[0][j] != rw[2][j]):
isame = False
#print ll,rw[0][j], rw[2][j]
break
if isame: thesame += 1
qt = float(thesame) / nn
sxprint("Proportion of the same orientations ", qt)
write_text_file([qt], os.path.join(outdir, howmanythesame))
######### PHASE 3
elif options.phase == 3 and len(args) == 2:
outdir = args[0]
outgrouparms = args[1]
radius = options.ou
thresherr = options.thresherr #for chc5 1, for ribo 4#4.0
sym = int(options.sym[1:])
qsym = 360.0 / sym
blocks = ['A', 'B', 'C', 'D']
params = {}
ll = 0
for i in range(3):
for j in range(i + 1, 4):
params[chr(65 + i) + chr(48 + ll)] = []
params[chr(65 + j) + chr(48 + ll)] = []
ll += 1
chunks = {}
chunklengths = {}
for i in range(4):
chunks[chr(65 + i)] = list(
map(
int,
read_text_file(
os.path.join(outdir, options.chunk + "%01d.txt" % i))))
chunklengths[chr(65 + i)] = len(chunks[chr(65 + i)])
for i in range(6):
prms = read_text_row(
os.path.join(outdir, options.params + "%01d.txt" % i))
for q in blocks:
if q + chr(48 + i) in params:
params[q + chr(48 + i)] = prms[:chunklengths[q]]
del prms[:chunklengths[q]]
pairs = [["A0", "A1"], ["B3", "B4"], ["C1", "C5"], ["D2", "D4"]]
lefts = ["A2", "B0", "C3", "D5"]
# Compute average projection params and pixel errors
avgtrans = {}
pixer = {}
for i, q in enumerate(pairs):
avgtrans[q[0][0]] = [0.0] * chunklengths[q[0][0]]
pixer[q[0][0]] = [0.0] * chunklengths[q[0][0]]
for j in range(chunklengths[q[0][0]]):
fifi = [params[q[0]][j], params[q[1]][j]]
nas = [0.0, 0.0, 0.0]
if (sym == 1):
pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0],
fifi[1],
r=radius)
for i in range(2):
n1, n2, n3 = getfvec(fifi[i][0], fifi[i][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1, m2, m3 = getfvec(fifi[0][0], fifi[0][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
#if(k == 2):
# print "XXXX"
# print fifi[0],nas
for i in range(1, 2):
qnom = -1.e10
for j in range(-1, 2, 1):
t1, t2, t3 = getfvec(fifi[i][0] + j * qsym,
fifi[i][1])
nom = t1 * m1 + t2 * m2 + t3 * m3
if (nom > qnom):
qnom = nom
n1 = t1
n2 = t2
n3 = t3
#if(k == 2):
# print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
sxprint(qnom, n1, n2, n3, nas)
# To get the correct pixer phi angle has to be taken from the above!!
pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0],
fifi[1],
r=radius)
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if (nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2] / nom)) % 360.0
if (sym > 1 and ntheta > 90.0):
nphi = (degrees(atan2(nas[1], nas[0])) -
180.0) % qsym + 180.0
else:
nphi = degrees(atan2(nas[1], nas[0])) % qsym
#print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform([fifi[0][2:], fifi[1][2:]])
avgtrans[q[0][0]][j] = [
nphi, ntheta, twod[0], twod[1], twod[2]
]
perr = {}
tgood = 0
for q in blocks:
perr[q] = [True] * chunklengths[q]
for k in range(chunklengths[q]):
if (pixer[q][k] > thresherr): perr[q][k] = False
if perr[q][k]: tgood += 1
if (tgood < 4):
ERROR(
"No good images within the pixel error threshold specified"
)
return
sxprint(" tgood ", tgood)
hi = hist_list(
[pixer[q][k] for q in blocks for k in range(chunklengths[q])], 16)
for i in range(len(hi[0])):
sxprint("%4d %12.3f %12.0f " % (i, hi[0][i], hi[1][i]))
# Finished, store average orientation params and table of good images
# store lists of good images for each group
# blocks is indexed by first letter
good = [[] for i in range(4)]
bad = [[] for i in range(4)]
for i, q in enumerate(blocks):
for k in range(chunklengths[q]):
#deprt = max_3D_pixel_error(params[lefts[i]][k],avgtrans[q][k],r=radius)
if perr[q][k]:
good[i].append(chunks[q][k])
#[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k] ])
else:
bad[i].append(chunks[q][k])
#[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k]])
write_text_file(good[i], os.path.join(outdir,
"newgood%01d.txt" % i))
write_text_file(bad[i], os.path.join(outdir, "newbad%01d.txt" % i))
# write out parameters, for those in pairs write out average, for leftouts leave them as they were
# These parameters refer to the original X files.
ll = 0
for m in range(6):
if q + chr(48 + m) in params:
prmsgood = []
prmsbad = []
try:
j = lefts.index(q + chr(48 + m))
for k in range(chunklengths[q]):
if perr[q][k]:
prmsgood.append(params[q + chr(48 + m)][k])
else:
prmsbad.append(params[q + chr(48 + m)][k])
except:
for k in range(chunklengths[q]):
if perr[q][k]:
prmsgood.append(avgtrans[q][k])
else:
prmsbad.append(avgtrans[q][k])
write_text_row(
prmsgood,
os.path.join(outdir,
"params-newgood%01d%01d.txt" % (i, ll)))
write_text_row(
prmsbad,
os.path.join(outdir,
"params-newbad%01d%01d.txt" % (i, ll)))
ll += 1
# Generate newlili files from newgood, these contain original numbering of the total single file
ll = 0
for i in range(3):
for j in range(i + 1, 4):
write_text_file(good[i] + good[j],
os.path.join(outdir, "newlili%01d.txt" % ll))
ll += 1
# write out parameters, for those in pairs write out average, for leftouts leave them as they were
# These parameters refer to the original X files.
for i in range(6):
prms = []
for q in blocks:
if q + chr(48 + i) in params:
try:
j = lefts.index(q + chr(48 + i))
prms += params[q + chr(48 + i)]
except:
prms += avgtrans[q]
write_text_row(prms,
os.path.join(outdir, outgrouparms + "%01d.txt" % i))
# Write chunklengths
chunklengths = [len(good[i]) for i in range(4)]
write_text_file(chunklengths, os.path.join(outdir, "chunklengths.txt"))
"""
# We do not use consecutive numbering anymore
# Generate newx files from newgood, these contain consecutive (with gaps) numbering that allows to generate truncated X files from the previous X files
ll = 0
for i in xrange(3):
firstblock = []
l = len(perr[blocks[i]])
for k in xrange(l):
if perr[blocks[i]][k]:
firstblock.append(k)
for j in xrange(i+1,4):
secondblock = []
for k in xrange(len(perr[blocks[j]])):
if perr[blocks[j]][k]:
secondblock.append(k+l)
write_text_file( firstblock + secondblock, os.path.join(outdir,"goodX%01d.txt"%ll) )
ll += 1
del good,bad,firstblock,secondblock,perr
"""
"""
0 1 2 3 4 5
A A A = = =
B = = B B =
= C = C = C
= = D = D D
"""
elif options.phase == 4 and len(args) == 1:
outdir = args[0]
bp = 'badparams'
#outgrouparms= args[1]
radius = options.ou
thresherr = options.thresherr
sym = int(options.sym[1:])
qsym = 360.0 / sym
#params = [[None for i in xrange(3)] for j in xrange(4)]
ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general.
for jj in range(4):
params = [None for ii in range(ll)]
newbad = list(
map(int, read_text_file(options.params + "%01d.txt" % jj)))
nn = len(newbad)
for ii in range(ll):
params[ii] = read_text_row(
os.path.join(outdir, bp + "%01d%01d.txt" % (jj, ii)))
assert (nn == len(params[ii]))
# Compute average projection params and pixel errors
avgtrans = [None] * nn
pixer = [0.0] * nn
for j in range(nn):
nas = [0.0, 0.0, 0.0]
if (sym == 1):
#pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius)
for i in range(ll):
n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
for i in range(1, ll):
qnom = -1.e10
for j in range(-1, 2, 1):
t1, t2, t3 = getfvec(params[i][j][0] + j * qsym,
params[i][j][1])
nom = t1 * m1 + t2 * m2 + t3 * m3
if (nom > qnom):
qnom = nom
n1 = t1
n2 = t2
n3 = t3
#if(k == 2):
# sxprint ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
sxprint(qnom, n1, n2, n3, nas)
# To get the correct pixer phi angle has to be taken from the above!!
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if (nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2] / nom)) % 360.0
if (sym > 1 and ntheta > 90.0):
nphi = (degrees(atan2(nas[1], nas[0])) -
180.0) % qsym + 180.0
else:
nphi = degrees(atan2(nas[1], nas[0])) % qsym
#sxprint "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform(
[params[ii][j][2:] for ii in range(ll)])
avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]]
perr = errors_per_image(params, avgtrans, thresherr, radius)
rescued = []
rejects = []
for j in range(nn):
#sxprint chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)]
if (perr[j][0] <= thresherr): rescued.append([newbad[j], j])
else: rejects.append([newbad[j], j])
if (len(rescued) == 0):
write_text_row([-1, -1],
os.path.join(outdir, "rescued%01d.txt" % jj))
else:
write_text_row(rescued,
os.path.join(outdir, "rescued%01d.txt" % jj))
# We also have to write params.
if (len(rescued) != 0):
for ii in range(ll):
write_text_row(
[
params[ii][rescued[k][1]]
for k in range(len(rescued))
],
os.path.join(outdir,
"params-rescued%01d%01d.txt" % (jj, ii)))
if (len(rejects) == 0):
write_text_row([-1, -1],
os.path.join(outdir,
'rejects' + "%01d.txt" % jj))
else:
write_text_row(
rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj))
if (len(rejects) != 0):
for ii in range(ll):
write_text_row(
[
params[ii][rejects[k][1]]
for k in range(len(rejects))
],
os.path.join(outdir,
"params-rejects%01d%01d.txt" % (jj, ii)))
hi = hist_list([perr[j][0] for j in range(nn)], 16)
sxprint("Pixel errors for BAD GROUP ", chr(65 + jj))
for ii in range(len(hi[0])):
sxprint("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii]))
elif options.phase == 5 and len(args) == 1:
# This version is for meridien refinement. There are simply three full sets of params.
outdir = args[0]
bp = 'badparams'
#outgrouparms= args[1]
radius = options.ou
thresherr = options.thresherr
sym = int(options.sym[1:])
qsym = 360.0 / sym
#params = [[None for i in xrange(3)] for j in xrange(4)]
ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general.
for jj in range(1):
params = [None for ii in range(ll)]
#newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) )
#nn = len(newbad)
for ii in range(ll):
params[ii] = read_text_row(
os.path.join(outdir, "params%01d.txt" % (ii)))
#assert(nn == len(params[ii]) )
nn = len(params[0])
newbad = list(range(nn))
# Compute average projection params and pixel errors
avgtrans = [None] * nn
pixer = [0.0] * nn
for j in range(nn):
nas = [0.0, 0.0, 0.0]
if (sym == 1):
#pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius)
for i in range(ll):
n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1])
nas[0] += n1
nas[1] += n2
nas[2] += n3
else:
m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1])
nas[0] = m1
nas[1] = m2
nas[2] = m3
for i in range(1, ll):
qnom = -1.e10
for j in range(-1, 2, 1):
t1, t2, t3 = getfvec(params[i][j][0] + j * qsym,
params[i][j][1])
nom = t1 * m1 + t2 * m2 + t3 * m3
if (nom > qnom):
qnom = nom
n1 = t1
n2 = t2
n3 = t3
#if(k == 2):
# sxprint ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom
nas[0] += n1
nas[1] += n2
nas[2] += n3
sxprint(qnom, n1, n2, n3, nas)
# To get the correct pixer phi angle has to be taken from the above!!
nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2)
if (nom < 1.e-6):
nphi = 0.0
ntheta = 0.0
else:
ntheta = degrees(acos(nas[2] / nom)) % 360.0
if (sym > 1 and ntheta > 90.0):
nphi = (degrees(atan2(nas[1], nas[0])) -
180.0) % qsym + 180.0
else:
nphi = degrees(atan2(nas[1], nas[0])) % qsym
#sxprint "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta)
twod = average2dtransform(
[params[ii][j][2:] for ii in range(ll)])
avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]]
write_text_row(avgtrans, os.path.join(outdir, "avgtrans.txt"))
perr = errors_per_image(params, avgtrans, thresherr, radius)
rescued = []
rejects = []
for j in range(nn):
#sxprint chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)]
if (perr[j][0] <= thresherr): rescued.append([newbad[j], j])
else: rejects.append([newbad[j], j])
if (len(rescued) == 0):
write_text_row([-1, -1],
os.path.join(outdir, "rescued%01d.txt" % jj))
else:
write_text_row(rescued,
os.path.join(outdir, "rescued%01d.txt" % jj))
# We also have to write params.
if (len(rescued) != 0):
for ii in range(ll):
write_text_row(
[
params[ii][rescued[k][1]]
for k in range(len(rescued))
],
os.path.join(outdir,
"params-rescued%01d%01d.txt" % (jj, ii)))
if (len(rejects) == 0):
write_text_row([-1, -1],
os.path.join(outdir,
'rejects' + "%01d.txt" % jj))
else:
write_text_row(
rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj))
if (len(rejects) != 0):
for ii in range(ll):
write_text_row(
[
params[ii][rejects[k][1]]
for k in range(len(rejects))
],
os.path.join(outdir,
"params-rejects%01d%01d.txt" % (jj, ii)))
hi = hist_list([perr[j][0] for j in range(nn)], 16)
sxprint("Pixel errors for BAD GROUP ", chr(65 + jj))
for ii in range(len(hi[0])):
sxprint("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii]))
else:
sxprint("Usage: ")
sxprint("""
Phase 1: sxconsistency.py --phase=1 bdb:data outdir
output files are:
in directory outdir: lili0.txt to lili5.txt contain indices of images in resampled six groups
bdb:dataX0 to bdb:dataX5 are metafiles containing six resampled groups of images derived from bdb:data
Phase 2 sxconsistency.py --phase=2 outdir --ndigits=1 --params=paramsb howmanythesame.txt
outdir - directory containing files lili0.txt to lili5.txt produced in phase 1
--params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be
master/main/params0.txt to master/main/params5.txt
output files:
howmanythesame.txt - contains one number, a ratio of number of images that did not change orientations
to the total number of images
Phase 3: sxconsistency.py --phase=3 outdir --ou=133 --thresherr=3.0 --params=paramsa outgrouparms
input files:
outdir - directory containing files lili0.txt to lili5.txt produced in phase 1
--params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be
master/main/params0.txt to master/main/params5.txt
output files:
outgrouparms*.txt - Root of of six parameters files with average 3D orientation parameters computed from six runs, can be imported into bdb:data
The next two files contain the original image numbers refering to the top bdb.
outdir/newgood.txt - Text file with indices of images whose orientation parameters arrors are below specified thresherr (to be kept)
outdir/bad.txt - Text file with indices of images whose orientation parameters arrors are above specified thresherr (to be rejected)
Phase 4: sxconsistency.py --phase=4 outdir --ou=133 --thresherr=3.0 --params=master/main001/newbad
input files:
outdir - directory containing files badparamsij.txt i= 0,3, j=0,2, which resulted from three alignments of four badchunk images
--params= - Root of four files with original indices of bad images, they will be read and a subset corresponding to rescued ones will be outputed
output files:
rescued*.txt - four files with indices of accepted images from badchunk.
rejects*.txt - four files with indices of rejected images from badchunk.
There are two columns: [number in original bdb:chunk, number in bdb:newbad]
""")
sxprint("Please run '" + progname + " -h' for detailed options")
sp_global_def.BATCH = False
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = optparse.os.path.basename(arglist[0])
usage = progname + """ inputvolume locresvolume maskfile outputfile --radius --falloff --MPI
Locally filer a volume based on local resolution volume (sxlocres.py) within area outlined by the maskfile
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--radius",
type="int",
default=-1,
help=
"if there is no maskfile, sphere with r=radius will be used, by default the radius is nx/2-1"
)
parser.add_option("--falloff",
type="float",
default=0.1,
help="falloff of tanl filter (default 0.1)")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 3 or len(args) > 4:
sp_global_def.sxprint("See usage " + usage)
sp_global_def.ERROR(
"Wrong number of parameters. Please see usage information above.")
return
if sp_global_def.CACHE_DISABLE:
pass #IMPORTIMPORTIMPORT from sp_utilities import disable_bdb_cache
sp_utilities.disable_bdb_cache()
if options.MPI:
number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
if (myid == main_node):
#print sys.argv
vi = sp_utilities.get_im(sys.argv[1])
ui = sp_utilities.get_im(sys.argv[2])
#print Util.infomask(ui, None, True)
radius = options.radius
nx = vi.get_xsize()
ny = vi.get_ysize()
nz = vi.get_zsize()
dis = [nx, ny, nz]
else:
falloff = 0.0
radius = 0
dis = [0, 0, 0]
vi = None
ui = None
dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node)
if (myid != main_node):
nx = int(dis[0])
ny = int(dis[1])
nz = int(dis[2])
radius = sp_utilities.bcast_number_to_all(radius, main_node)
if len(args) == 3:
if (radius == -1):
radius = min(nx, ny, nz) // 2 - 1
m = sp_utilities.model_circle(radius, nx, ny, nz)
outvol = args[2]
elif len(args) == 4:
if (myid == main_node):
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
else:
m = sp_utilities.model_blank(nx, ny, nz)
outvol = args[3]
sp_utilities.bcast_EMData_to_all(m, myid, main_node)
pass #IMPORTIMPORTIMPORT from sp_filter import filterlocal
filteredvol = sp_filter.filterlocal(ui, vi, m, options.falloff, myid,
main_node, number_of_proc)
if (myid == 0):
filteredvol.write_image(outvol)
else:
vi = sp_utilities.get_im(args[0])
ui = sp_utilities.get_im(
args[1]
) # resolution volume, values are assumed to be from 0 to 0.5
nn = vi.get_xsize()
falloff = options.falloff
if len(args) == 3:
radius = options.radius
if (radius == -1):
radius = nn // 2 - 1
m = sp_utilities.model_circle(radius, nn, nn, nn)
outvol = args[2]
elif len(args) == 4:
m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5)
outvol = args[3]
sp_fundamentals.fftip(vi) # this is the volume to be filtered
# Round all resolution numbers to two digits
for x in range(nn):
for y in range(nn):
for z in range(nn):
ui.set_value_at_fast(x, y, z,
round(ui.get_value_at(x, y, z), 2))
st = EMAN2_cppwrap.Util.infomask(ui, m, True)
filteredvol = sp_utilities.model_blank(nn, nn, nn)
cutoff = max(st[2] - 0.01, 0.0)
while (cutoff < st[3]):
cutoff = round(cutoff + 0.01, 2)
pt = EMAN2_cppwrap.Util.infomask(
sp_morphology.threshold_outside(ui, cutoff - 0.00501,
cutoff + 0.005), m, True)
if (pt[0] != 0.0):
vovo = sp_fundamentals.fft(
sp_filter.filt_tanl(vi, cutoff, falloff))
for x in range(nn):
for y in range(nn):
for z in range(nn):
if (m.get_value_at(x, y, z) > 0.5):
if (round(ui.get_value_at(x, y, z),
2) == cutoff):
filteredvol.set_value_at_fast(
x, y, z, vovo.get_value_at(x, y, z))
sp_global_def.write_command(optparse.os.path.dirname(outvol))
filteredvol.write_image(outvol)
def main():
progname = os.path.basename(sys.argv[0])
usage = """%prog [options] input.pdb output.hdf
Converts a pdb file into an electron density map. 0,0,0 in PDB space will
map to the center of the volume."""
parser = OptionParser(usage=usage, version=EMANVERSION)
parser.add_option("--apix",
"-A",
type="float",
help="Angstrom/voxel",
default=1.0)
parser.add_option("--box",
"-B",
type="string",
help="Box size in pixels, <xyz> or <x,y,z>")
parser.add_option("--het",
action="store_true",
help="Include HET atoms in the map",
default=False)
parser.add_option(
"--chains",
type="string",
help=
"String list of chain identifiers to include, e.g. 'ABEFG'; default: include all chains",
default='')
parser.add_option(
"--center",
type="string",
default="a",
help=
"center: c - coordinates; a (default) - center of gravity; <x,y,z> - vector (in Angstrom) to subtract from all coordinates; n - none"
)
parser.add_option("--O",
action="store_true",
default=False,
help="use O system of coordinates")
parser.add_option("--quiet",
action="store_true",
default=False,
help="Verbose is the default")
parser.add_option("--tr0",
type="string",
default="none",
help="Filename of initial 3x4 transformation matrix")
parser.add_option("--set_apix_value",
action="store_true",
help="Set apix value in header of the ouput map",
default=False)
(options, args) = parser.parse_args() #
if len(args) < 2:
ERROR("Input and output files required")
return
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
chains = options.chains
if chains == '':
chains = None
try:
infile = open(args[0], "r")
except:
ERROR("Cannot open input file")
return
aavg = [0, 0, 0] # calculate atomic center
asig = [0, 0, 0] # to compute radius of gyration
natm = 0
atoms = [] # we store a list of atoms to process to avoid multiple passes
nelec = 0
mass = 0
# read in initial-transformation file:
if (options.tr0 != "none"):
cols = read_text_file(options.tr0, -1)
txlist = []
for i in range(3):
txlist.append(cols[0][i])
txlist.append(cols[1][i])
txlist.append(cols[2][i])
txlist.append(cols[3][i])
tr0 = Transform(txlist)
# parse the pdb file and pull out relevant atoms
for line in infile:
if (line[:4] == 'ATOM' or (line[:6] == 'HETATM' and options.het)):
if chains and (line[21] not in chains): continue
try:
a = line[12:14].strip()
aseq = int(line[6:11].strip())
res = int(line[22:26].strip())
if (options.O):
x = float(line[38:46])
y = float(line[30:38])
z = -float(line[46:54])
else:
x = float(line[30:38])
y = float(line[38:46])
z = float(line[46:54])
except:
sxprint(
"PDB Parse error:\n%s\n'%s','%s','%s' '%s','%s','%s'\n" %
(line, line[12:14], line[6:11], line[22:26], line[30:38],
line[38:46], line[46:54]))
sxprint(a, aseq, res, x, y, z)
try:
nelec += atomdefs[a.upper()][0]
mass += atomdefs[a.upper()][1]
except:
sxprint(("Unknown atom %s ignored at %d" % (a, aseq)))
atoms.append([a, x, y, z])
natm += 1
if (options.center == "a"):
aavg[0] += x * atomdefs[a.upper()][1]
aavg[1] += y * atomdefs[a.upper()][1]
aavg[2] += z * atomdefs[a.upper()][1]
asig[0] += x**2 * atomdefs[a.upper()][1]
asig[1] += y**2 * atomdefs[a.upper()][1]
asig[2] += z**2 * atomdefs[a.upper()][1]
else:
aavg[0] += x
aavg[1] += y
aavg[2] += z
asig[0] += x**2
asig[1] += y**2
asig[2] += z**2
infile.close()
if (options.center == "a"):
rad_gyr = sqrt((asig[0] + asig[1] + asig[2]) / mass -
(aavg[0] / mass)**2 - (aavg[1] / mass)**2 -
(aavg[2] / mass)**2)
else:
rad_gyr = sqrt((asig[0] + asig[1] + asig[2]) / natm -
(aavg[0] / natm)**2 - (aavg[1] / natm)**2 -
(aavg[2] / natm)**2)
if not options.quiet:
sxprint(
"%d atoms; total charge = %d e-; mol mass = %.2f kDa; radius of gyration = %.2f A"
% (natm, nelec, mass / 1000.0, rad_gyr))
# center PDB according to option:
if (options.center == "a"):
if not options.quiet:
sxprint(
"center of gravity at %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)"
% (aavg[0] / mass, aavg[1] / mass, aavg[2] / mass))
for i in range(len(atoms)):
atoms[i][1] -= aavg[0] / mass
atoms[i][2] -= aavg[1] / mass
atoms[i][3] -= aavg[2] / mass
if (options.center == "c"):
if not options.quiet:
sxprint(
"atomic center at %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)"
% (aavg[0] / natm, aavg[1] / natm, aavg[2] / natm))
for i in range(len(atoms)):
atoms[i][1] -= aavg[0] / natm
atoms[i][2] -= aavg[1] / natm
atoms[i][3] -= aavg[2] / natm
spl = options.center.split(',')
if len(spl) == 3: # substract the given vector from all coordinates
if not options.quiet:
sxprint(
"vector to substract: %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)"
% (float(spl[0]), float(spl[1]), float(spl[2])))
for i in range(len(atoms)):
atoms[i][1] -= float(spl[0])
atoms[i][2] -= float(spl[1])
atoms[i][3] -= float(spl[2])
# apply given initial transformation (this used to be done before centering,
# thereby loosing the translation. This is the right place to apply tr0):
if (options.tr0 != "none"):
if not options.quiet:
sxprint("Applying initial transformation to PDB coordinates... ")
for i in range(len(atoms)):
atom_coords = Vec3f(atoms[i][1], atoms[i][2], atoms[i][3])
new_atom_coords = tr0 * atom_coords
atoms[i][1] = new_atom_coords[0]
atoms[i][2] = new_atom_coords[1]
atoms[i][3] = new_atom_coords[2]
if not options.quiet:
sxprint("done.\n")
# bounding box:
amin = [atoms[0][1], atoms[0][2], atoms[0][3]]
amax = [atoms[0][1], atoms[0][2], atoms[0][3]]
for i in range(1, len(atoms)):
for k in range(3):
amin[k] = min(atoms[i][k + 1], amin[k])
amax[k] = max(atoms[i][k + 1], amax[k])
if not options.quiet:
sxprint("Range of coordinates [A]: x: %7.2f - %7.2f" %
(amin[0], amax[0]))
sxprint(" y: %7.2f - %7.2f" %
(amin[1], amax[1]))
sxprint(" z: %7.2f - %7.2f" %
(amin[2], amax[2]))
# find the output box size, either user specified or from bounding box
box = [0, 0, 0]
try:
spl = options.box.split(',')
if len(spl) == 1: box[0] = box[1] = box[2] = int(spl[0])
else:
box[0] = int(spl[0])
box[1] = int(spl[1])
box[2] = int(spl[2])
except:
for i in range(3):
box[i] = int(2 * max(fabs(amax[i]), fabs(amin[i])) / options.apix)
# Increase the box size by 1/4.
box[i] += box[i] // 4
if not options.quiet:
sxprint("Bounding box [pixels]: x: %5d " % box[0])
sxprint(" y: %5d " % box[1])
sxprint(" z: %5d " % box[2])
# figure oversampled box size
#bigb = max(box[0],box[1],box[2])
fcbig = 1
"""Multiline Comment0"""
#MULTILINEMULTILINEMULTILINE 0
#MULTILINEMULTILINEMULTILINE 0
#MULTILINEMULTILINEMULTILINE 0
#MULTILINEMULTILINEMULTILINE 0
#MULTILINEMULTILINEMULTILINE 0
#MULTILINEMULTILINEMULTILINE 0
#MULTILINEMULTILINEMULTILINE 0
if not options.quiet:
sxprint("Box size: %d x %d x %d" % (box[0], box[1], box[2]),
", oversampling ", fcbig)
# Calculate working dimensions
pixelbig = options.apix / fcbig
bigbox = []
for i in range(3):
bigbox.append(box[i] * fcbig)
# initialize the final output volume
outmap = EMData(bigbox[0], bigbox[1], bigbox[2], True)
nc = []
for i in range(3):
nc.append(bigbox[i] // 2)
# fill in the atoms
for i in range(len(atoms)):
#print "Adding %d '%s'"%(i,atoms[i][0])
if not options.quiet and i % 1000 == 0:
sxprint('\r %d' % i, end=' ')
sys.stdout.flush()
try:
elec = atomdefs[atoms[i][0].upper()][0]
#outmap[int(atoms[i][1]/pixelbig+bigbox[0]//2),int(atoms[i][2]/pixelbig+bigbox[1]//2),int(atoms[i][3]/pixelbig+bigbox[2]//2)] += elec
for k in range(2):
pz = atoms[i][3] / pixelbig + nc[2]
dz = pz - int(pz)
uz = ((1 - k) + (2 * k - 1) * dz) * elec
for l in range(2):
py = atoms[i][2] / pixelbig + nc[1]
dy = py - int(py)
uy = ((1 - l) + (2 * l - 1) * dy) * uz
for m in range(2):
px = atoms[i][1] / pixelbig + nc[0]
dx = px - int(px)
outmap[int(px) + m,
int(py) + l,
int(pz) + k] += ((1 - m) +
(2 * m - 1) * dx) * uy
except:
sxprint("Skipping %d '%s'" % (i, atoms[i][0]))
if not options.quiet:
sxprint('\r %d\nConversion complete.' %
len(atoms)) #," Now shape atoms."
"""Multiline Comment1"""
#MULTILINEMULTILINEMULTILINE 1
#MULTILINEMULTILINEMULTILINE 1
#MULTILINEMULTILINEMULTILINE 1
#MULTILINEMULTILINEMULTILINE 1
#MULTILINEMULTILINEMULTILINE 1
#MULTILINEMULTILINEMULTILINE 1
#MULTILINEMULTILINEMULTILINE 1
#MULTILINEMULTILINEMULTILINE 1
#MULTILINEMULTILINEMULTILINE 1
(filename_path, filextension) = os.path.splitext(args[1])
if filextension == ".hdf":
if options.set_apix_value:
outmap.set_attr("apix_x", options.apix)
outmap.set_attr("apix_y", options.apix)
outmap.set_attr("apix_z", options.apix)
outmap.set_attr("pixel_size", options.apix)
else:
sxprint("Pixel_size is not set in the header!")
outmap.write_image(args[1], 0, EMUtil.ImageType.IMAGE_HDF)
elif filextension == ".spi":
outmap.write_image(args[1], 0, EMUtil.ImageType.IMAGE_SINGLE_SPIDER)
else:
ERROR("Unknown image type")
return
def _main_():
_, child_argparser = setup_argparser()
# These global variables are kind of ugly, but are necessary in python2. Will be removed in
# python 3.
global STACK_FILE_PATH
global PROJECTION_PARAMETERS
global VOLUME1
global VOLUME2
global DEFOCUS_SEARCH_RANGE
global DEFOCUS_STEP_SIZE
global MASK_VOLUME
global RESOLUTION
global PIXEL_SIZE
global HALF_MAP_ASSIGNEMENTS
args = child_argparser.parse_args()
if "meridien" in sys.argv[1]:
meridien_path = args.meridien_path
files = sp_ctf_refine_io.read_meridien_data(meridien_path)
volume1_file_path = files["first_halfmap"]
volume2_file_path = files["second_halfmap"]
chunk_file_path = files["chunk1"]
params_file_path = files["final_params"]
else:
volume1_file_path = args.volume
volume2_file_path = args.volume2
params_file_path = args.params_path
chunk_file_path = args.chunk
if volume2_file_path is None and chunk_file_path is not None:
sp_global_def.ERROR(
"If chunk file is specified, you need to specify a second volume (-v2)"
)
STACK_FILE_PATH = args.inputstack # "bdb:name"
mask_file_path = args.mask
DEFOCUS_SEARCH_RANGE = args.range
DEFOCUS_STEP_SIZE = args.delta
output_folder = args.outputdir
if os.path.exists(output_folder):
sp_global_def.ERROR("Output folder already exists. Stop execution.")
else:
os.makedirs(output_folder)
sp_global_def.write_command(output_folder)
output_virtual_stack_path = "bdb:" + os.path.join(output_folder, "ctf_refined")
output_stats_path = os.path.join(output_folder, "statistics")
number_of_particles_to_read = args.number_particles
volume_nominal_resolution = args.resolution
RESOLUTION = volume_nominal_resolution
PIXEL_SIZE = args.apix
PROJECTION_PARAMETERS = sp_ctf_refine_io.read_meridien_params(params_file_path)
num_cpu = multiprocessing.cpu_count() - 1
volume1, volume2, MASK_VOLUME = sp_ctf_refine_io.read_volume(
path_vol_1=volume1_file_path,
path_vol_2=volume2_file_path,
path_mask=mask_file_path,
resolution=volume_nominal_resolution,
pixel_size=PIXEL_SIZE,
)
VOLUME1 = volume1
VOLUME2 = volume2
particle_chunks, number_of_particles = indices_to_chunks(
STACK_FILE_PATH,
chunk_size=100,
number_of_particles_to_read=number_of_particles_to_read,
)
if chunk_file_path:
HALF_MAP_ASSIGNEMENTS = get_half_map_assigments_per_particle(
stack_path=STACK_FILE_PATH,
chunk_path=chunk_file_path,
number_of_particles_to_read=number_of_particles_to_read,
)
sp_global_def.sxprint("####Start refinement####")
start = time.time()
# for chunk in particle_chunks:
# refine_set(chunk)
#####################################################################
# Python 2 workaround to get rid of a memory leak.
# Pool eats up memory and only give it back after it returns.
# So let it return more often by deviding chunks into chunks :-)
#####################################################################
num_chunks = len(particle_chunks)
refinement_results = []
with tqdm(total=num_chunks, file=sys.stdout) as pbar:
for i in range(0, num_chunks, num_cpu):
subset_chunk = particle_chunks[i: (i + num_cpu)]
pool = multiprocessing.Pool(num_cpu)
refinement_result = pool.map_async(refine_set, subset_chunk, chunksize=1)
pool.close()
# print_progress(refinement_result)
pool.join()
for res in refinement_result.get():
refinement_results.append(res)
pbar.update(len(subset_chunk))
#####################################################################
end = time.time()
sp_global_def.sxprint("Time for ", number_of_particles, " Particles:", end - start)
# Ouput results
refined_ctfs_as_list, refinement_results_per_micrograph = merge_ctf_refinement_results(
refinement_results=refinement_results
)
sp_ctf_refine_io.write_virtual_bdb_stack(
output_stack_path=output_virtual_stack_path,
origin_stack_path=STACK_FILE_PATH,
refined_ctfs=refined_ctfs_as_list,
number_of_particles=number_of_particles,
)
sp_global_def.sxprint("Write statistics...")
# WRITE STATISTICS
if not os.path.exists(output_stats_path):
os.makedirs(output_stats_path)
refinement_stats_per_micrograh = calc_statistics(refinement_results_per_micrograph)
sp_ctf_refine_io.write_statistics(output_stats_path, refinement_stats_per_micrograh)
# Estimate error Range
min_max_error, min_max_ratio = calculate_result_ranges(
refinement_results_per_micrograph
)
# Save particle plots
sp_global_def.sxprint("Write images...")
path_output_img = os.path.join(output_stats_path, "img/")
if not os.path.exists(path_output_img):
os.makedirs(path_output_img)
sp_ctf_refine_plotting.create_and_save_particle_plots(
path_output_img=path_output_img,
stack_file_path=STACK_FILE_PATH,
refinement_results_per_micrograph=refinement_results_per_micrograph,
min_max_error=min_max_error,
min_max_ratio=min_max_ratio,
)
sp_global_def.sxprint("Write other...")
refinement_results_matrix = get_refinement_results_matrix(
refinement_results_per_micrograph
)
particle_error_path = os.path.join(output_stats_path, "particle_results.txt")
refinement_results_matrix = refinement_results_matrix[
refinement_results_matrix[:, 0].argsort()
]
np.savetxt(
particle_error_path,
refinement_results_matrix,
delimiter=",",
fmt=["%d", "%f", "%f", "%f"],
)
sp_global_def.sxprint("Done")
def main():
"""
Main function.
Arguments:
None
Returns:
None
"""
command_args = parse_command_line()
# Import volume
sp_global_def.sxprint("Import volume.")
input_vol = sp_utilities.get_im(command_args.input_volume)
# Sanity checks
sanity_checks(command_args, input_vol)
try:
os.makedirs(command_args.output_dir)
except OSError:
sp_global_def.sxprint(
"Output directory already exists. No need to create it.")
else:
sp_global_def.sxprint("Created output directory.")
sp_global_def.write_command(command_args.output_dir)
output_prefix = os.path.join(command_args.output_dir, command_args.prefix)
# Filter volume if specified
if command_args.low_pass_filter_resolution is not None:
sp_global_def.sxprint("Filter volume to {0}A.".format(
command_args.low_pass_filter_resolution))
input_vol = sp_filter.filt_tanl(
input_vol,
old_div(command_args.pixel_size,
command_args.low_pass_filter_resolution),
command_args.low_pass_filter_falloff,
)
input_vol.write_image(output_prefix + "_filtered_volume.hdf")
else:
sp_global_def.sxprint("Skip filter volume.")
# Create a mask based on the filtered volume
sp_global_def.sxprint("Create mask")
density_threshold = -9999.0
nsigma = 1.0
if command_args.mol_mass:
density_threshold = input_vol.find_3d_threshold(
command_args.mol_mass, command_args.pixel_size)
sp_global_def.sxprint(
"Mask molecular mass translated into binary threshold: ",
density_threshold)
elif command_args.threshold:
density_threshold = command_args.threshold
elif command_args.nsigma:
nsigma = command_args.nsigma
else:
assert False
if command_args.edge_type == "cosine":
mode = "C"
elif command_args.edge_type == "gaussian":
mode = "G"
else:
assert False
mask_first = sp_morphology.adaptive_mask_scipy(
input_vol,
nsigma=nsigma,
threshold=density_threshold,
ndilation=command_args.ndilation,
nerosion=command_args.nerosion,
edge_width=command_args.edge_width,
allow_disconnected=command_args.allow_disconnected,
mode=mode,
do_approx=command_args.do_old,
do_fill=command_args.fill_mask,
do_print=True,
)
# Create a second mask based on the filtered volume
s_mask = None
s_density_threshold = 1
s_nsigma = 1.0
if command_args.second_mask is not None:
sp_global_def.sxprint("Prepare second mask")
s_mask = sp_utilities.get_im(command_args.second_mask)
s_density_threshold = -9999.0
s_nsigma = 1.0
if command_args.s_mol_mass:
s_density_threshold = input_vol.find_3d_threshold(
command_args.s_mol_mass, command_args.s_pixel_size)
sp_global_def.sxprint(
"Second mask molecular mass translated into binary threshold: ",
s_density_threshold,
)
elif command_args.s_threshold:
s_density_threshold = command_args.s_threshold
elif command_args.s_nsigma:
s_nsigma = command_args.s_nsigma
else:
assert False
elif command_args.second_mask_shape is not None:
sp_global_def.sxprint("Prepare second mask")
nx = mask_first.get_xsize()
ny = mask_first.get_ysize()
nz = mask_first.get_zsize()
if command_args.second_mask_shape == "cube":
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
s_mask = sp_utilities.model_blank(s_nx, s_ny, s_nz, 1)
elif command_args.second_mask_shape == "cylinder":
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
try:
s_mask = sp_utilities.model_cylinder(s_radius, s_nx, s_ny,
s_nz)
except RuntimeError as e:
sp_global_def.sxprint(
"An error occured! Please check the error log")
raise
elif command_args.second_mask_shape == "sphere":
s_radius = command_args.s_radius
s_nx = command_args.s_nx
s_ny = command_args.s_ny
s_nz = command_args.s_nz
try:
s_mask = sp_utilities.model_circle(s_radius, s_nx, s_ny, s_nz)
except RuntimeError as e:
sp_global_def.sxprint(
"An error occured! Please check the error log")
raise
else:
assert False
s_mask = sp_utilities.pad(s_mask, nx, ny, nz, 0)
if s_mask is not None:
sp_global_def.sxprint("Create second mask")
if command_args.s_edge_type == "cosine":
mode = "C"
elif command_args.s_edge_type == "gaussian":
mode = "G"
else:
assert False
s_mask = sp_morphology.adaptive_mask_scipy(
s_mask,
nsigma=s_nsigma,
threshold=s_density_threshold,
ndilation=command_args.s_ndilation,
nerosion=command_args.s_nerosion,
edge_width=command_args.s_edge_width,
allow_disconnected=command_args.s_allow_disconnected,
mode=mode,
do_approx=command_args.s_do_old,
do_fill=command_args.s_fill_mask,
do_print=True,
)
if command_args.s_invert:
s_mask = 1 - s_mask
sp_global_def.sxprint("Write outputs.")
mask_first.write_image(output_prefix + "_mask_first.hdf")
s_mask.write_image(output_prefix + "_mask_second.hdf")
masked_combined = mask_first * s_mask
masked_combined.write_image(output_prefix + "_mask.hdf")
else:
sp_global_def.sxprint("Write outputs.")
mask_first.write_image(output_prefix + "_mask.hdf")
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " stack outdir <maskfile> --K=10 --trials=2 --debug --maxit=100 --rand_seed=10 --crit='all' --F=0.9 --T0=2.0 --init_method='rnd' --normalize --CTF --MPI --CUDA"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--K",
type="int",
default=2,
help="Number of classes (default 2)")
parser.add_option("--trials",
type="int",
default=1,
help="Number of trials of K-means (default 1)")
parser.add_option("--maxit",
type="int",
default=100,
help="Maximum number of iterations within K-means")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Perform classification using CTF information")
parser.add_option("--rand_seed",
type="int",
default=-1,
help="Random seed of initial (default random)")
parser.add_option(
"--crit",
type="string",
default="D",
help=
"Criterions: Coleman [C], Harabasz[H], Davies-Bouldin[D], All [all]")
#parser.add_option("--F", type="float", default=0.0, help="Cooling in simulated annealing, ex.: 0.9")
#parser.add_option("--T0", type="float", default=0.0, help="Initial temperature in simulated annealing, ex: 100")
parser.add_option("--MPI",
action="store_true",
default=False,
help="Use MPI version")
parser.add_option("--debug", action="store_true", default=False, help="")
parser.add_option("--normalize",
action="store_true",
default=False,
help="Normalize images under the mask")
parser.add_option(
'--init_method',
type='string',
default='rnd',
help=
'Method used to initialize partition: "rnd" randomize or "d2w" for d2 weighting initialization (default is rnd)'
)
(options, args) = parser.parse_args()
if len(args) < 2 or len(args) > 3:
sxprint("usage: " + usage)
sxprint("Please run '" + progname + " -h' for detailed options")
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
elif options.trials < 1:
ERROR("Number of trials should be at least 1")
return
else:
if len(args) == 2: mask = None
else: mask = args[2]
if options.K < 2:
ERROR("K must be > 1 group")
return
if options.CTF:
ERROR("CTF option not implemented")
return
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
from sp_applications import k_means_main
sp_global_def.BATCH = True
k_means_main(args[0], args[1], mask, "SSE", options.K,
options.rand_seed, options.maxit, options.trials,
options.crit, options.CTF, 0.0, 0.0, options.MPI, False,
options.debug, options.normalize, options.init_method)
sp_global_def.BATCH = False
def resample( prjfile, outdir, bufprefix, nbufvol, nvol, seedbase,\
delta, d, snr, CTF, npad,\
MPI, myid, ncpu, verbose = 0 ):
from sp_utilities import even_angles
from random import seed, jumpahead, shuffle
import os
from sys import exit
nprj = EMUtil.get_image_count( prjfile )
if MPI:
if myid == 0:
if os.path.exists(outdir): nx = 1
else: nx = 0
else: nx = 0
ny = bcast_number_to_all(nx, source_node = 0)
if ny == 1: ERROR('Output directory exists, please change the name and restart the program', "resample", 1,myid)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
if myid == 0:
os.makedirs(outdir)
sp_global_def.write_command(outdir)
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
else:
if os.path.exists(outdir):
ERROR('Output directory exists, please change the name and restart the program', "resample", 1,0)
os.makedirs(outdir)
sp_global_def.write_command(outdir)
if(verbose == 1): finfo=open( os.path.join(outdir, "progress%04d.txt" % myid), "w" )
else: finfo = None
#print " before evenangles",myid
from sp_utilities import getvec
from numpy import array, reshape
refa = even_angles(delta)
nrefa = len(refa)
refnormal = zeros((nrefa,3),'float32')
tetref = [0.0]*nrefa
for i in range(nrefa):
tr = getvec( refa[i][0], refa[i][1] )
for j in range(3): refnormal[i][j] = tr[j]
tetref[i] = refa[i][1]
del refa
vct = array([0.0]*(3*nprj),'float32')
if myid == 0:
sxprint(" will read ",myid)
tr = EMUtil.get_all_attributes(prjfile,'xform.projection')
tetprj = [0.0]*nprj
for i in range(nprj):
temp = tr[i].get_params("spider")
tetprj[i] = temp["theta"]
if(tetprj[i] > 90.0): tetprj[i] = 180.0 - tetprj[i]
vct[3*i+0] = tr[i].at(2,0)
vct[3*i+1] = tr[i].at(2,1)
vct[3*i+2] = tr[i].at(2,2)
del tr
else:
tetprj = [0.0]*nprj
#print " READ ",myid
if MPI:
#print " will bcast",myid
vct = mpi.mpi_bcast( vct, len(vct), mpi.MPI_FLOAT, 0, mpi.MPI_COMM_WORLD )
from sp_utilities import bcast_list_to_all
tetprj = bcast_list_to_all(tetprj, myid, 0)
#print " reshape ",myid
vct = reshape(vct,(nprj,3))
assignments = [[] for i in range(nrefa)]
dspn = 1.25*delta
for k in range(nprj):
best_s = -1.0
best_i = -1
for i in range( nrefa ):
if(abs(tetprj[k] - tetref[i]) <= dspn):
s = abs(refnormal[i][0]*vct[k][0] + refnormal[i][1]*vct[k][1] + refnormal[i][2]*vct[k][2])
if s > best_s:
best_s = s
best_i = i
assignments[best_i].append(k)
am = len(assignments[0])
mufur = 1.0/am
for i in range(1,len(assignments)):
ti = len(assignments[i])
am = min(am, ti)
if(ti>0): mufur += 1.0/ti
del tetprj,tetref
dp = 1.0 - d # keep that many in each direction
keep = int(am*dp +0.5)
mufur = keep*nrefa/(1.0 - mufur*keep/float(nrefa))
if myid == 0:
sxprint(" Number of projections ",nprj,". Number of reference directions ",nrefa,", multiplicative factor for the variance ",mufur)
sxprint(" Minimum number of assignments ",am," Number of projections used per stratum ", keep," Number of projections in resampled structure ",int(am*dp +0.5)*nrefa)
if am <2 or am == keep:
sxprint("incorrect settings")
exit() # FIX
if(seedbase < 1):
seed()
jumpahead(17*myid+123)
else:
seed(seedbase)
jumpahead(17*myid+123)
volfile = os.path.join(outdir, "bsvol%04d.hdf" % myid)
from random import randint
niter = nvol/ncpu/nbufvol
for kiter in range(niter):
if(verbose == 1):
finfo.write( "Iteration %d: \n" % kiter )
finfo.flush()
iter_start = time()
# the following has to be converted to resample mults=1 means take given projection., mults=0 means omit
mults = [ [0]*nprj for i in range(nbufvol) ]
for i in range(nbufvol):
for l in range(nrefa):
mass = assignments[l][:]
shuffle(mass)
mass = mass[:keep]
mass.sort()
#print l, " * ",mass
for k in range(keep):
mults[i][mass[k]] = 1
'''
lout = []
for l in xrange(len(mults[i])):
if mults[i][l] == 1: lout.append(l)
write_text_file(lout, os.path.join(outdir, "list%04d_%03d.txt" %(i, myid)))
del lout
'''
del mass
rectors, fftvols, wgtvols = resample_prepare( prjfile, nbufvol, snr, CTF, npad )
resample_insert( bufprefix, fftvols, wgtvols, mults, CTF, npad, finfo )
del mults
resample_finish( rectors, fftvols, wgtvols, volfile, kiter, nprj, finfo )
rectors = None
fftvols = None
wgtvols = None
if(verbose == 1):
finfo.write( "time for iteration: %10.3f\n" % (time() - iter_start) )
finfo.flush()
if not options.outdir:
outdir = os.path.dirname(os.path.realpath(options.classavgs))
else:
outdir = options.outdir
if options.mode == 'viper':
selectdoc = options.classselect
elif options.mode == 'projmatch':
selectdoc = None
elif options.mode == 'meridien':
selectdoc = options.partselect
else:
sp_global_def.ERROR(
"\nERROR!! Valid mode not specified. Valid modes are: viper, projmatch, and meridien.",
__file__, 1)
sxprint('Type %s --help to see available options\n' %
os.path.basename(__file__))
exit()
main_proj_compare(options.classavgs,
options.vol3d,
outdir,
options,
mode=options.mode,
prjmethod=options.prjmethod,
classangles=options.classangles,
partangles=options.partangles,
selectdoc=selectdoc,
verbose=options.verbose,
displayYN=options.display)
sp_global_def.print_timestamp("Finish")
def main(args):
"""
Main function contains the logic behind the program.
Arguments:
args - Command line argument options
Returns:
None
"""
sp_global_def.BATCH = True
output_file = sanity_checks(args)
output_dtype = []
array_list = []
create_stack = False
particle_data = None
partres_data = None
params_2d_data = None
params_3d_data = None
params_3d_subset_data = None
if args.particle_stack:
sxprint("Import particle stack")
particle_data, create_stack = import_particle_stack(
args.particle_stack, args.output_directory,
args.relion_project_dir)
output_dtype.extend(particle_data.dtype.descr)
if args.partres_file:
sxprint("Import partres file")
partres_data = import_partres_file(args.partres_file)
output_dtype.extend(partres_data.dtype.descr)
if args.params_2d_file:
sxprint("Import params 2d file")
params_2d_data = import_params(args.params_2d_file, dim="2d")
output_dtype.extend(params_2d_data.dtype.descr)
if args.params_3d_file:
sxprint("Import params 3d file")
params_3d_data = import_params(args.params_3d_file, dim="3d")
output_dtype.extend(params_3d_data.dtype.descr)
if args.params_3d_chunk_file_0 != None and args.params_3d_chunk_file_1 != None:
sxprint("Import params 3d chunk files")
params_3d_subset_data = np.empty(params_3d_data.shape[0],
dtype=[("_rlnRandomSubset", "<i8")])
params_3d_subset_data.fill(np.nan)
params_import = []
for idx, file_name in enumerate(
[args.params_3d_chunk_file_0, args.params_3d_chunk_file_1]):
chunk_import = np.genfromtxt(file_name, int)
params_3d_subset_data["_rlnRandomSubset"][chunk_import] = idx + 1
params_import.extend(chunk_import.tolist())
output_dtype.extend(params_3d_subset_data.dtype.descr)
assert params_3d_subset_data.shape[0] == params_3d_data.shape[0]
assert np.unique(params_import).shape[0] == params_3d_data.shape[0]
if args.params_3d_index_file:
sxprint("Import params 3d index")
params_index_data = np.genfromtxt(args.params_3d_index_file, dtype=int)
assert params_3d_data.shape[0] == params_index_data.shape[0]
assert np.unique(params_index_data).shape[0] == params_3d_data.shape[0]
elif args.particle_stack:
params_index_data = np.arange(particle_data.shape[0])
else:
params_index_data = np.arange(partres_data.shape[0])
mask_array = np.ones(particle_data.shape[0], dtype=np.bool)
if args.list or args.exlist:
sxprint("Import list/exlist information")
mask_array = create_particle_data_mask(args.list, args.exlist,
particle_data.shape[0])
output_data = np.empty(params_index_data.shape[0],
dtype=sorted(list(set(output_dtype))))
particle_data_params = particle_data[params_index_data]
mask_array_params = mask_array[params_index_data]
array_list.append(particle_data_params)
array_list.append(params_2d_data)
array_list.append(params_3d_data)
array_list.append(params_3d_subset_data)
sxprint("Adjust header")
for array in array_list:
if array is not None:
for name in array.dtype.names:
output_data[name] = array[name]
if partres_data is not None:
for row in partres_data:
mask = output_data["_rlnMicrographName"] == row[
"_rlnMicrographName"]
for name in partres_data.dtype.names:
output_data[name][mask] = row[name]
final_output = output_data[mask_array_params]
sxprint("Write star file")
header = ["", "data_", "", "loop_"]
header.extend([
"{0} #{1}".format(name, idx + 1)
for idx, name in enumerate(final_output.dtype.names)
])
dtype_dict = final_output.dtype.fields
fmt = []
for name in final_output.dtype.names:
max_length = (len(
max([str(entry).split(".")[0] for entry in final_output[name]],
key=len)) + 2)
if "float" in str(dtype_dict[name][0]):
fmt.append("%{0}.6f".format(max_length + 7))
elif "int" in str(dtype_dict[name][0]):
fmt.append("%{0}d".format(max_length))
elif "|S" in str(dtype_dict[name][0]):
fmt.append("%{0}s".format(max_length))
else:
assert False
np.savetxt(
output_file,
final_output,
fmt=" ".join(fmt),
header="\n".join(header),
comments="",
)
if create_stack:
sxprint("Create particle stacks")
create_particle_stack(args.particle_stack, args.output_directory,
particle_data)
sxprint("Done!")
sp_global_def.BATCH = False
def import_params(params_file, dim):
"""
Import 2D or 3D parameters.
The 2d params file has the format:
angle, shift x, shift y, mirror
The 3d params file has the format:
angle_phi, angle_theta, angle_psi, tx, ty
Arguments:
params_2d_file - File name of the 2d parameter files
Returns:
parameter array
"""
if dim == "2d":
dtype_import_list = [
("angle_psi", float),
("shift_x", float),
("shift_y", float),
("mirror", int),
]
sxprint("What happens with mirror?")
elif dim == "3d":
dtype_import_list = [
("angle_rot", float),
("angle_theta", float),
("angle_psi", float),
("shift_x", float),
("shift_y", float),
]
else:
sp_global_def.ERROR(
"Dimension {0} not supported. Only '2d' and '3d' are supported.".
format(dim),
"sp_sphire2relion",
)
input_data = np.genfromtxt(params_file, dtype=dtype_import_list)
dtype_output_list = [
("_rlnOriginX", float),
("_rlnOriginY", float),
("_rlnAngleRot", float),
("_rlnAngleTilt", float),
("_rlnAnglePsi", float),
]
params_array = np.empty(len(input_data), dtype=sorted(dtype_output_list))
params_array["_rlnOriginX"] = input_data["shift_x"]
params_array["_rlnOriginY"] = input_data["shift_y"]
params_array["_rlnAnglePsi"] = input_data["angle_psi"]
if dim == "2d":
params_array["_rlnAngleTilt"] = 0
params_array["_rlnAngleRot"] = 0
elif dim == "3d":
params_array["_rlnAngleTilt"] = input_data["angle_theta"]
params_array["_rlnAngleRot"] = input_data["angle_rot"]
return params_array
def ctf_fit(im_1d,
bg_1d,
im_2d,
bg_2d,
voltage,
cs,
ac,
apix,
bgadj=0,
autohp=False):
"""Determines CTF parameters given power spectra produced by powspec_with_bg()
The bgadj option will result in adjusting the bg_1d curve to better match the zeroes
of the CTF (in which case bg_1d is modified in place)."""
# defocus estimation
global debug
ys = im_2d.get_ysize()
ds = 1.0 / (apix * ys)
ctf = EMAN2Ctf()
ctf.from_dict({
"defocus": 1.0,
"voltage": voltage,
"bfactor": 0.0,
"cs": cs,
"ampcont": ac,
"apix": apix,
"dsbg": ds,
"background": bg_1d
})
sf = sfact([i * ds for i in range(ys)], "ribosome", "nono")
#sxprint " SF ",sf
if debug: dfout = open("ctf.df.txt", "w")
dfbest1 = (0, -1.0e20)
for dfi in range(5, 128): # loop over defocus
ac = 10
df = dfi / 20.0
ctf.defocus = df
ctf.ampcont = ac
cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP)
st = .04 / ds
norm = 0
for fz in range(len(cc)):
if cc[fz] < 0: break
tot, totr = 0, 0
for s in range(int(st), ys / 2):
tot += (cc[s]**2) * (im_1d[s] - bg_1d[s])
totr += cc[s]**4
#for s in range(int(ys/2)): tot+=(cc[s*ctf.CTFOS]**2)*ps1d[-1][s]/norm
#for s in range(int(fz/ctf.CTFOS),ys/2): tot+=(cc[s*ctf.CTFOS]**2)*ps1d[-1][s]
#for s in range(int(fz/ctf.CTFOS),ys/2): tot+=(cc[s*ctf.CTFOS]**2)*snr[s]
tot /= sqrt(totr)
#tot/=totr
if tot > dfbest1[1]: dfbest1 = (df, tot)
try:
dfout.write("%1.2f\t%g\n" % (df, tot))
except:
pass
#out=file("bg1d2.txt","w")
#for a,b in enumerate(bg2): out.write("%1.4f\t%1.5f\n"%(a*ds,b))
#out.close()
dfbest = dfbest1
for dfi in range(-10, 10): # loop over defocus
df = dfi / 100.0 + dfbest1[0]
ctf.defocus = df
cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP)
st = .04 / ds
norm = 0
for fz in range(len(cc)):
#norm+=cc[fz]**2
if cc[fz] < 0: break
tot, totr = 0, 0
for s in range(int(st), ys / 2):
tot += (cc[s]**2) * (im_1d[s] - bg_1d[s])
totr += cc[s]**4
tot /= sqrt(totr)
if tot > dfbest[1]:
dfbest = (df, tot)
if debug: dfout.write("%1.2f\t%g\n" % (df, tot))
ctf.defocus = dfbest[0]
cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP)
Util.save_data(0, ds, cc, "ctf.ctf.txt")
if bgadj:
# now we try to construct a better background based on the CTF zeroes being zero
bg2 = bg_1d[:]
last = 0, 1.0
for x in range(1, len(bg2) - 1):
if cc[x] * cc[x + 1] < 0:
# we search +-1 point from the zero for the minimum
cur = (x,
min(im_1d[x] / bg_1d[x], im_1d[x - 1] / bg_1d[x - 1],
im_1d[x + 1] / bg_1d[x + 1]))
# once we have a pair of zeros we adjust the background values between
for xx in range(last[0], cur[0]):
w = (xx - last[0]) / float(cur[0] - last[0])
bg_1d[xx] = bg2[xx] * (cur[1] * w + last[1] * (1.0 - w))
# sxprint xx,"\t",(cur[1]*w+last[1]*(1.0-w)) #,"\t",cur[1],last[1]
last = cur
# cover the area from the last zero crossing to the end of the curve
for xx in range(last[0], len(bg2)):
bg_1d[xx] = bg2[xx] * last[1]
snr = [snr_safe(im_1d[i], bg_1d[i]) for i in range(len(im_1d))]
# This will dramatically reduce the intensity of the initial sharp peak found in almost all single particle data
# this applies to the SNR curve only, downweighting the importance of this section of the spectrum without actually
# removing the information by filtering the image data. It will, of course also impact Wiener filters.
if autohp:
for x in range(2, len(snr) - 2):
if snr[x] > snr[x + 1] and snr[x + 1] < snr[x + 2]:
break # we find the first minimum
snr1max = max(snr[1:x]) # find the intensity of the first peak
snr2max = max(snr[x + 2:len(snr) /
2]) # find the next highest snr peak
qtmp = 0.5 * snr2max / snr1max
for xx in range(1, x + 1):
snr[xx] *= qtmp # scale the initial peak to 50% of the next highest peak
# store the final results
ctf.snr = snr
ctf.defocus = dfbest[0]
# This smooths the SNR curve
#snr=ctf.compute_1d(ys,ds,Ctf.CtfType.CTF_SNR_SMOOTH)
#snr=snr[:len(ctf.snr)]
#ctf.snr=snr
# Last, we try to get a decent B-factor
# This is a quick hack and not very efficiently coded
bfs = [
0.0, 50.0, 100.0, 200.0, 400.0, 600.0, 800.0, 1200.0, 1800.0, 2500.0,
4000.0
]
best = (0, 0)
s0 = int(.05 / ds) + 1
s1 = min(int(0.14 / ds), len(bg_1d) - 1)
sxprint(" FREQ RANGE", s0, s1)
for b in range(1, len(bfs) - 1):
ctf.bfactor = bfs[b]
cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP)
sf = sfact([ds * i for i in range(len(cc))], "ribosome", "original")
cc = [sf[i] * cc[i]**2 for i in range(len(cc))]
# adjust the amplitude to match well
a0, a1 = 0, 0
for s in range(s0, s1):
a0 += cc[s]
a1 += fabs(im_1d[s] - bg_1d[s])
if a1 == 0: a1 = 1.0
a0 /= a1
cc = [i / a0 for i in cc]
er = 0
# compute the error
for s in range(s0, len(bg_1d) - 1):
er += (cc[s] - im_1d[s] + bg_1d[s])**2
if best[0] == 0 or er < best[0]: best = (er, b)
# Stupid replication here, in a hurry
bb = best[1]
best = (best[0], bfs[best[1]])
for b in range(20):
ctf.bfactor = bfs[bb - 1] * (1.0 - b / 20.0) + bfs[bb + 1] * (b / 20.0)
cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP)
sf = sfact([i * ds for i in range(len(cc))], "ribosome", "original")
cc = [sf[i] * cc[i]**2 for i in range(len(cc))]
# adjust the amplitude to match well
a0, a1 = 0, 0
for s in range(s0, s1):
a0 += cc[s]
a1 += fabs(im_1d[s] - bg_1d[s])
if a1 == 0: a1 = 1.0
a0 /= a1
cc = [i / a0 for i in cc]
er = 0
# compute the error
for s in range(s0, len(bg_1d) - 1):
er += (cc[s] - im_1d[s] + bg_1d[s])**2
if best[0] == 0 or er < best[0]:
best = (er, ctf.bfactor)
sxprint(" BEST ", best)
#for s in xrange(s0,len(bg_1d)-1):
# sxprint s,cc[s],im_1d[s]-bg_1d[s],sfact(ds*s)
#for i in xrange(3*len(cc)):
# sxprint ds*i," ",sfact(ds*i)
# sxprint bfs[b],best
ctf.bfactor = best[1]
if 1:
sxprint("Best DF = %1.3f B-factor = %1.0f" %
(dfbest[0], ctf.bfactor))
return ctf
def main():
global debug, logid
progname = os.path.basename(sys.argv[0])
usage = """%prog [options] <input stack/image> ...
Various CTF-related operations on images, including automatic fitting. Note that automatic fitting is limited to 5 microns
underfocus at most. Input particles should be unmasked and unfiltered. A minimum of ~20% padding around the
particles is required for background extraction, even if this brings the edge of another particle into the box in some cases.
Particles should be reasonably well centered. Can also optionally phase flip and Wiener filter particles. Wiener filtration comes
after phase-flipping, so if phase flipping is performed Wiener filtered particles will also be phase-flipped. Note that both
operations are performed on oversampled images if specified (though final real-space images are clipped back to their original
size. Increasing padding during the particle picking process will improve the accuracy of phase-flipping, particularly for
images far from focus."""
parser = OptionParser(usage=usage, version=EMANVERSION)
parser.add_option("--gui",
action="store_true",
help="Start the GUI for interactive fitting",
default=False)
parser.add_option("--auto_fit",
action="store_true",
help="Runs automated CTF fitting on the input images",
default=False)
parser.add_option(
"--bgmask",
type="int",
help=
"Compute the background power spectrum from the edge of the image, specify a mask radius in pixels which would largely mask out the particles. Default is boxsize/2.",
default=0)
parser.add_option("--apix",
type="float",
help="Angstroms per pixel for all images",
default=0)
parser.add_option("--voltage",
type="float",
help="Microscope voltage in KV",
default=0)
parser.add_option("--cs",
type="float",
help="Microscope Cs (spherical aberation)",
default=0)
parser.add_option("--ac",
type="float",
help="Amplitude contrast (percentage, default=10)",
default=10)
parser.add_option(
"--autohp",
action="store_true",
help=
"Automatic high pass filter of the SNR only to remove initial sharp peak, phase-flipped data is not directly affected (default false)",
default=False)
#parser.add_option("--invert",action="store_true",help="Invert the contrast of the particles in output files (default false)",default=False)
parser.add_option("--nonorm",
action="store_true",
help="Suppress per image real-space normalization",
default=False)
parser.add_option(
"--nosmooth",
action="store_true",
help=
"Disable smoothing of the background (running-average of the log with adjustment at the zeroes of the CTF)",
default=False)
#parser.add_option("--phaseflip",action="store_true",help="Perform phase flipping after CTF determination and writes to specified file.",default=False)
#parser.add_option("--wiener",action="store_true",help="Wiener filter (optionally phaseflipped) particles.",default=False)
parser.add_option("--oversamp",
type="int",
help="Oversampling factor",
default=1)
parser.add_option(
"--sf",
type="string",
help=
"The name of a file containing a structure factor curve. Can improve B-factor determination.",
default=None)
parser.add_option("--debug", action="store_true", default=False)
(options, args) = parser.parse_args()
if len(args) < 1: parser.error("Input image required")
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
if options.auto_fit:
if options.voltage == 0: parser.error("Please specify voltage")
if options.cs == 0: parser.error("Please specify Cs")
if options.apix == 0: sxprint("Using A/pix from header")
debug = options.debug
global sfcurve
if options.sf:
sfcurve = XYData()
sfcurve.read_file(options.sf)
logid = E2init(sys.argv)
# if options.oversamp>1 : options.apix/=float(options.oversamp)
db_project = db_open_dict("bdb:project")
db_parms = db_open_dict("bdb:e2ctf.parms")
db_misc = db_open_dict("bdb:e2ctf.misc")
options.filenames = args
### Power spectrum and CTF fitting
if options.auto_fit:
img_sets = pspec_and_ctf_fit(
options,
debug) # converted to a function so to work with the workflow
### This computes the intensity of the background subtracted power spectrum at each CTF maximum for all sets
global envelopes # needs to be a global for the Simplex minimizer
# envelopes is essentially a cache of information that could be useful at later stages of refinement
# as according to Steven Ludtke
for i in img_sets:
envelopes.append(ctf_env_points(i[2], i[3], i[1]))
# we use a simplex minimizer to try to rescale the individual sets to match as best they can
scales = [1.0] * len(img_sets)
if (len(img_sets) > 3):
incr = [0.2] * len(img_sets)
simp = Simplex(env_cmp, scales, incr)
scales = simp.minimize(maxiters=1000)[0]
# sxprint scales
sxprint(" ")
# apply the final rescaling
envelope = []
for i in range(len(scales)):
cur = envelopes[i]
for j in range(len(cur)):
envelope.append((cur[j][0], cur[j][1] * scales[i]))
envelope.sort()
envelope = [i for i in envelope
if i[1] > 0] # filter out all negative peak values
db_misc = db_open_dict("bdb:e2ctf.misc")
db_misc["envelope"] = envelope
#db_close_dict("bdb:e2ctf.misc")
#out=file("envelope.txt","w")
#for i in envelope: out.write("%f\t%f\n"%(i[0],i[1]))
#out.close()
### GUI - user can update CTF parameters interactively
if options.gui:
img_sets = get_gui_arg_img_sets(options.filenames)
if len(img_sets) == 0:
E2end(logid)
ERROR("img_sets == 0")
return
app = EMApp()
gui = GUIctf(app, img_sets)
gui.show_guis()
app.exec_()
sxprint("done execution")
### Process input files
#if debug : print "Phase flipping / Wiener filtration"
# write wiener filtered and/or phase flipped particle data to the local database
#if options.phaseflip or options.wiener: # only put this if statement here to make the program flow obvious
# write_e2ctf_output(options) # converted to a function so to work with the workflow
E2end(logid)
def powspec_with_bg(stackfile, radius=0, edgenorm=True, oversamp=1):
"""This routine will read the images from the specified file, optionally edgenormalize,
then apply a gaussian mask with the specified radius then compute the average 2-D power
spectrum for the stack. It will also compute the average 2-D power spectrum using 1-mask + edge
apotization to get an appoximate 'background' power spectrum. 2-D results returned as a 2-D FFT
intensity/0 image. 1-D results returned as a list of floats.
returns a 4-tuple with spectra for (1d particle,1d background,2d particle,2d background)
"""
global masks
im = EMData()
im.read_image(stackfile, 0)
ys = im.get_ysize() * oversamp
ys2 = im.get_ysize()
n = EMUtil.get_image_count(stackfile)
if (radius == 0): radius = ys2 // 2 - ys2 // 4
# set up the inner and outer Gaussian masks
try:
mask1, ratio1, mask2, ratio2 = masks[(ys, radius)]
except:
mask1 = EMData(ys2, ys2, 1)
mask1.to_one()
#mask1.process_inplace("mask.gaussian",{"outer_radius":radius})
mask1.process_inplace("mask.gaussian", {
"inner_radius": radius,
"outer_radius": ys2 // 10
})
mask2 = mask1.copy() * -1 + 1
# mask1.process_inplace("mask.decayedge2d",{"width":4})
#mask2.process_inplace("mask.decayedge2d",{"width":4})
mask1.clip_inplace(
Region(-(ys2 * (oversamp - 1) / 2), -(ys2 * (oversamp - 1) / 2),
ys, ys))
mask2.clip_inplace(
Region(-(ys2 * (oversamp - 1) / 2), -(ys2 * (oversamp - 1) / 2),
ys, ys))
ratio1 = mask1.get_attr("square_sum") / (ys * ys) #/1.035
ratio2 = mask2.get_attr("square_sum") / (ys * ys)
masks[(ys, radius)] = (mask1, ratio1, mask2, ratio2)
sxprint(" RATIOS ", ratio1, ratio2, " ", radius)
#mask1.write_image("mask1.hdf",0)
#mask2.write_image("mask2.hdf",0)
pav1 = model_blank(ys2, ys2)
pva1 = model_blank(ys2, ys2)
pav2 = model_blank(ys2, ys2)
pva2 = model_blank(ys2, ys2)
pf = float(ys2 * ys2)**2 / 4.0
fofo = []
for i in range(n):
im1 = EMData()
im1.read_image(stackfile, i)
# im1=EMData(stackfile,i)
#info(im1)
#if edgenorm : im1.process_inplace("normalize.edgemean")
if oversamp > 1:
im1.clip_inplace(
Region(-(ys2 * (oversamp - 1) / 2),
-(ys2 * (oversamp - 1) / 2), ys, ys))
im2 = im1.copy()
im1 *= mask1
pp = periodogram(im1) * pf / ratio1
fofo.append(rot_avg_table(pp))
Util.add_img(pav1, pp)
Util.add_img2(pva1, pp)
imf = im1.do_fft()
#sxprint_col(imf,0)
imf.ri2inten()
#info(imf)
imf.set_complex(0)
#sxprint_col(imf,0)
#imf.write_image("imf.hdf",0)
#pop = periodogram(im1)
#sxprint_col(pop,90)
#pop.write_image("pop.hdf",0)
if i == 0: av1 = imf
else: av1 += imf
im2 *= mask2
pp = periodogram(im2) * pf / ratio2
Util.add_img(pav2, pp)
Util.add_img2(pva2, pp)
imf = im2.do_fft()
imf.ri2inten()
if i == 0: av2 = imf
else: av2 += imf
av1 /= (float(n) * av1.get_ysize() * av1.get_ysize() * ratio1)
av1.set_value_at(0, 0, 0.0)
av1.set_complex(1)
av1.set_attr("is_intensity", 1)
av2 /= (float(n) * av2.get_ysize() * av2.get_ysize() * ratio2)
av2.set_value_at(0, 0, 0.0)
av2.set_complex(1)
av2.set_attr("is_intensity", 1)
av1_1d = av1.calc_radial_dist(av1.get_ysize() / 2, 0.0, 1.0, 1)
av2_1d = av2.calc_radial_dist(av2.get_ysize() / 2, 0.0, 1.0, 1)
pav1 /= n
pva1 = square_root((pva1 - pav1 * pav1 * n) / (n - 1) / n)
pav2 /= n
pva2 = square_root((pva2 - pav2 * pav2 * n) / (n - 1) / n)
pav1.write_image("av1.hdf", 0)
pav2.write_image("av2.hdf", 0)
pva1.write_image("va1.hdf", 0)
pva2.write_image("va2.hdf", 0)
write_text_file([
list(range(ys2 // 2 + 1)),
rot_avg_table(pav1),
rot_avg_table(pva1),
rot_avg_table(pav2),
rot_avg_table(pva2)
], "pwsa.txt")
#write_text_file([range(ys2//2+1), fofo[0], fofo[1], fofo[2], fofo[3], fofo[4], fofo[5]],"ipwsa.txt")
return (av1_1d, av2_1d, av1, av2)
def multi_align_stability(
ali_params,
mir_stab_thld=0.0,
grp_err_thld=10000.0,
err_thld=1.732,
print_individual=False,
d=64,
):
def sqerr(a):
n = len(a)
avg = sum(a)
sq = 0.0
for i in range(n):
sq += a[i]**2
return (sq - avg * avg / n) / n
# args - G, data - [T, d]
def func(args, data, return_avg_pixel_error=True):
ali_params = data[0]
d = data[1]
L = len(ali_params)
N = len(ali_params[0]) / 4
args_list = [0.0] * (L * 3)
for i in range(L * 3 - 3):
args_list[i] = args[i]
cosa = [0.0] * L
sina = [0.0] * L
for i in range(L):
cosa[i] = numpy.cos(numpy.radians(args_list[i * 3]))
sina[i] = numpy.sin(numpy.radians(args_list[i * 3]))
sqr_pixel_error = [0.0] * N
ave_params = []
for i in range(N):
sum_cosa = 0.0
sum_sina = 0.0
sx = [0.0] * L
sy = [0.0] * L
for j in range(L):
if int(ali_params[j][i * 4 + 3]) == 0:
sum_cosa += numpy.cos(
numpy.radians(args_list[j * 3] + ali_params[j][i * 4]))
sum_sina += numpy.sin(
numpy.radians(args_list[j * 3] + ali_params[j][i * 4]))
sx[j] = (args_list[j * 3 + 1] +
ali_params[j][i * 4 + 1] * cosa[j] +
ali_params[j][i * 4 + 2] * sina[j])
sy[j] = (args_list[j * 3 + 2] -
ali_params[j][i * 4 + 1] * sina[j] +
ali_params[j][i * 4 + 2] * cosa[j])
else:
sum_cosa += numpy.cos(
numpy.radians(-args_list[j * 3] +
ali_params[j][i * 4]))
sum_sina += numpy.sin(
numpy.radians(-args_list[j * 3] +
ali_params[j][i * 4]))
sx[j] = (-args_list[j * 3 + 1] +
ali_params[j][i * 4 + 1] * cosa[j] -
ali_params[j][i * 4 + 2] * sina[j])
sy[j] = (args_list[j * 3 + 2] +
ali_params[j][i * 4 + 1] * sina[j] +
ali_params[j][i * 4 + 2] * cosa[j])
sqrtP = numpy.sqrt(sum_cosa**2 + sum_sina**2)
sqr_pixel_error[i] = max(
0.0, d * d / 4.0 * (1 - sqrtP / L) + sqerr(sx) + sqerr(sy))
# Get ave transform params
H = EMAN2_cppwrap.Transform({"type": "2D"})
H.set_matrix([
sum_cosa / sqrtP,
sum_sina / sqrtP,
0.0,
sum(sx) / L,
-sum_sina / sqrtP,
sum_cosa / sqrtP,
0.0,
sum(sy) / L,
0.0,
0.0,
1.0,
0.0,
])
dd = H.get_params("2D")
# We are using here mirror of the LAST SET.
H = EMAN2_cppwrap.Transform({
"type":
"2D",
"alpha":
dd["alpha"],
"tx":
dd["tx"],
"ty":
dd["ty"],
"mirror":
int(ali_params[L - 1][i * 4 + 3]),
"scale":
1.0,
})
dd = H.get_params("2D")
ave_params.append([dd["alpha"], dd["tx"], dd["ty"], dd["mirror"]])
# Warning: Whatever I return here is squared pixel error, this is for the easy expression of derivative
# Don't forget to square root it after getting the value
if return_avg_pixel_error:
return sum(sqr_pixel_error) / N
else:
return sqr_pixel_error, ave_params
"""Multiline Comment0"""
# I decided not to use scipy in order to reduce the dependency, I wrote the C++ code instead
# from scipy import array, int32
# from scipy.optimize.lbfgsb import fmin_l_bfgs_b
# Find out the subset which is mirror stable over all runs
all_part = []
num_ali = len(ali_params)
nima = len(ali_params[0]) / 4
for i in range(num_ali):
mirror0 = []
mirror1 = []
for j in range(nima):
if ali_params[i][j * 4 + 3] == 0:
mirror0.append(j)
else:
mirror1.append(j)
mirror0 = numpy.array(mirror0, "int32")
mirror1 = numpy.array(mirror1, "int32")
all_part.append([mirror0, mirror1])
match, stab_part, CT_s, CT_t, ST, st = sp_statistics.k_means_stab_bbenum(
all_part, T=0, nguesses=1)
mir_stab_part = stab_part[0] + stab_part[1]
mir_stab_rate = len(mir_stab_part) / float(nima)
if mir_stab_rate <= mir_stab_thld:
return [], mir_stab_rate, -1.0
mir_stab_part.sort()
del all_part, match, stab_part, CT_s, CT_t, ST, st
# for i in xrange(len(mir_stab_part)): print i, mir_stab_part[i]
nima2 = len(mir_stab_part)
# print " mirror stable ",nima2
# Keep the alignment parameters of mirror stable particles
ali_params_mir_stab = [[] for i in range(num_ali)]
for i in range(num_ali):
for j in mir_stab_part:
ali_params_mir_stab[i].extend(ali_params[i][j * 4:j * 4 + 4])
# Find out the alignment parameters for each iteration against the last one
args = []
for i in range(num_ali - 1):
alpha, sx, sy, mirror = align_diff_params(
ali_params_mir_stab[i], ali_params_mir_stab[num_ali - 1])
args.extend([alpha, sx, sy])
# Do an initial analysis, purge all outlier particles, whose pixel error are larger than three times the threshold
data = [ali_params_mir_stab, d]
pixel_error_before, ave_params = func(numpy.array(args),
data,
return_avg_pixel_error=False)
# We have to return mir_stab_rate (see above), even if the group does not survive it and the pixel error before cleaning,
# see below, and in ISAC print the user the overall statistics (histograms?)
# so one can get on overall idea how good/bad data is. PAP 01/25/2015
# print " >>> ",sqrt(sum(pixel_error_before)/nima2)
ali_params_cleaned = [[] for i in range(num_ali)]
cleaned_part = []
for j in range(nima2):
pixel_error_before[j] = max(0.0,
pixel_error_before[j]) # prevent sqrt of 0
if numpy.sqrt(pixel_error_before[j]) > 3 * err_thld:
pass # print " removed ",3*err_thld,j,sqrt(pixel_error_before[j])
else:
cleaned_part.append(mir_stab_part[j])
for i in range(num_ali):
ali_params_cleaned[i].extend(
ali_params_mir_stab[i][j * 4:j * 4 + 4])
nima3 = len(cleaned_part)
prever = numpy.sqrt(sum(pixel_error_before) / nima2)
if nima3 <= 1:
return [], mir_stab_rate, prever
# print " cleaned part ",nima3
# Use LBFGSB to minimize the sum of pixel errors
data = [ali_params_cleaned, d]
# Use Python code
# ps_lp, val, d = fmin_l_bfgs_b(func, array(args), args=[data], fprime=dfunc, bounds=None, m=10, factr=1e3, pgtol=1e-4, iprint=-1, maxfun=100)
# Use C++ code
ali_params_cleaned_list = []
for params in ali_params_cleaned:
ali_params_cleaned_list.extend(params)
results = EMAN2_cppwrap.Util.multi_align_error(args,
ali_params_cleaned_list, d)
ps_lp = results[:-1]
# Negative val can happen in some rare cases, it should be due to rounding errors,
# because all results show the val is about 1e-13.
# print "Strange results"
# print "args =", args
# print "ali_params_cleaned_list =", ali_params_cleaned_list
# print "results = ", results
val = max(0.0, results[-1])
del ali_params_cleaned_list
if numpy.sqrt(val) > grp_err_thld:
return [], mir_stab_rate, numpy.sqrt(val)
pixel_error_after, ave_params = func(ps_lp,
data,
return_avg_pixel_error=False)
stable_set = []
val = 0.0
for i in range(nima):
if i in cleaned_part:
j = cleaned_part.index(i)
err = numpy.sqrt(pixel_error_after[j])
if err < err_thld:
stable_set.append([err, i, ave_params[j]])
val += err
if print_individual:
sp_global_def.sxprint(
"Particle %4d : pixel error = %18.4f" % (i, err))
else:
if print_individual:
sp_global_def.sxprint(
"Particle %4d : pixel error = %18.4f unstable" %
(i, err))
else:
if print_individual:
sp_global_def.sxprint("Particle %4d : Mirror unstable" % i)
# return average pixel error before pruning as it is more informative
return stable_set, mir_stab_rate, prever # sqrt(val/len(stable_set))
def main_proj_compare(classavgstack,
reconfile,
outdir,
options,
mode='viper',
prjmethod='trilinear',
classangles=None,
partangles=None,
selectdoc=None,
verbose=False,
displayYN=False):
"""
Main function overseeing various projection-comparison modes.
Arguments:
classavgstack : Input image stack
reconfile : Map of which to generate projections (an optionally perform alignment)
outdir : Output directory
mode : Mode, viper (pre-existing angles for each input image), projmatch (angles from internal projection-matching)
verbose : (boolean) Whether to write additional information to screen
options : (list) Command-line options, run 'sxproj_compare.py -h' for an exhaustive list
classangles : Angles and shifts for each input class average
partangles : Angles and shifts for each particle (mode meridien)
selectdoc : Selection file for included images
prjmethod : Interpolation method to use
displayYN : (boolean) Whether to automatically open montage
"""
# Expand path for outputs
refprojstack = os.path.join(outdir, 'refproj.hdf')
refanglesdoc = os.path.join(outdir, 'refangles.txt')
outaligndoc = os.path.join(outdir, 'docalign2d.txt')
# If not an input, will create an output, in modes projmatch
if classangles == None:
classangles = os.path.join(outdir, 'docangles.txt')
# You need either input angles (mode viper) or to calculate them on the fly (mode projmatch)
if mode == 'viper':
sp_global_def.ERROR(
"\nERROR!! Input alignment parameters not specified.",
__file__, 1)
sxprint('Type %s --help to see available options\n' %
os.path.basename(__file__))
exit()
# Check if inputs exist
check(classavgstack, verbose=verbose)
check(reconfile, verbose=verbose)
if verbose: sxprint('')
# Check that dimensions of images and volume agree (maybe rescale volume)
voldim = EMAN2.EMData(reconfile).get_xsize()
imgdim = EMAN2.EMData(classavgstack, 0).get_xsize()
if voldim != imgdim:
sp_global_def.ERROR(
"\nERROR!! Dimension of input volume doesn't match that of image stack: %s vs. %s"
% (voldim, imgdim), __file__, 1)
scale = float(
imgdim
) / voldim # only approximate, since full-sized particle radius is arbitrary
msg = 'The command to resize the volume will be of the form:\n'
msg += 'e2proc3d.py %s resized_vol.hdf --scale=%1.5f --clip=%s,%s,%s\n' % (
reconfile, scale, imgdim, imgdim, imgdim)
msg += 'Check the file in the ISAC directory named "README_shrink_ratio.txt" for confirmation.\n'
sxprint(msg)
exit()
# Here if you want to be fancy, there should be an option to chose the projection method,
# the mechanism can be copied from sxproject3d.py PAP
if prjmethod == 'trilinear':
method_num = 1
elif prjmethod == 'gridding':
method_num = -1
elif prjmethod == 'nn':
method_num = 0
else:
sp_global_def.ERROR(
"\nERROR!! Valid projection methods are: trilinear (default), gridding, and nn (nearest neighbor).",
__file__, 1)
sxprint('Usage:\n%s' % USAGE)
exit()
# Set output directory and log file name
log, verbose = prepare_outdir_log(outdir, verbose)
# In case class averages include discarded images, apply selection file
if mode == 'viper':
if selectdoc:
goodavgs, extension = os.path.splitext(
os.path.basename(classavgstack))
newclasses = os.path.join(outdir, goodavgs + "_kept" + extension)
# e2proc2d appends to existing files, so rename existing output
if os.path.exists(newclasses):
renamefile = newclasses + '.bak'
print_log_msg(
"Selected-classes stack %s exists, renaming to %s" %
(newclasses, renamefile), log, verbose)
print_log_msg("mv %s %s\n" % (newclasses, renamefile), log,
verbose)
os.rename(newclasses, renamefile)
print_log_msg(
'Creating subset of %s to %s based on selection list %s' %
(classavgstack, newclasses, selectdoc), log, verbose)
cmd = "e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses,
selectdoc)
print_log_msg(cmd, log, verbose)
os.system(cmd)
sxprint('')
# Update class-averages
classavgstack = newclasses
# align de novo to reference map
if mode == 'projmatch':
# Generate reference projections
print_log_msg(
'Projecting %s to output %s using an increment of %s degrees using %s symmetry'
% (reconfile, refprojstack, options.delta, options.symmetry), log,
verbose)
cmd = 'sxproject3d.py %s %s --delta=%s --method=S --phiEqpsi=Minus --symmetry=%s' % (
reconfile, refprojstack, options.delta, options.symmetry)
if options.prjmethod == 'trilinear': cmd += ' --trilinear'
cmd += '\n'
print_log_msg(cmd, log, verbose)
project3d(reconfile,
refprojstack,
delta=options.delta,
symmetry=options.symmetry)
# Export projection angles
print_log_msg(
"Exporting projection angles from %s to %s" %
(refprojstack, refanglesdoc), log, verbose)
cmd = "sp_header.py %s --params=xform.projection --import=%s\n" % (
refprojstack, refanglesdoc)
print_log_msg(cmd, log, verbose)
header(refprojstack, 'xform.projection', fexport=refanglesdoc)
# Perform multi-reference alignment
if options.align == 'ali2d':
projdir = os.path.join(
outdir, 'Projdir') # used if input angles no provided
if os.path.isdir(projdir):
print_log_msg('Removing pre-existing directory %s' % projdir,
log, verbose)
print_log_msg('rm -r %s\n' % projdir, log, verbose)
shutil.rmtree(
projdir) # os.rmdir only removes empty directories
# Zero out alignment parameters in header
print_log_msg(
'Zeroing out alignment parameters in header of %s' %
classavgstack, log, verbose)
cmd = 'sxheader.py %s --params xform.align2d --zero\n' % classavgstack
print_log_msg(cmd, log, verbose)
header(classavgstack, 'xform.align2d', zero=True)
# Perform multi-reference alignment
msg = 'Aligning images in %s to projections %s with a radius of %s and a maximum allowed shift of %s' % (
classavgstack, refprojstack, options.matchrad,
options.matchshift)
print_log_msg(msg, log, verbose)
cmd = 'sxmref_ali2d.py %s %s %s --ou=%s --xr=%s --yr=%s\n' % (
classavgstack, refprojstack, projdir, options.matchrad,
options.matchshift, options.matchshift)
print_log_msg(cmd, log, verbose)
mref_ali2d(classavgstack,
refprojstack,
projdir,
ou=options.matchrad,
xrng=options.matchshift,
yrng=options.matchshift)
# Export alignment parameters
print_log_msg(
'Exporting angles from %s into %s' %
(classavgstack, classangles), log, verbose)
cmd = "sp_header.py %s --params=xform.align2d --export=%s\n" % (
classavgstack, classangles)
print_log_msg(cmd, log, verbose)
header(classavgstack, 'xform.align2d', fexport=classangles)
# By default, use AP SH
else:
apsh(refprojstack,
classavgstack,
outangles=classangles,
refanglesdoc=refanglesdoc,
outaligndoc=outaligndoc,
outerradius=options.matchrad,
maxshift=options.matchshift,
ringstep=options.matchstep,
log=log,
verbose=verbose)
# Diagnostic
alignlist = read_text_row(
classangles) # contain 2D alignment parameters
nimg1 = EMAN2.EMUtil.get_image_count(classavgstack)
assert len(alignlist) == nimg1, "MRK_DEBUG"
# Get alignment parameters from MERIDIEN
if mode == 'meridien':
continueTF = True # Will proceed unless some information is missing
if not partangles:
sp_global_def.ERROR(
"\nERROR!! Input alignment parameters not provided.", __file__,
1)
continueTF = False
if not continueTF:
sxprint('Type %s --help to see available options\n' %
os.path.basename(__file__))
exit()
if not options.classdocs or options.outliers:
classdir = os.path.join(outdir, 'Byclass')
if not os.path.isdir(classdir): os.makedirs(classdir)
if options.outliers:
goodclassparttemplate = os.path.join(
classdir, 'goodpartsclass{0:03d}.txt')
else:
goodclassparttemplate = None
if not options.classdocs:
classmap = os.path.join(classdir, 'classmap.txt')
classdoc = os.path.join(classdir, 'docclass{0:03d}.txt')
options.classdocs = os.path.join(classdir, 'docclass*.txt')
# Separate particles by class
vomq(classavgstack,
classmap,
classdoc,
log=log,
verbose=verbose)
mode_meridien(reconfile,
classavgstack,
options.classdocs,
partangles,
selectdoc,
options.refineshift,
options.refinerad,
classangles,
outaligndoc,
interpolation_method=method_num,
outliers=options.outliers,
goodclassparttemplate=goodclassparttemplate,
alignopt=options.align,
ringstep=options.refinestep,
log=log,
verbose=verbose)
# Import Euler angles
print_log_msg(
"Importing parameter information into %s from %s" %
(classavgstack, classangles), log, verbose)
cmd = "sp_header.py %s --params=xform.projection --import=%s\n" % (
classavgstack, classangles)
print_log_msg(cmd, log, verbose)
header(classavgstack, 'xform.projection', fimport=classangles)
# Make comparison stack between class averages (images 0,2,4,...) and re-projections (images 1,3,5,...)
compstack = compare_projs(reconfile,
classavgstack,
classangles,
outdir,
interpolation_method=method_num,
log=log,
verbose=verbose)
# Optionally pop up e2display
if displayYN:
sxprint('Opening montage')
cmd = "e2display.py %s\n" % compstack
sxprint(cmd)
os.system(cmd)
sxprint("Done!")
nsigma=s_nsigma,
threshold=s_density_threshold,
ndilation=command_args.s_ndilation,
nerosion=command_args.s_nerosion,
edge_width=command_args.s_edge_width,
allow_disconnected=command_args.s_allow_disconnected,
mode=mode,
do_approx=command_args.s_do_old,
do_fill=command_args.s_fill_mask,
do_print=True,
)
if command_args.s_invert:
s_mask = 1 - s_mask
sp_global_def.sxprint("Write outputs.")
mask_first.write_image(output_prefix + "_mask_first.hdf")
s_mask.write_image(output_prefix + "_mask_second.hdf")
masked_combined = mask_first * s_mask
masked_combined.write_image(output_prefix + "_mask.hdf")
else:
sp_global_def.sxprint("Write outputs.")
mask_first.write_image(output_prefix + "_mask.hdf")
if __name__ == "__main__":
sp_global_def.print_timestamp("Start")
sp_global_def.BATCH = True
main()
sp_global_def.BATCH = False
sp_global_def.sxprint("Done!")
sp_global_def.print_timestamp("Finish")
def run_unblur(
unblur_path,
input_image,
input_dir,
output_dir,
corrected_path,
uncorrected_path,
shift_path,
frc_path,
temp_path,
log_path,
file_list,
options,
):
# Lists to write the text files later
micrograph_list = []
shift_list = []
if options.save_frames:
frames_list = []
# If micrograph list is provided just process the images in the list
mic_list = options.selection_list
if mic_list:
# Import list file
try:
set_selection = numpy.genfromtxt(mic_list, dtype=None)
except TypeError:
sp_global_def.ERROR("no entrys in list file {0}".format(mic_list))
# List of files which are in pattern and list
file_list = [
entry for entry in file_list if
entry[len(input_dir):] in set_selection and os.path.exists(entry)
]
# If no match is there abort
if len(file_list) == 0:
sp_global_def.ERROR(
"no files in {0} matched the file pattern:\n".format(mic_list),
1)
# Get the number of files
nr_files = len(file_list)
# Timeing stuff
time_start = time.time()
time_list = []
# Loop over all files
for index, inputfile in enumerate(sorted(file_list)):
# Check, if there is an prefix and suffix.
# If there is more then one entry: the suffix is the last one.
# Otherwhise its just the one after the dot.
input_suffix = inputfile.split("/")[-1].split(".")[-1]
# First output to introduce the programm
if index == 0:
sp_global_def.sxprint(
"Progress: 0.0%; Time: --h:--m:--s/--h:--m:--s; Unblur started!"
)
# Time begin
t1 = time.time()
# Get the output names
file_name = inputfile[len(input_dir):-len(input_suffix) - 1]
if options.skip_dose_filter:
micrograph_name = "{0}/{1}{2}.mrc".format(uncorrected_path,
file_name,
options.sum_suffix)
frames_name = "{0}/{1}{2}.mrc".format(uncorrected_path, file_name,
options.frames_suffix)
frc_name = "{0}/{1}{2}.txt".format(frc_path, file_name,
options.frc_suffix)
else:
micrograph_name = "{0}/{1}{2}.mrc".format(corrected_path,
file_name,
options.sum_suffix)
frames_name = "{0}/{1}{2}.mrc".format(corrected_path, file_name,
options.frames_suffix)
micrograph_name_skip = "{0}/{1}{2}.mrc".format(
uncorrected_path, file_name, options.sum_suffix)
frames_name_skip = "{0}/{1}{2}.mrc".format(uncorrected_path,
file_name,
options.frames_suffix)
frc_name = "{0}/{1}{2}.txt".format(frc_path, file_name,
options.frc_suffix)
frc_summovie_name = "{0}/{1}_summovie{2}.txt".format(
frc_path, file_name, options.frc_suffix)
shift_name = "{0}/{1}{2}.txt".format(shift_path, file_name,
options.shift_suffix)
if not options.unblur_ready:
temp_name = "{0}/{1}{2}.mrc".format(temp_path, file_name,
options.sum_suffix)
else:
temp_name = inputfile
log_name = "{0}/{1}.log".format(log_path, file_name)
error_name = "{0}/{1}.err".format(log_path, file_name)
# Append the names to the lists
micrograph_list.append("{0}{1}.mrc".format(file_name,
options.sum_suffix))
shift_list.append(shift_name)
if options.save_frames:
frames_list.append("{0}{1}.mrc".format(file_name,
options.frames_suffix))
# First build the unblur/summovie command
if not options.skip_dose_filter:
unblur_command = create_unblur_command(temp_name, micrograph_name,
shift_name, frames_name,
frc_name, options)
# Options for the summovie command
options_summovie = {
"first": 1,
"last": -1,
"nr_frames": options.nr_frames,
"pixel_size": options.pixel_size,
"exposure_per_frame": options.exposure_per_frame,
"voltage": options.voltage,
"pre_exposure": options.pre_exposure,
"dont_restore_noise": options.dont_restore_noise,
"apply_dose_filter": False,
}
summovie_command = sp_utilities.create_summovie_command(
temp_name,
micrograph_name_skip,
shift_name,
frc_summovie_name,
options_summovie,
)
else:
unblur_command = create_unblur_command(temp_name, micrograph_name,
shift_name, frc_name,
frames_name, options)
# Export the number of threads
export_threads_command = []
# Export
export_threads_command.append("export")
# Nr of threads
export_threads_command.append("OMP_NUM_THREADS={0}".format(
options.nr_threads))
if not options.unblur_ready:
# Do a e2proc3d.py
e2proc3d_command = []
# e2proc3d
e2proc3d_command.append("e2proc3d.py")
# inputfile
e2proc3d_command.append("{0}".format(inputfile))
# outputfile
e2proc3d_command.append("{0}".format(temp_name))
# Translate the command to single strings
if not options.unblur_ready:
e2proc3d_command = r" ".join(e2proc3d_command)
export_threads_command = r" ".join(export_threads_command)
unblur_command = "\n".join(unblur_command)
if not options.skip_dose_filter:
summovie_command = "\n".join(summovie_command)
# Build full command
if not options.unblur_ready:
if not options.skip_dose_filter:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command,
e2proc3d_command,
unblur_command,
unblur_path,
)
full_command_summovie = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command,
options.summovie_path)
else:
full_command = r'{0}; {1}; echo "{2}" | {3}'.format(
export_threads_command,
e2proc3d_command,
unblur_command,
unblur_path,
)
else:
if not options.skip_dose_filter:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, unblur_command, unblur_path)
full_command_summovie = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, summovie_command,
options.summovie_path)
else:
full_command = r'{0}; echo "{1}" | {2}'.format(
export_threads_command, unblur_command, unblur_path)
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
with open(log_name, "w") as f:
with open(error_name, "w") as e:
# Execute Command
if not options.skip_dose_filter:
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
subprocess.Popen([full_command_summovie],
shell=True,
stdout=f,
stderr=e).wait()
else:
subprocess.Popen([full_command],
shell=True,
stdout=f,
stderr=e).wait()
# Remove temp unblur files
temp_unblur_files = glob.glob(".UnBlur*")
for entry in temp_unblur_files:
os.remove(entry)
# Remove temp summovie files
temp_summovie_files = glob.glob(".SumMovie*")
for entry in temp_summovie_files:
os.remove(entry)
if not options.unblur_ready:
if os.path.exists(temp_name):
# Remove temp file
os.remove(temp_name)
else:
sp_global_def.sxprint(
("Error with file:\n{0}".format(inputfile)))
# Check if SumMovie and UnBlur finished cleanly
with open(log_name, "r") as r:
clean_summovie = False
clean_unblur = False
for line in r:
if "SumMovie finished cleanly." in line:
clean_summovie = True
if "UnBlur finished cleanly." in line:
clean_unblur = True
if clean_unblur:
sp_global_def.sxprint("UnBlur finished cleanly.")
else:
sp_global_def.ERROR(
"unblur error. check the logfile for more information: {0}".
format(log_name),
action=0,
)
if clean_summovie:
sp_global_def.sxprint("SumMovie finished cleanly.")
else:
sp_global_def.ERROR(
"summovie error. check the logfile for more information: {0}".
format(log_name),
action=0,
)
time_list.append(time.time() - t1)
# Do progress output
percent = round(100 * (index + 1) / float(nr_files), 2)
estimated_time = nr_files * sum(time_list) / float(len(time_list))
estimated_time_h = estimated_time // 3600
estimated_time_m = (estimated_time - estimated_time_h * 3600) // 60
estimated_time_s = (estimated_time - estimated_time_h * 3600 -
estimated_time_m * 60)
current_time = time.time() - time_start
current_time_h = current_time // 3600
current_time_m = (current_time - current_time_h * 3600) // 60
current_time_s = current_time - current_time_h * 3600 - current_time_m * 60
sp_global_def.sxprint((
"Progress: {0:.2f}%; Time: {1:.0f}h:{2:.0f}m:{3:.0f}s/{4:.0f}h:{5:.0f}m:{6:.0f}s; Micrograph done:{7}"
.format(
percent,
current_time_h,
current_time_m,
current_time_s,
estimated_time_h,
estimated_time_m,
estimated_time_s,
file_name,
)))
# Write micrograph and shift list
with open("{0}/unblur_micrographs.txt".format(output_dir), "w") as f:
for entry in sorted(micrograph_list):
f.write("{0}\n".format(entry))
with open("{0}/unblur_shiftfiles.txt".format(output_dir), "w") as f:
for entry in sorted(shift_list):
f.write("{0}\n".format(entry))
if options.save_frames:
with open("{0}/unblur_frames.txt".format(output_dir), "w") as f:
for entry in sorted(frames_list):
f.write("{0}\n".format(entry))
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " stack <maskfile> --search_rng=10 --maxit=max_iteration --CTF --snr=SNR --Fourvar=Fourier_variance --oneDx --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--search_rng",
type="int",
default=-1,
help="Search range for x-shift")
parser.add_option("--search_ang",
type="int",
default=-1,
help="Search range for inplane rotation angle")
parser.add_option(
"--search_rng_y",
type="int",
default=-1,
help=
"Search range for x-shift. Not used for 1D search (oneDx flag set).")
parser.add_option("--maxit",
type="int",
default=100,
help="Maximum number of iterations program will perform")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF correction")
parser.add_option(
"--snr",
type="float",
default=1.0,
help="signal-to-noise ratio of the data (default is 1.0)")
parser.add_option("--Fourvar",
action="store_true",
default=False,
help="compute Fourier variance")
parser.add_option("--oneDx",
action="store_true",
default=False,
help="1D search along x-axis")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI")
parser.add_option("--curvature",
action="store_true",
default=False,
help="for curved filament alignment")
(options, args) = parser.parse_args()
if not (options.MPI):
sxprint("Only MPI version is currently implemented.")
sxprint("Please run '" + progname + " -h' for detailed options")
return
if len(args) < 1 or len(args) > 2:
sxprint("usage: " + usage)
sxprint("Please run '" + progname + " -h' for detailed options")
else:
if len(args) == 1: mask = None
else: mask = args[1]
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
sp_global_def.BATCH = True
if options.oneDx:
helicalshiftali_MPI(args[0], mask, options.maxit, options.CTF,
options.snr, options.Fourvar,
options.search_rng)
else:
shiftali_MPI(args[0], mask, options.maxit, options.CTF,
options.snr, options.Fourvar, options.search_rng,
options.oneDx, options.search_rng_y)
sp_global_def.BATCH = False
return
def main():
# Parse the Options
progname = os.path.basename(sys.argv[0])
usage = (progname +
""" unblur_path input_micrograph_pattern output_directory
--summovie_path
--selection_list
--nr_frames=nr_frames
--pixel_size=pixel_size
--voltage=voltage
--exposure_per_frame=exposure_per_frame
--pre_exposure=pre_exposure
--nr_threads
--save_frames
--skip_dose_filter
--expert_mode
--shift_initial=shift_initial
--shift_radius=shift_radius
--b_factor=b_factor
--fourier_vertical=fourier_vertical
--fourier_horizontal=fourier_horizontal
--shift_threshold=shift_threshold
--iterations=iterations
--dont_restore_noise
--verbose
sxunblur exists only in non-MPI version.
Perform unblur and with dose filtering and summovie without dose filtering.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --nr_threads=1
Perform unblur with dose filtering and summovie without dose filtering with selection list.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--selection_list=selected_micrograph_file
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --nr_threads=1
Perform unblur without dose filtering.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--nr_frames=25 --pixel_size=1.19 --skip_dose_filter --nr_threads=1
Perform unblur without dose filtering and save the frames.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--nr_frames=25 --pixel_size=1.19 --skip_dose_filter --save_frames --nr_threads=1
Perform unblur with dose filtering and summovie without dose filtering with all options.
sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur
--summovie_path=~/my_app/summovie
--nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0
--voltage=300.0 --pre_exposure=0.0 --save_frames --expert_mode
--shift_initial=2.0 --shift_radius=200.0 --b_factor=1500.0
--fourier_vertical=1 --fourier_horizontal=1 --shift_threshold=0.1
--iterations=10 --verbose --nr_threads=1
""")
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option(
"--summovie_path",
type="str",
default="",
help=
"summovie executable path (SPHIRE specific): Specify the file path of summovie executable. (default none)",
)
parser.add_option(
"--selection_list",
type="str",
default="",
help=
"Micrograph selecting list (SPHIRE specific): Specify a name of micrograph selection list text file. The file extension must be '.txt'. If this is not provided, all files matched with the micrograph name pattern will be processed. (default none)",
)
parser.add_option(
"--nr_frames",
type="int",
default=3,
help=
"Number of movie frames: The number of movie frames in each input micrograph. (default 3)",
)
parser.add_option("--sum_suffix",
type="str",
default="_sum",
help=optparse.SUPPRESS_HELP)
parser.add_option("--shift_suffix",
type="str",
default="_shift",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--pixel_size",
type="float",
default=-1.0,
help=
"Pixel size [A]: The pixel size of input micrographs. (default required float)",
)
parser.add_option(
"--voltage",
type="float",
default=300.0,
help=
"Microscope voltage [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)",
)
parser.add_option(
"--exposure_per_frame",
type="float",
default=2.0,
help=
"Per frame exposure [e/A^2]: The electron dose per frame in e/A^2. (default 2.0)",
)
parser.add_option(
"--pre_exposure",
type="float",
default=0.0,
help=
"Pre-exposure [e/A^2]: The electron does in e/A^2 used for exposure prior to imaging .(default 0.0)",
)
parser.add_option(
"--nr_threads",
type="int",
default=1,
help=
"Number of threads: The number of threads unblur can use. The higher the faster, but it requires larger memory. (default 1)",
)
parser.add_option(
"--save_frames",
action="store_true",
default=False,
help=
"Save aligned movie frames: Save aligned movie frames. This option slows down the process. (default False)",
)
parser.add_option("--frames_suffix",
type="string",
default="_frames",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--skip_dose_filter",
action="store_true",
default=False,
help=
"Skip dose filter step: With this option, voltage, exposure per frame, and pre exposure will be ignored. (default False)",
)
parser.add_option(
"--expert_mode",
action="store_true",
default=False,
help=
"Use expert mode: Requires initial shift, shift radius, b-factor, fourier_vertical, fourier_horizontal, shift threshold, iterations, restore noise, and verbosity options. (default False)",
)
parser.add_option("--frc_suffix",
type="string",
default="_frc",
help=optparse.SUPPRESS_HELP)
parser.add_option(
"--shift_initial",
type="float",
default=2.0,
help=
"Minimum shift for initial search [A] (expert mode): Effective with unblur expert mode. (default 2.0)",
)
parser.add_option(
"--shift_radius",
type="float",
default=200.0,
help=
"Outer radius shift limit [A] (expert mode): Effective with unblur expert mode. (default 200.0)",
)
parser.add_option(
"--b_factor",
type="float",
default=1500.0,
help=
"Apply B-factor to images [A^2] (expert mode): Effective with unblur expert mode. (default 1500.0)",
)
parser.add_option(
"--fourier_vertical",
type="int",
default=1,
help=
"Vertical Fourier central mask size (expert mode): The half-width of central vertical line of Fourier mask. Effective with unblur expert mode. (default 1)",
)
parser.add_option(
"--fourier_horizontal",
type="int",
default=1,
help=
"Horizontal Fourier central mask size (expert mode): The half-width of central horizontal line of Fourier mask. Effective with unblur expert mode. (default 1)",
)
parser.add_option(
"--shift_threshold",
type="float",
default=0.1,
help=
"Termination shift threshold (expert mode): Effective with unblur expert mode. (default 0.1)",
)
parser.add_option(
"--iterations",
type="int",
default=10,
help=
"Maximum iterations (expert mode): Effective with unblur expert mode. (default 10)",
)
parser.add_option(
"--dont_restore_noise",
action="store_true",
default=False,
help=
"Do not restore noise power (expert mode): Effective with unblur expert mode. (default False)",
)
parser.add_option(
"--verbose",
action="store_true",
default=False,
help=
"Verbose (expert mode): Effective with unblur expert mode. (default False)",
)
parser.add_option(
"--unblur_ready",
action="store_true",
default=False,
help=optparse.SUPPRESS_HELP,
)
# list of the options and the arguments
(options, args) = parser.parse_args(sys.argv[1:])
sp_global_def.BATCH = True
# If there arent enough arguments, stop the script
if len(args) != 3:
sp_global_def.sxprint("Usage: " + usage)
sp_global_def.ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
# Convert the realtive parts to absolute ones
unblur_path = os.path.realpath(args[0]) # unblur_path
input_image = os.path.realpath(args[1]) # input_micrograph_pattern
output_dir = os.path.realpath(args[2]) # output_directory
# If the unblur executable file does not exists, stop the script
if not os.path.exists(unblur_path):
sp_global_def.ERROR(
"Unblur directory does not exist, please change the name and restart the program"
)
return
# If the output directory exists, stop the script
if os.path.exists(output_dir):
sp_global_def.ERROR(
"Output directory exists, please change the name and restart the program"
)
return
# If the input file does not exists, stop the script
file_list = glob.glob(input_image)
if not file_list:
sp_global_def.ERROR(
"Input file does not exist, please change the name and restart the program"
)
return
# If the skip_dose_filter option is false, the summovie path is necessary
if not options.skip_dose_filter and not os.path.exists(
options.summovie_path):
sp_global_def.ERROR(
"Path to the SumMovie executable is necessary when dose weighting is performed"
)
return
# Output paths
corrected_path = "{:s}/corrsum_dose_filtered".format(output_dir)
uncorrected_path = "{:s}/corrsum".format(output_dir)
shift_path = "{:s}/shift".format(output_dir)
frc_path = "{:s}/frc".format(output_dir)
log_path = "{:s}/logfiles".format(output_dir)
temp_path = "{0}/temp".format(output_dir)
# Split the path of the image name at the "/" Characters.
# The last entry contains the micrograph name.
# Split the micrograph name at the wildcard character for the
# prefix and suffix.
input_split = input_image.split("/")
input_name = input_split[-1].split("*")
# Get the input directory
if len(input_split) != 1:
input_dir = input_image[:-len(input_split[-1])]
else:
input_dir = ""
# Create output directorys
if not os.path.exists(output_dir):
os.makedirs(output_dir)
sp_global_def.write_command(output_dir)
if not os.path.exists(uncorrected_path):
os.mkdir(uncorrected_path)
if not os.path.exists(shift_path):
os.mkdir(shift_path)
if not os.path.exists(corrected_path):
if not options.skip_dose_filter:
os.mkdir(corrected_path)
if not os.path.exists(frc_path):
if options.expert_mode or not options.skip_dose_filter:
os.mkdir(frc_path)
if not os.path.exists(temp_path) and not options.unblur_ready:
os.mkdir(temp_path)
if not os.path.exists(log_path):
os.mkdir(log_path)
# Run unblur
run_unblur(
unblur_path=unblur_path,
input_image=input_image,
input_dir=input_dir,
output_dir=output_dir,
corrected_path=corrected_path,
uncorrected_path=uncorrected_path,
shift_path=shift_path,
frc_path=frc_path,
temp_path=temp_path,
log_path=log_path,
file_list=file_list,
options=options,
)
if not options.unblur_ready:
# Remove temp folder
for entry in glob.glob("{0}/*".format(temp_path)):
os.remove(entry)
os.rmdir(temp_path)
sp_global_def.sxprint("All Done!")
sp_global_def.BATCH = False
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " proj_stack output_averages --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--img_per_group",
type="int",
default=100,
help="number of images per group")
parser.add_option("--radius",
type="int",
default=-1,
help="radius for alignment")
parser.add_option(
"--xr",
type="string",
default="2 1",
help="range for translation search in x direction, search is +/xr")
parser.add_option(
"--yr",
type="string",
default="-1",
help=
"range for translation search in y direction, search is +/yr (default = same as xr)"
)
parser.add_option(
"--ts",
type="string",
default="1 0.5",
help=
"step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional"
)
parser.add_option(
"--iter",
type="int",
default=30,
help="number of iterations within alignment (default = 30)")
parser.add_option(
"--num_ali",
type="int",
default=5,
help="number of alignments performed for stability (default = 5)")
parser.add_option("--thld_err",
type="float",
default=1.0,
help="threshold of pixel error (default = 1.732)")
parser.add_option(
"--grouping",
type="string",
default="GRP",
help=
"do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size"
)
parser.add_option(
"--delta",
type="float",
default=-1.0,
help="angular step for reference projections (required for GEV method)"
)
parser.add_option(
"--fl",
type="float",
default=0.3,
help="cut-off frequency of hyperbolic tangent low-pass Fourier filter")
parser.add_option(
"--aa",
type="float",
default=0.2,
help="fall-off of hyperbolic tangent low-pass Fourier filter")
parser.add_option("--CTF",
action="store_true",
default=False,
help="Consider CTF correction during the alignment ")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args()
myid = mpi.mpi_comm_rank(MPI_COMM_WORLD)
number_of_proc = mpi.mpi_comm_size(MPI_COMM_WORLD)
main_node = 0
if len(args) == 2:
stack = args[0]
outdir = args[1]
else:
sp_global_def.ERROR("Incomplete list of arguments",
"sxproj_stability.main",
1,
myid=myid)
return
if not options.MPI:
sp_global_def.ERROR("Non-MPI not supported!",
"sxproj_stability.main",
1,
myid=myid)
return
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
sp_global_def.BATCH = True
img_per_grp = options.img_per_group
radius = options.radius
ite = options.iter
num_ali = options.num_ali
thld_err = options.thld_err
xrng = get_input_from_string(options.xr)
if options.yr == "-1":
yrng = xrng
else:
yrng = get_input_from_string(options.yr)
step = get_input_from_string(options.ts)
if myid == main_node:
nima = EMUtil.get_image_count(stack)
img = get_image(stack)
nx = img.get_xsize()
ny = img.get_ysize()
else:
nima = 0
nx = 0
ny = 0
nima = bcast_number_to_all(nima)
nx = bcast_number_to_all(nx)
ny = bcast_number_to_all(ny)
if radius == -1: radius = nx / 2 - 2
mask = model_circle(radius, nx, nx)
st = time()
if options.grouping == "GRP":
if myid == main_node:
sxprint(" A ", myid, " ", time() - st)
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
proj_params = []
for i in range(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
# Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta,
# So I will briefly explain it here
# proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers
# except for the last one. Depending on the number of particles left, they will either form a
# group or append themselves to the last group
# angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi,
# theta) is the projection angle of the center of the group, delta is the range of this group
# mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates
# whether it should take mirror position.
# In this program angle_list and mirror list are not of interest.
proj_list_all, angle_list, mirror_list = group_proj_by_phitheta(
proj_params, img_per_grp=img_per_grp)
del proj_params
sxprint(" B number of groups ", myid, " ", len(proj_list_all),
time() - st)
mpi_barrier(MPI_COMM_WORLD)
# Number of groups, actually there could be one or two more groups, since the size of the remaining group varies
# we will simply assign them to main node.
n_grp = nima / img_per_grp - 1
# Divide proj_list_all equally to all nodes, and becomes proj_list
proj_list = []
for i in range(n_grp):
proc_to_stay = i % number_of_proc
if proc_to_stay == main_node:
if myid == main_node: proj_list.append(proj_list_all[i])
elif myid == main_node:
mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT,
proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
elif myid == proc_to_stay:
img_per_grp = mpi_recv(1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
img_per_grp = int(img_per_grp[0])
temp = mpi_recv(img_per_grp, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
proj_list.append(list(map(int, temp)))
del temp
mpi_barrier(MPI_COMM_WORLD)
sxprint(" C ", myid, " ", time() - st)
if myid == main_node:
# Assign the remaining groups to main_node
for i in range(n_grp, len(proj_list_all)):
proj_list.append(proj_list_all[i])
del proj_list_all, angle_list, mirror_list
# Compute stability per projection projection direction, equal number assigned, thus overlaps
elif options.grouping == "GEV":
if options.delta == -1.0:
ERROR(
"Angular step for reference projections is required for GEV method"
)
return
from sp_utilities import even_angles, nearestk_to_refdir, getvec
refproj = even_angles(options.delta)
img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid)
# Now each processor keeps its own share of reference projections
refprojdir = refproj[img_begin:img_end]
del refproj
ref_ang = [0.0] * (len(refprojdir) * 2)
for i in range(len(refprojdir)):
ref_ang[i * 2] = refprojdir[0][0]
ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1
sxprint(" A ", myid, " ", time() - st)
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
# the solution below is very slow, do not use it unless there is a problem with the i/O
"""
for i in xrange(number_of_proc):
if myid == i:
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
mpi_barrier(MPI_COMM_WORLD)
"""
sxprint(" B ", myid, " ", time() - st)
proj_ang = [0.0] * (nima * 2)
for i in range(nima):
dp = proj_attr[i].get_params("spider")
proj_ang[i * 2] = dp["phi"]
proj_ang[i * 2 + 1] = dp["theta"]
sxprint(" C ", myid, " ", time() - st)
asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp)
del proj_ang, ref_ang
proj_list = []
for i in range(len(refprojdir)):
proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp])
del asi
sxprint(" D ", myid, " ", time() - st)
#from sys import exit
#exit()
# Compute stability per projection
elif options.grouping == "PPR":
sxprint(" A ", myid, " ", time() - st)
proj_attr = EMUtil.get_all_attributes(stack, "xform.projection")
sxprint(" B ", myid, " ", time() - st)
proj_params = []
for i in range(nima):
dp = proj_attr[i].get_params("spider")
phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[
"psi"], -dp["tx"], -dp["ty"]
proj_params.append([phi, theta, psi, s2x, s2y])
img_begin, img_end = MPI_start_end(nima, number_of_proc, myid)
sxprint(" C ", myid, " ", time() - st)
from sp_utilities import nearest_proj
proj_list, mirror_list = nearest_proj(
proj_params, img_per_grp,
list(range(img_begin, img_begin + 1))) #range(img_begin, img_end))
refprojdir = proj_params[img_begin:img_end]
del proj_params, mirror_list
sxprint(" D ", myid, " ", time() - st)
else:
ERROR("Incorrect projection grouping option")
return
###########################################################################################################
# Begin stability test
from sp_utilities import get_params_proj, read_text_file
#if myid == 0:
# from utilities import read_text_file
# proj_list[0] = map(int, read_text_file("lggrpp0.txt"))
from sp_utilities import model_blank
aveList = [model_blank(nx, ny)] * len(proj_list)
if options.grouping == "GRP":
refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list)
for i in range(len(proj_list)):
sxprint(" E ", myid, " ", time() - st)
class_data = EMData.read_images(stack, proj_list[i])
#print " R ",myid," ",time()-st
if options.CTF:
from sp_filter import filt_ctf
for im in range(len(class_data)): # MEM LEAK!!
atemp = class_data[im].copy()
btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1)
class_data[im] = btemp
#class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1)
for im in class_data:
try:
t = im.get_attr(
"xform.align2d") # if they are there, no need to set them!
except:
try:
t = im.get_attr("xform.projection")
d = t.get_params("spider")
set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0])
except:
set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0])
#print " F ",myid," ",time()-st
# Here, we perform realignment num_ali times
all_ali_params = []
for j in range(num_ali):
if (xrng[0] == 0.0 and yrng[0] == 0.0):
avet = ali2d_ras(class_data,
randomize=True,
ir=1,
ou=radius,
rs=1,
step=1.0,
dst=90.0,
maxit=ite,
check_mirror=True,
FH=options.fl,
FF=options.aa)
else:
avet = within_group_refinement(class_data, mask, True, 1,
radius, 1, xrng, yrng, step,
90.0, ite, options.fl,
options.aa)
ali_params = []
for im in range(len(class_data)):
alpha, sx, sy, mirror, scale = get_params2D(class_data[im])
ali_params.extend([alpha, sx, sy, mirror])
all_ali_params.append(ali_params)
#aveList[i] = avet
#print " G ",myid," ",time()-st
del ali_params
# We determine the stability of this group here.
# stable_set contains all particles deemed stable, it is a list of list
# each list has two elements, the first is the pixel error, the second is the image number
# stable_set is sorted based on pixel error
#from utilities import write_text_file
#write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid)
stable_set, mir_stab_rate, average_pix_err = multi_align_stability(
all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1)
#print " H ",myid," ",time()-st
if (len(stable_set) > 5):
stable_set_id = []
members = []
pix_err = []
# First put the stable members into attr 'members' and 'pix_err'
for s in stable_set:
# s[1] - number in this subset
stable_set_id.append(s[1])
# the original image number
members.append(proj_list[i][s[1]])
pix_err.append(s[0])
# Then put the unstable members into attr 'members' and 'pix_err'
from sp_fundamentals import rot_shift2D
avet.to_zero()
if options.grouping == "GRP":
aphi = 0.0
atht = 0.0
vphi = 0.0
vtht = 0.0
l = -1
for j in range(len(proj_list[i])):
# Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses
if j in stable_set_id:
l += 1
avet += rot_shift2D(class_data[j], stable_set[l][2][0],
stable_set[l][2][1],
stable_set[l][2][2],
stable_set[l][2][3])
if options.grouping == "GRP":
phi, theta, psi, sxs, sy_s = get_params_proj(
class_data[j])
if (theta > 90.0):
phi = (phi + 540.0) % 360.0
theta = 180.0 - theta
aphi += phi
atht += theta
vphi += phi * phi
vtht += theta * theta
else:
members.append(proj_list[i][j])
pix_err.append(99999.99)
aveList[i] = avet.copy()
if l > 1:
l += 1
aveList[i] /= l
if options.grouping == "GRP":
aphi /= l
atht /= l
vphi = (vphi - l * aphi * aphi) / l
vtht = (vtht - l * atht * atht) / l
from math import sqrt
refprojdir[i] = [
aphi, atht,
(sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0
]
# Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION
aveList[i].set_attr('members', members)
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', pix_err)
else:
sxprint(" empty group ", i, refprojdir[i])
aveList[i].set_attr('members', [-1])
aveList[i].set_attr('refprojdir', refprojdir[i])
aveList[i].set_attr('pixerr', [99999.])
del class_data
if myid == main_node:
km = 0
for i in range(number_of_proc):
if i == main_node:
for im in range(len(aveList)):
aveList[im].write_image(args[1], km)
km += 1
else:
nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nl = int(nl[0])
for im in range(nl):
ave = recv_EMData(i, im + i + 70000)
nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
nm = int(nm[0])
members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
ave.set_attr('members', list(map(int, members)))
members = mpi_recv(nm, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('pixerr', list(map(float, members)))
members = mpi_recv(3, MPI_FLOAT, i,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
ave.set_attr('refprojdir', list(map(float, members)))
ave.write_image(args[1], km)
km += 1
else:
mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL,
MPI_COMM_WORLD)
for im in range(len(aveList)):
send_EMData(aveList[im], main_node, im + myid + 70000)
members = aveList[im].get_attr('members')
mpi_send(len(members), 1, MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
mpi_send(members, len(members), MPI_INT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
members = aveList[im].get_attr('pixerr')
mpi_send(members, len(members), MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
try:
members = aveList[im].get_attr('refprojdir')
mpi_send(members, 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
except:
mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node,
SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD)
sp_global_def.BATCH = False
mpi_barrier(MPI_COMM_WORLD)
def generate_runscript(filename, seg_ny, ptcl_dst, fract):
if ptcl_dst < 15:
ERROR(
"Distance in pixels between adjacent segments should be at least one rise!"
)
return
sxprint("Generating run script with the following parameters: \n")
sxprint("y-dimension of segment used for refinement: %d" % seg_ny)
sxprint("Distance in pixels between adjacent segments: %d" % ptcl_dst)
sxprint("Fraction of structure used for applying helical symmetry: %.2f" %
fract)
if os.path.exists(filename):
ERROR("File " + filename +
" already exists. Please either remove or rename the file")
return
f = open(filename, 'w')
f.write('#!/bin/csh\n')
f.write('\n')
f.write('set echo on\n')
f.write('\n')
f.write('#clean the previous outputs\n')
f.write('rm *.hdf *.txt *.bck rm *.pdb\n')
f.write('rm log*\n')
#f.write('rm -rf outsymsearch \n')
f.write('rm -rf mic result_* \n')
f.write('rm -rf EMAN*\n')
f.write('\n')
f.write('# get the previous runned results\n')
f.write('tar -zxvf answer.tar.gz\n')
f.write('\n')
f.write('#generate volume from pdb\n')
f.write('tar -zxvf 3MFP_1SU.tar.gz\n')
f.write('\n')
f.write('# Helicise the Atom coordinates\n')
f.write(
'# Input: pdb file containing atom coordinates to helicise (3MFP_1SU.pdb)\n'
)
f.write('# Output: pdb file containing helicised coordinates (rnew.pdb)\n')
f.write(
'sxhelical_demo.py 3MFP_1SU.pdb rnew.pdb --heli --dp=27.6 --dphi=166.715\n'
)
f.write('\n')
f.write('#generate the density map\n')
f.write('sxpdb2em.py rnew.pdb tmp.hdf --apix=1.84 --center=c \n')
f.write('\n')
f.write(
'# Generate three micrographs, where each micrograph contains one projection of a long filament.\n'
)
f.write(
'# Input: Reference volume from which projections are calculated (tmp.hdf)\n'
)
f.write(
'# Output: Output directory in which micrographs will be written (mic)\n'
)
f.write(
'sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84\n'
)
f.write('\n')
f.write('# generate initial volume for later helical refinement\n')
f.write(
'# Output: A noisy cylinder written to the output file name (ini.hdf).\n'
)
f.write(
'sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35\n'
)
f.write('\n')
f.write(
'# Estimate defocus value for each micrograph. This can be done either by GUI or pure command-line \n'
)
f.write('# 1. To estimate defocus by GUI:\n')
f.write('cd mic\n')
f.write('sxhelixboxer.py mic0.hdf --gui &\n')
f.write('\n')
f.write(
"# When the GUI starts up, there will be a checkbox at the bottom labelled 'CTF Estimation using CTER'. Check this box, and additional fields will appear in the GUI.\n"
)
f.write(
"# Under 'Parameters of CTF estimation,' enter 256 for 'Window size,' 2.0 for 'Cs, 10.0 for 'Amplitude Contrast,' 200 for 'Voltage', and 1.84 for Pixel size.\n"
)
f.write(
'# Click "Estimate CTF using CTER" button, and the estimated defocus will show up in the "Estimated defocus" box.\n'
)
f.write(
'# Now you can either estimate defocus for the remaining micrographs one by one using the GUI, or you can estimate defocus for the remaining micrographs in batch mode using \n'
)
f.write('# the parameters you entered in the GUI for mic0.hdf. \n')
f.write(
'# Make sure to remove any output directories (pwrot and partres followed by underscore and \n'
)
f.write(
'# then name of the micrograph, e.g., pwrot_mic0 and partres_mic0) generated in the previous run of CTF estimation using CTER.\n'
)
f.write('rm -r pwrot_mic0;rm -r partres_mic0\n')
f.write('sxhelixboxer.py mic*.hdf --gui &\n')
f.write('cd ..\n')
f.write('\n')
f.write('# 2. To estimate defocus by command-line\n')
f.write('cd mic\n')
f.write(
'sxhelixboxer.py pwrot partres --cter --indir=. --nameroot=mic --nx=200 --Cs=2.0 --voltage=200 --kboot=16 --apix=1.84 --ac=10.0 \n'
)
f.write('cd ..\n')
f.write('\n')
f.write('cd mic\n')
f.write('# have to open boxer\n')
f.write(
'# if want to save time, just close the window immediately and use saved coordinate\n'
)
f.write('# otherwise draw the box carefully to get the coordinate \n')
f.write('sxhelixboxer.py *.hdf --gui --helix-width=200 --ptcl-width=200\n')
f.write('cd ..\n')
f.write('\n')
f.write('# if use saved coordinate, please use below commands\n')
f.write('tar -zxvf saved_pos.tar.gz\n')
f.write('cp saved_db.tar.gz mic/.\n')
f.write('cp -r saved_pos/*box* mic/.\n')
f.write('rm -rf saved_pos\n')
f.write('cd mic\n')
f.write('tar -zxvf saved_db.tar.gz\n')
f.write('cd ..\n')
f.write('\n')
f.write('# Window segments from boxed helices\n')
f.write(
'# For more information on the windowing utility, see http://sparx-em.org/sparxwiki/windowallmic\n'
)
f.write('# distance between segments is 1 rise exactly\n')
f.write(
'sxhelixboxer.py houtdir --window --dirid=mic --micid=mic --micsuffix=hdf --dp=27.6 --apix=1.84 --boxsize=200 --outstacknameall=bdb:hadata --hcoords_suffix=_boxes.txt --ptcl-dst=%d --rmax=64\n'
% ptcl_dst)
f.write('\n')
f.write('# Generate 2D mask for centering EM images\n')
f.write(
'# Output: 2D mask written to output file name provided (mask2d.hdf)\n'
)
f.write(
'sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=70\n'
)
f.write('\n')
f.write('# center all the EM images\n')
f.write(
'# centering parameters will be saved in header, the input images will not be changed.\n'
)
# horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1
# f.write('sxheader.py bdb:hadata --params=active --one\n')
f.write('sxheader.py bdb:hadata --params=xform.align2d --zero\n')
f.write(
'mpirun -np 2 sxshiftali.py bdb:hadata mask2d.hdf --oneDx --search_rng=10 --maxit=20 --CTF --MPI\n'
)
f.write('\n')
f.write(
'# Apply the centering parameters stored in the header of each image in bdb:hadata to the image, \n'
)
f.write(
'# normalize using average and standard deviation outside the mask, and save the centered and normalized image to bdb:hdata\n'
)
f.write(
'# Input: Input stack containing 2D alignment parameters in the header (bdb:hadata)\n'
)
f.write('# Name of 2D mask to use for normalization\n')
f.write(
'# Output: Stack of images (bdb:hdata) after applying 2D alignment parameters to the images in input stack. \n'
)
f.write(
'sxhelical_demo.py bdb:hadata bdb:hdata mask2d.hdf --applyparams\n')
f.write('\n')
f.write('#Exhaustive search \n')
f.write('sxheader.py bdb:hdata --params=xform.projection --zero\n')
# horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1
# f.write('sxheader.py bdb:hdata --params=active --one\n')
f.write(
'mpirun -np 3 sxhelicon.py bdb:hdata ini.hdf result_helicon --CTF --seg_ny=%d --fract=%.2f --psi_max=2.0 --delta=1.0 --maxit=7 --function=[.,nofunc,helical3c] --searchxshift=3.68 --xwobble=1.84 --ywobble=0 --dp=27.6 --dphi=166.715 --apix=1.84 --rmin=0.0 --rmax=64 --MPI\n'
% (seg_ny, fract))
f.write('\n')
f.write('#Local search \n')
f.write(
'sxheader.py bdb:hdata --params=xform.projection --import=result_helicon/parameters0007.txt\n'
)
f.write(
'mpirun -np 3 sxlocalhelicon.py bdb:hdata result_helicon/volf007.hdf result_local --CTF --seg_ny=%d --fract=%.2f --psi_max=2.0 --delta=1.0 --maxit=11 --function=[.,nofunc,helical3c] --boundaryavg --MA --MA_WRAP=0 --xr=3.68 --txs=1.84 --an=20 --ynumber=16 --dp=27.6 --dphi=166.715 --apix=1.84 --rmin=0.0 --rmax=64 --MPI\n'
% (seg_ny, fract))
f.write('\n')
def refine_set(particle_indices):
"""
Runs the defocus refinement for a set of particles.
:param particle_indices: Indicies of the particles inside the stack
:return: List of refined CTFs.
"""
try:
refined_ctfs = []
particle_micrographs = {}
for particle_index in particle_indices:
particle = sp_ctf_refine_io.read_particle(STACK_FILE_PATH, particle_index)
# particle.write_image("/home/twagner/temp/refine/"+str(particle_index)+"particle.hdf")
if RESOLUTION is not None and PIXEL_SIZE is not None:
particle = particle.process(
"filter.lowpass.gauss",
{"cutoff_freq": 1.0 / RESOLUTION, "apix": PIXEL_SIZE},
)
# In case of phase plate a high pass might be necessary
# if PARTICLE_DIAMETER is not None
# particle = particle.process("filter.highpass.gauss",{"cutoff_freq":1.0/160})
particle_projection_params = PROJECTION_PARAMETERS[particle_index].tolist()
# As i shift the volume and not the particle, multiply by -1
particle_projection_params[3] = -1 * particle_projection_params[3]
particle_projection_params[4] = -1 * particle_projection_params[4]
# particle_projection_params[3] = add_proj_error(particle_projection_params, 3, 2)
# particle_projection_params[4] = add_proj_error(particle_projection_params, 4, 2)
# particle_projection_params[2] = add_proj_error(particle_projection_params, 2, 2)
particle_ctf = particle.get_attr("ctf")
volume = VOLUME1
if HALF_MAP_ASSIGNEMENTS is not None:
if HALF_MAP_ASSIGNEMENTS[particle_index] != 0:
volume = VOLUME2
best_ctf, error, drratio = refine_defocus_with_error_est(
particle_volume=volume,
particle_image=particle,
projection_angle_and_shift=particle_projection_params,
current_ctf=particle_ctf,
def_search_range=DEFOCUS_SEARCH_RANGE,
def_step_size=DEFOCUS_STEP_SIZE,
)
particle_micrograph_filename = particle.get_attr("ptcl_source_image")
if particle_micrograph_filename in particle_micrographs:
particle_micrographs[particle_micrograph_filename]["indices"].append(
particle_index
)
particle_micrographs[particle_micrograph_filename]["defocus"].append(
best_ctf.defocus
)
particle_micrographs[particle_micrograph_filename]["diff"].append(
particle_ctf.defocus - best_ctf.defocus
)
particle_micrographs[particle_micrograph_filename]["error"].append(
error
)
particle_micrographs[particle_micrograph_filename]["drratio"].append(
drratio
)
else:
particle_micrographs[particle_micrograph_filename] = {
"indices": [particle_index],
"diff": [particle_ctf.defocus - best_ctf.defocus],
"error": [error],
"defocus": [best_ctf.defocus],
"drratio": [drratio],
}
refined_ctfs.append(best_ctf)
del particle_projection_params
del volume
del particle_ctf
del particle
except Exception as err:
import traceback
sp_global_def.sxprint(
"Exception happend for particles in range ",
np.min(particle_indices),
"-",
np.max(particle_indices),
)
traceback.print_exc()
raise err
return refined_ctfs, particle_micrographs
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + """ Input Output [options]
Generate three micrographs, each micrograph contains one projection of a long filament.
Input: Reference Volume, output directory
Output: Three micrographs stored in output directory
sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84
Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200
sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35
Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels
sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60
Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data
sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams
Generate run through example script for helicon
sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35
"""
parser = OptionParser(usage, version=SPARXVERSION)
# helicise the Atom coordinates
# generate micrographs of helical filament
parser.add_option(
"--generate_micrograph",
action="store_true",
default=False,
help=
"Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory"
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="Use CTF correction")
parser.add_option("--apix",
type="float",
default=-1,
help="pixel size in Angstroms")
parser.add_option(
"--rand_seed",
type="int",
default=14567,
help=
"the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs."
)
parser.add_option("--Cs",
type="float",
default=2.0,
help="Microscope Cs (spherical aberation)")
parser.add_option("--voltage",
type="float",
default=200.0,
help="Microscope voltage in KV")
parser.add_option("--ac",
type="float",
default=10.0,
help="Amplitude contrast (percentage, default=10)")
parser.add_option("--nonoise",
action="store_true",
default=False,
help="Do not add noise to the micrograph.")
# generate initial volume
parser.add_option("--generate_noisycyl",
action="store_true",
default=False,
help="Generate initial volume of noisy cylinder.")
parser.add_option(
"--boxsize",
type="string",
default="100,100,200",
help=
"String containing x , y, z dimensions (separated by comma) in pixels")
parser.add_option("--rad",
type="int",
default=35,
help="Radius of initial volume in pixels")
# generate 2D mask
parser.add_option("--generate_mask",
action="store_true",
default=False,
help="Generate 2D rectangular mask.")
parser.add_option(
"--masksize",
type="string",
default="200,200",
help=
"String containing x and y dimensions (separated by comma) in pixels")
parser.add_option("--maskwidth",
type="int",
default=60,
help="Width of rectangular mask")
# Apply 2D alignment parameters to input stack and output new images to output stack
parser.add_option(
"--applyparams",
action="store_true",
default=False,
help=
"Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack"
)
# Generate run script
parser.add_option("--generate_script",
action="store_true",
default=False,
help="Generate script for helicon run through example")
parser.add_option("--filename",
type="string",
default="runhelicon",
help="Name of run script to generate")
parser.add_option("--seg_ny",
type="int",
default=180,
help="y-dimension of segment used for refinement")
parser.add_option(
"--ptcl_dist",
type="int",
default=15,
help=
"Distance in pixels between adjacent segments windowed from same filament"
)
parser.add_option(
"--fract",
type="float",
default=0.35,
help="Fraction of the volume used for applying helical symmetry.")
(options, args) = parser.parse_args()
if len(args) > 3:
sxprint("usage: " + usage)
sxprint("Please run '" + progname + " -h' for detailed options")
ERROR(
"Invalid number of parameters. Please see usage information above."
)
return
else:
if options.generate_script:
generate_runscript(options.filename, options.seg_ny,
options.ptcl_dist, options.fract)
if options.generate_micrograph:
if options.apix <= 0:
ERROR("Please enter pixel size.")
return
generate_helimic(args[0], args[1], options.apix, options.CTF,
options.Cs, options.voltage, options.ac,
options.nonoise, options.rand_seed)
if options.generate_noisycyl:
from sp_utilities import model_cylinder, model_gauss_noise
outvol = args[0]
boxdims = options.boxsize.split(',')
if len(boxdims) < 1 or len(boxdims) > 3:
ERROR(
"Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize=\'100,100,200\'"
)
return
nx = int(boxdims[0])
if len(boxdims) == 1:
ny = nx
nz = nx
else:
ny = int(boxdims[1])
if len(boxdims) == 3:
nz = int(boxdims[2])
(model_cylinder(options.rad, nx, ny, nz) *
model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol)
if options.generate_mask:
from sp_utilities import model_blank, pad
outvol = args[0]
maskdims = options.masksize.split(',')
if len(maskdims) < 1 or len(maskdims) > 2:
ERROR(
"Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize=\'200,200\'"
)
return
nx = int(maskdims[0])
if len(maskdims) == 1:
ny = nx
else:
ny = int(maskdims[1])
mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1,
0.0)
mask.write_image(outvol)
if options.applyparams:
from sp_utilities import get_im, get_params2D, set_params2D
from sp_fundamentals import cyclic_shift
stack = args[0]
newstack = args[1]
mask = get_im(args[2])
nima = EMUtil.get_image_count(stack)
for im in range(nima):
prj = get_im(stack, im)
alpha, sx, sy, mirror, scale = get_params2D(prj)
prj = cyclic_shift(prj, int(sx))
set_params2D(prj, [0.0, 0., 0.0, 0, 1])
stat = Util.infomask(prj, mask, False)
prj = (prj - stat[0]) / stat[1]
ctf_params = prj.get_attr("ctf")
prj.set_attr('ctf_applied', 0)
prj.write_image(newstack, im)
def main():
progname = os.path.basename(sys.argv[0])
usage = progname + " averages1 averages2 --th_grp"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--T",
type="int",
default=0,
help=" Threshold for matching")
parser.add_option("--J", type="int", default=50, help=" J")
parser.add_option("--max_branching",
type="int",
default=40,
help=" maximum branching")
parser.add_option("--verbose",
action="store_true",
default=False,
help=" Threshold for matching")
parser.add_option("--timing",
action="store_true",
default=False,
help=" Get the timing")
(options, args) = parser.parse_args()
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
sp_global_def.BATCH = True
from numpy import array
from sp_statistics import k_means_stab_bbenum
R = len(args)
Parts = []
mem = [0] * R
avg = [0] * R
for r in range(R):
data = EMData.read_images(args[r])
avg[r] = len(data)
part = []
for k in range(len(data)):
lid = data[k].get_attr('members')
mem[r] += len(lid)
lid = array(lid, 'int32')
lid.sort()
part.append(lid.copy())
Parts.append(part)
if options.timing:
from time import time
time1 = time()
MATCH, STB_PART, CT_s, CT_t, ST, st = k_means_stab_bbenum(
Parts,
T=options.T,
J=options.J,
max_branching=options.max_branching,
stmult=0.1,
branchfunc=2)
if options.verbose:
sxprint(MATCH)
sxprint(STB_PART)
sxprint(CT_s)
sxprint(CT_t)
sxprint(ST)
sxprint(st)
sxprint(" ")
for i in range(len(MATCH)):
u = MATCH[i][0] # u is the group in question in partition 1
assert len(STB_PART[u]) == CT_s[u]
sxprint("Group ", end=' ')
for r in range(R):
sxprint("%3d " % (MATCH[i][r]), end=' ')
sxprint(" matches: group size = ", end=' ')
for r in range(R):
sxprint(" %3d" % len(Parts[r][MATCH[i][r]]), end=' ')
sxprint(" matched size = %3d" % (CT_s[u]), end=' ')
if options.verbose:
sxprint(" matched group = %s" % (STB_PART[u]))
else:
sxprint("")
sxprint("\nNumber of averages = ", end=' ')
for r in range(R):
sxprint("%3d" % (avg[r]), end=' ')
sxprint("\nTotal number of particles = ", end=' ')
for r in range(R):
sxprint("%3d" % (mem[r]), end=' ')
sxprint(" number of matched particles = %5d" % (sum(CT_s)))
if options.timing:
sxprint("Elapsed time = ", time() - time1)
sp_global_def.BATCH = False
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + " stack ref_vol outdir <maskfile> --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --ynumber=y_numbers --txs=translational_search_stepx --delta=angular_step --an=angular_neighborhood --center=1 --maxit=max_iter --CTF --snr=1.0 --ref_a=S --sym=c1 --datasym=symdoc --new"
parser = OptionParser(usage, version=SPARXVERSION)
#parser.add_option("--ir", type="float", default= -1, help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)")
parser.add_option(
"--ou",
type="float",
default=-1,
help=
"outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)"
)
parser.add_option(
"--rs",
type="int",
default=1,
help="step between rings in rotational correlation >0 (set to 1)")
parser.add_option(
"--xr",
type="string",
default=" 4 2 1 1 1",
help=
"range for translation search in x direction, search is +/-xr (Angstroms) "
)
parser.add_option(
"--txs",
type="string",
default="1 1 1 0.5 0.25",
help=
"step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)"
)
parser.add_option(
"--y_restrict",
type="string",
default="-1 -1 -1 -1 -1",
help=
"range for translational search in y-direction, search is +/-y_restrict in Angstroms. This only applies to local search, i.e., when an is not -1. If y_restrict < 0, then for ihrsrlocalcons (option --localcons local search with consistency), the y search range is set such that it is the same ratio to dp as angular search range is to dphi. For regular ihrsr, y search range is the full range when y_restrict< 0. Default is -1."
)
parser.add_option(
"--ynumber",
type="string",
default="4 8 16 32 32",
help=
"even number of the translation search in y direction, search is (-dpp/2,-dpp/2+dpp/ny,,..,0,..,dpp/2-dpp/ny dpp/2]"
)
parser.add_option("--delta",
type="string",
default=" 10 6 4 3 2",
help="angular step of reference projections")
parser.add_option(
"--an",
type="string",
default="-1",
help=
"angular neighborhood for local searches (default -1, meaning do exhaustive search)"
)
parser.add_option(
"--maxit",
type="int",
default=30,
help=
"maximum number of iterations performed for each angular step (default 30) "
)
parser.add_option("--CTF",
action="store_true",
default=False,
help="CTF correction")
parser.add_option("--snr",
type="float",
default=1.0,
help="Signal-to-Noise Ratio of the data (default 1)")
parser.add_option("--MPI",
action="store_true",
default=True,
help="use MPI version")
#parser.add_option("--fourvar", action="store_true", default=False, help="compute Fourier variance")
parser.add_option("--apix",
type="float",
default=-1.0,
help="pixel size in Angstroms")
parser.add_option("--dp",
type="float",
default=-1.0,
help="delta z - translation in Angstroms")
parser.add_option("--dphi",
type="float",
default=-1.0,
help="delta phi - rotation in degrees")
parser.add_option(
"--ndp",
type="int",
default=12,
help=
"In symmetrization search, number of delta z steps equals to 2*ndp+1")
parser.add_option(
"--ndphi",
type="int",
default=12,
help="In symmetrization search,number of dphi steps equas to 2*ndphi+1"
)
parser.add_option("--dp_step",
type="float",
default=0.1,
help="delta z (Angstroms) step for symmetrization")
parser.add_option("--dphi_step",
type="float",
default=0.1,
help="dphi step for symmetrization")
parser.add_option(
"--psi_max",
type="float",
default=10.0,
help=
"maximum psi - how far rotation in plane can can deviate from 90 or 270 degrees (default 10)"
)
parser.add_option("--rmin",
type="float",
default=0.0,
help="minimal radius for hsearch (Angstroms)")
parser.add_option("--rmax",
type="float",
default=80.0,
help="maximal radius for hsearch (Angstroms)")
parser.add_option("--fract",
type="float",
default=0.7,
help="fraction of the volume used for helical search")
parser.add_option("--sym",
type="string",
default="c1",
help="symmetry of the structure")
parser.add_option("--function",
type="string",
default="helical",
help="name of the reference preparation function")
parser.add_option("--datasym",
type="string",
default="datasym.txt",
help="symdoc")
parser.add_option(
"--nise",
type="int",
default=200,
help="start symmetrization after nise steps (default 200)")
parser.add_option("--npad",
type="int",
default=2,
help="padding size for 3D reconstruction, (default 2)")
parser.add_option("--debug",
action="store_true",
default=False,
help="debug")
parser.add_option("--new",
action="store_true",
default=False,
help="use rectangular recon and projection version")
parser.add_option(
"--initial_theta",
type="float",
default=90.0,
help="intial theta for reference projection (default 90)")
parser.add_option(
"--delta_theta",
type="float",
default=1.0,
help="delta theta for reference projection (default 1.0)")
parser.add_option("--WRAP",
type="int",
default=1,
help="do helical wrapping (default 1, meaning yes)")
(options, args) = parser.parse_args(arglist[1:])
if len(args) < 1 or len(args) > 5:
sxprint("usage: " + usage + "\n")
sxprint("Please run '" + progname + " -h' for detailed options")
ERROR(
"Invalid number of parameters used. please see usage information above."
)
return
else:
# Convert input arguments in the units/format as expected by ihrsr_MPI in applications.
if options.apix < 0:
ERROR("Please enter pixel size")
return
rminp = int((float(options.rmin) / options.apix) + 0.5)
rmaxp = int((float(options.rmax) / options.apix) + 0.5)
from sp_utilities import get_input_from_string, get_im
xr = get_input_from_string(options.xr)
txs = get_input_from_string(options.txs)
y_restrict = get_input_from_string(options.y_restrict)
irp = 1
if options.ou < 0: oup = -1
else: oup = int((options.ou / options.apix) + 0.5)
xrp = ''
txsp = ''
y_restrict2 = ''
for i in range(len(xr)):
xrp += " " + str(float(xr[i]) / options.apix)
for i in range(len(txs)):
txsp += " " + str(float(txs[i]) / options.apix)
# now y_restrict has the same format as x search range .... has to change ihrsr accordingly
for i in range(len(y_restrict)):
y_restrict2 += " " + str(float(y_restrict[i]) / options.apix)
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
from sp_applications import ihrsr
sp_global_def.BATCH = True
if len(args) < 4: mask = None
else: mask = args[3]
ihrsr(args[0], args[1], args[2], mask, irp, oup, options.rs, xrp,
options.ynumber, txsp, options.delta, options.initial_theta,
options.delta_theta, options.an, options.maxit, options.CTF,
options.snr, options.dp, options.ndp, options.dp_step,
options.dphi, options.ndphi, options.dphi_step, options.psi_max,
rminp, rmaxp, options.fract, options.nise, options.npad,
options.sym, options.function, options.datasym, options.apix,
options.debug, options.MPI, options.WRAP, y_restrict2)
sp_global_def.BATCH = False
def main():
arglist = []
for arg in sys.argv:
arglist.append(arg)
progname = os.path.basename(arglist[0])
usage = progname + " stack ref_vol outdir --dp=rise --dphi=rotation --apix=pixel_size --phistep=phi_step --zstep=z_step --fract=helicising_fraction --rmax=maximum_radius --rmin=min_radius --CTF --sym=c1 --function=user_function --maxit=max_iter --MPI"
parser = OptionParser(usage, version=SPARXVERSION)
parser.add_option("--dp",
type="float",
default=1.0,
help="delta z - translation in Angstroms")
parser.add_option("--dphi",
type="float",
default=1.0,
help="delta phi - rotation in degrees")
parser.add_option("--apix",
type="float",
default=1.84,
help="pixel size in Angstroms")
parser.add_option("--rmin",
type="int",
default=0,
help="minimal radial extent of structure")
parser.add_option("--rmax",
type="int",
default=70,
help="maximal radial extent of structure")
parser.add_option("--fract",
type="float",
default=0.66,
help="fraction of the volume used for helical search")
parser.add_option("--sym",
type="string",
default="c1",
help="symmetry of the structure")
parser.add_option("--function",
type="string",
default="helical",
help="name of the reference preparation function")
parser.add_option("--zstep",
type="int",
default=1,
help="Step size for translational search along z")
parser.add_option("--CTF",
action="store_true",
default=False,
help="CTF correction")
parser.add_option("--maxit",
type="int",
default=5,
help="maximum number of iterations performed")
parser.add_option("--MPI",
action="store_true",
default=False,
help="use MPI version")
(options, args) = parser.parse_args(arglist[1:])
if len(args) != 3:
sxprint("Usage: " + usage)
sxprint("Please run \'" + progname + " -h\' for detailed options")
ERROR(
"Invalid number of parameters used. Please see usage information above."
)
return
else:
if not options.MPI:
ERROR(
"There is only MPI version of sxfilrecons3d.py. See SPARX wiki page for downloading MyMPI details."
)
return
if sp_global_def.CACHE_DISABLE:
from sp_utilities import disable_bdb_cache
disable_bdb_cache()
from sp_development import filrecons3D_MPI
sp_global_def.BATCH = True
filrecons3D_MPI(args[0], args[1], args[2], options.dp, options.dphi,
options.apix, options.function, options.zstep,
options.fract, options.rmax, options.rmin, options.CTF,
options.maxit, options.sym)
sp_global_def.BATCH = False
def align_diff_params(ali_params1, ali_params2):
"""
This function determines the relative angle, shifts and mirrorness between
two sets of alignment parameters.
"""
nima = len(ali_params1)
nima2 = len(ali_params2)
if nima2 != nima:
sp_global_def.sxprint("Error: Number of images do not agree!")
return 0.0, 0.0, 0.0, 0
else:
nima /= 4
del nima2
# Read the alignment parameters and determine the relative mirrorness
mirror_same = 0
for i in range(nima):
if ali_params1[i * 4 + 3] == ali_params2[i * 4 + 3]:
mirror_same += 1
if mirror_same > nima / 2:
mirror = 0
else:
mirror_same = nima - mirror_same
mirror = 1
# Determine the relative angle
cosi = 0.0
sini = 0.0
angle1 = []
angle2 = []
for i in range(nima):
mirror1 = ali_params1[i * 4 + 3]
mirror2 = ali_params2[i * 4 + 3]
if abs(mirror1 - mirror2) == mirror:
alpha1 = ali_params1[i * 4]
alpha2 = ali_params2[i * 4]
if mirror1 == 1:
alpha1 = -alpha1
alpha2 = -alpha2
angle1.append(alpha1)
angle2.append(alpha2)
alphai = angle_diff(angle1, angle2)
# Determine the relative shift
sxi = 0.0
syi = 0.0
for i in range(nima):
mirror1 = ali_params1[i * 4 + 3]
mirror2 = ali_params2[i * 4 + 3]
if abs(mirror1 - mirror2) == mirror:
alpha1 = ali_params1[i * 4]
# alpha2 = ali_params2[i*4]
sx1 = ali_params1[i * 4 + 1]
sx2 = ali_params2[i * 4 + 1]
sy1 = ali_params1[i * 4 + 2]
sy2 = ali_params2[i * 4 + 2]
alpha12, sx12, sy12, mirror12 = sp_utilities.combine_params2(
alpha1, sx1, sy1, int(mirror1), alphai, 0.0, 0.0, 0)
if mirror1 == 0:
sxi += sx2 - sx12
else:
sxi -= sx2 - sx12
syi += sy2 - sy12
sxi /= mirror_same
syi /= mirror_same
return alphai, sxi, syi, mirror
def helicalshiftali_MPI(stack,
maskfile=None,
maxit=100,
CTF=False,
snr=1.0,
Fourvar=False,
search_rng=-1):
nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
main_node = 0
ftp = file_type(stack)
if myid == main_node:
print_begin_msg("helical-shiftali_MPI")
max_iter = int(maxit)
if (myid == main_node):
infils = EMUtil.get_all_attributes(stack, "filament")
ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord')
filaments = ordersegments(infils, ptlcoords)
total_nfils = len(filaments)
inidl = [0] * total_nfils
for i in range(total_nfils):
inidl[i] = len(filaments[i])
linidl = sum(inidl)
nima = linidl
tfilaments = []
for i in range(total_nfils):
tfilaments += filaments[i]
del filaments
else:
total_nfils = 0
linidl = 0
total_nfils = bcast_number_to_all(total_nfils, source_node=main_node)
if myid != main_node:
inidl = [-1] * total_nfils
inidl = bcast_list_to_all(inidl, myid, source_node=main_node)
linidl = bcast_number_to_all(linidl, source_node=main_node)
if myid != main_node:
tfilaments = [-1] * linidl
tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node)
filaments = []
iendi = 0
for i in range(total_nfils):
isti = iendi
iendi = isti + inidl[i]
filaments.append(tfilaments[isti:iendi])
del tfilaments, inidl
if myid == main_node:
print_msg("total number of filaments: %d" % total_nfils)
if total_nfils < nproc:
ERROR(
'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used'
% (nproc, total_nfils),
myid=myid)
# balanced load
temp = chunks_distribution([[len(filaments[i]), i]
for i in range(len(filaments))],
nproc)[myid:myid + 1][0]
filaments = [filaments[temp[i][1]] for i in range(len(temp))]
nfils = len(filaments)
#filaments = [[0,1]]
#print "filaments",filaments
list_of_particles = []
indcs = []
k = 0
for i in range(nfils):
list_of_particles += filaments[i]
k1 = k + len(filaments[i])
indcs.append([k, k1])
k = k1
data = EMData.read_images(stack, list_of_particles)
ldata = len(data)
sxprint("ldata=", ldata)
nx = data[0].get_xsize()
ny = data[0].get_ysize()
if maskfile == None:
mrad = min(nx, ny) // 2 - 2
mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0)
else:
mask = get_im(maskfile)
# apply initial xform.align2d parameters stored in header
init_params = []
for im in range(ldata):
t = data[im].get_attr('xform.align2d')
init_params.append(t)
p = t.get_params("2d")
data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'],
p['mirror'], p['scale'])
if CTF:
from sp_filter import filt_ctf
from sp_morphology import ctf_img
ctf_abs_sum = EMData(nx, ny, 1, False)
ctf_2_sum = EMData(nx, ny, 1, False)
else:
ctf_2_sum = None
ctf_abs_sum = None
from sp_utilities import info
for im in range(ldata):
data[im].set_attr('ID', list_of_particles[im])
st = Util.infomask(data[im], mask, False)
data[im] -= st[0]
if CTF:
ctf_params = data[im].get_attr("ctf")
qctf = data[im].get_attr("ctf_applied")
if qctf == 0:
data[im] = filt_ctf(fft(data[im]), ctf_params)
data[im].set_attr('ctf_applied', 1)
elif qctf != 1:
ERROR('Incorrectly set qctf flag', myid=myid)
ctfimg = ctf_img(nx, ctf_params, ny=ny)
Util.add_img2(ctf_2_sum, ctfimg)
Util.add_img_abs(ctf_abs_sum, ctfimg)
else:
data[im] = fft(data[im])
del list_of_particles
if CTF:
reduce_EMData_to_root(ctf_2_sum, myid, main_node)
reduce_EMData_to_root(ctf_abs_sum, myid, main_node)
if CTF:
if myid != main_node:
del ctf_2_sum
del ctf_abs_sum
else:
temp = EMData(nx, ny, 1, False)
tsnr = 1. / snr
for i in range(0, nx + 2, 2):
for j in range(ny):
temp.set_value_at(i, j, tsnr)
temp.set_value_at(i + 1, j, 0.0)
#info(ctf_2_sum)
Util.add_img(ctf_2_sum, temp)
#info(ctf_2_sum)
del temp
total_iter = 0
shift_x = [0.0] * ldata
for Iter in range(max_iter):
if myid == main_node:
start_time = time()
print_msg("Iteration #%4d\n" % (total_iter))
total_iter += 1
avg = EMData(nx, ny, 1, False)
for im in range(ldata):
Util.add_img(avg, fshift(data[im], shift_x[im]))
reduce_EMData_to_root(avg, myid, main_node)
if myid == main_node:
if CTF: tavg = Util.divn_filter(avg, ctf_2_sum)
else: tavg = Util.mult_scalar(avg, 1.0 / float(nima))
else:
tavg = model_blank(nx, ny)
if Fourvar:
bcast_EMData_to_all(tavg, myid, main_node)
vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF)
if myid == main_node:
if Fourvar:
tavg = fft(Util.divn_img(fft(tavg), vav))
vav_r = Util.pack_complex_to_real(vav)
# normalize and mask tavg in real space
tavg = fft(tavg)
stat = Util.infomask(tavg, mask, False)
tavg -= stat[0]
Util.mul_img(tavg, mask)
tavg.write_image("tavg.hdf", Iter)
# For testing purposes: shift tavg to some random place and see if the centering is still correct
#tavg = rot_shift3D(tavg,sx=3,sy=-4)
if Fourvar: del vav
bcast_EMData_to_all(tavg, myid, main_node)
tavg = fft(tavg)
sx_sum = 0.0
nxc = nx // 2
for ifil in range(nfils):
"""
# Calculate filament average
avg = EMData(nx, ny, 1, False)
filnima = 0
for im in xrange(indcs[ifil][0], indcs[ifil][1]):
Util.add_img(avg, data[im])
filnima += 1
tavg = Util.mult_scalar(avg, 1.0/float(filnima))
"""
# Calculate 1D ccf between each segment and filament average
nsegms = indcs[ifil][1] - indcs[ifil][0]
ctx = [None] * nsegms
pcoords = [None] * nsegms
for im in range(indcs[ifil][0], indcs[ifil][1]):
ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx,
1)
pcoords[im - indcs[ifil][0]] = data[im].get_attr(
'ptcl_source_coord')
#ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0])
#print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True)
# search for best x-shift
cents = nsegms // 2
dst = sqrt(
max((pcoords[cents][0] - pcoords[0][0])**2 +
(pcoords[cents][1] - pcoords[0][1])**2,
(pcoords[cents][0] - pcoords[-1][0])**2 +
(pcoords[cents][1] - pcoords[-1][1])**2))
maxincline = atan2(ny // 2 - 2 - float(search_rng), dst)
kang = int(dst * tan(maxincline) + 0.5)
#print " settings ",nsegms,cents,dst,search_rng,maxincline,kang
# ## C code for alignment. @ming
results = [0.0] * 3
results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline,
kang, search_rng, nxc)
sib = int(results[0])
bang = results[1]
qm = results[2]
#print qm, sib, bang
# qm = -1.e23
#
# for six in xrange(-search_rng, search_rng+1,1):
# q0 = ctx[cents].get_value_at(six+nxc)
# for incline in xrange(kang+1):
# qt = q0
# qu = q0
# if(kang>0): tang = tan(maxincline/kang*incline)
# else: tang = 0.0
# for kim in xrange(cents+1,nsegms):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# #print " A ", ifil,six,incline,kim,xl,ixl,dxl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# for kim in xrange(cents):
# dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2)
# xl = -dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# xl = dst*tang+six+nxc
# ixl = int(xl)
# dxl = xl - ixl
# qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1)
# if( qt > qm ):
# qm = qt
# sib = six
# bang = tang
# if( qu > qm ):
# qm = qu
# sib = six
# bang = -tang
#if incline == 0: print "incline = 0 ",six,tang,qt,qu
#print qm,six,sib,bang
#print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True)
for im in range(indcs[ifil][0], indcs[ifil][1]):
kim = im - indcs[ifil][0]
dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 +
(pcoords[cents][1] - pcoords[kim][1])**2)
if (kim < cents): xl = -dst * bang + sib
else: xl = dst * bang + sib
shift_x[im] = xl
# Average shift
sx_sum += shift_x[indcs[ifil][0] + cents]
# #print myid,sx_sum,total_nfils
sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_FLOAT, mpi.MPI_SUM,
main_node, mpi.MPI_COMM_WORLD)
if myid == main_node:
sx_sum = float(sx_sum[0]) / total_nfils
print_msg("Average shift %6.2f\n" % (sx_sum))
else:
sx_sum = 0.0
sx_sum = 0.0
sx_sum = bcast_number_to_all(sx_sum, source_node=main_node)
for im in range(ldata):
shift_x[im] -= sx_sum
#print " %3d %6.3f"%(im,shift_x[im])
#exit()
# combine shifts found with the original parameters
for im in range(ldata):
t1 = Transform()
##import random
##shix=random.randint(-10, 10)
##t1.set_params({"type":"2D","tx":shix})
t1.set_params({"type": "2D", "tx": shift_x[im]})
# combine t0 and t1
tt = t1 * init_params[im]
data[im].set_attr("xform.align2d", tt)
# write out headers and STOP, under MPI writing has to be done sequentially
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
par_str = ["xform.align2d", "ID"]
if myid == main_node:
from sp_utilities import file_type
if (file_type(stack) == "bdb"):
from sp_utilities import recv_attr_dict_bdb
recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata,
nproc)
else:
from sp_utilities import recv_attr_dict
recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc)
else:
send_attr_dict(main_node, data, par_str, 0, ldata)
if myid == main_node: print_end_msg("helical-shiftali_MPI")
def update_plot(self):
val = self.curset
ctf = self.data[val][1]
ds = self.data[val][1].dsbg
s = [ds * i for i in range(len(ctf.background))]
if self.plotmode == 1:
self.guiplot.set_data((s, self.data[val][2]), "fg", True, True)
self.guiplot.set_data((s, self.data[val][3]), "bg")
self.guiplot.setAxisParms("s (1/A)", "Intensity (a.u)")
elif self.plotmode == 0:
bgsub = [
self.data[val][2][i] - self.data[val][3][i]
for i in range(len(self.data[val][2]))
]
self.guiplot.set_data((s, bgsub), "fg-bg", True, True)
fit = ctf.compute_1d(len(s) * 2, ds,
Ctf.CtfType.CTF_AMP) # The fit curve
sf = sfact(s, "ribosome", "nono")
fit = [sf[i] * fit[i]**2 for i in range(len(s))
] # squared * a generic structure factor
# auto-amplitude for b-factor adjustment
rto, nrto = 0, 0
for i in range(int(.04 / ds) + 1, min(int(0.15 / ds), len(s) - 1)):
if bgsub[i] > 0:
#rto+=fit[i]**2/fabs(bgsub[i])
#nrto+=fit[i]
#rto+=fit[i]**2
#nrto+=bgsub[i]**2
rto += fit[i]
nrto += fabs(bgsub[i])
if nrto == 0: rto = 1.0
else: rto /= nrto
fit = [fit[i] / rto for i in range(len(s))]
self.guiplot.set_data((s, fit), "fit")
self.guiplot.setAxisParms("s (1/A)", "Intensity (a.u)")
elif self.plotmode == 2:
snr = ctf.compute_1d(len(s) * 2, ds,
Ctf.CtfType.CTF_SNR) # The snr curve
self.guiplot.set_data((s, snr[:len(s)]), "snr", True)
self.guiplot.setAxisParms("s (1/A)", "SNR (intensity ratio)")
elif self.plotmode == 3:
snr = ctf.compute_1d(len(s) * 2, ds,
Ctf.CtfType.CTF_SNR) # The snr curve
self.guiplot.set_data((s, snr[:len(s)]), "snr", True)
ssnr = ctf.compute_1d(len(s) * 2, ds,
Ctf.CtfType.CTF_SNR_SMOOTH) # The fit curve
self.guiplot.set_data((s, ssnr[:len(s)]), "ssnr")
self.guiplot.setAxisParms("s (1/A)", "SNR (intensity ratio)")
elif self.plotmode == 4:
snr = ctf.compute_1d(len(s) * 2, ds,
Ctf.CtfType.CTF_SNR) # The snr curve
for i in range(1, len(snr)):
snr[i] = snr[i] * i + snr[i - 1] # integrate SNR*s
# for i in range(1,len(snr)): snr[i]/=snr[-1] # normalize
for i in range(1, len(snr)):
snr[i] /= len(
snr
) # this way the relative quality of images can be compared
self.guiplot.set_data((s, snr[:len(s)]), "snr", True)
self.guiplot.setAxisParms("s (1/A)", "Integrated SNR")
elif self.plotmode == 5:
inten = [
fabs(i)
for i in ctf.compute_1d(len(s) * 2, ds, Ctf.CtfType.CTF_AMP)
] # The snr curve
self.guiplot.set_data((s, inten[:len(s)]), "single", True)
all = [0 for i in inten]
for st in self.data:
sxprint(st)
inten = [
fabs(i) for i in st[1].compute_1d(
len(s) * 2, ds, Ctf.CtfType.CTF_AMP)
]
for i in range(len(all)):
all[i] += inten[i]
self.guiplot.set_data((s, all[:len(s)]), "total")
