def process_stack(stackfile, phaseflip=None, wiener=None, edgenorm=True, oversamp=1, default_ctf=None, invert=False): """Will phase-flip and/or Wiener filter particles in a file based on their stored CTF parameters. phaseflip should be the path for writing the phase-flipped particles wiener should be the path for writing the Wiener filtered (and possibly phase-flipped) particles oversamp will oversample as part of the processing, ostensibly permitting phase-flipping on a wider range of defocus values """ im = EMData() im.read_image( stackfile, 0) # can't use the constructor if bdb terminology is being used ys = im.get_ysize() * oversamp ys2 = im.get_ysize() n = EMUtil.get_image_count(stackfile) lctf = None for i in range(n): im1 = EMData() im1.read_image(stackfile, i) try: ctf = im1["ctf"] except: ctf = default_ctf if type(ctf) == EMAN1Ctf: ctf = default_ctf # EMAN1 ctf needs a structure factor for this to work if edgenorm: im1.process_inplace("normalize.edgemean") if oversamp > 1: im1.clip_inplace( Region(-(ys2 * (oversamp - 1) / 2), -(ys2 * (oversamp - 1) / 2), ys, ys)) fft1 = im1.do_fft() if phaseflip: if not lctf or not lctf.equal(ctf): flipim = fft1.copy() ctf.compute_2d_complex(flipim, Ctf.CtfType.CTF_SIGN) fft1.mult(flipim) out = fft1.do_ift() out["ctf"] = ctf out.clip_inplace( Region(int(ys2 * (oversamp - 1) / 2.0), int(ys2 * (oversamp - 1) / 2.0), ys2, ys2)) if invert: out.mult(-1.0) out.process("normalize.edgemean") out.write_image(phaseflip, i) if wiener: if not lctf or not lctf.equal(ctf): wienerim = fft1.copy() ctf.compute_2d_complex(wienerim, Ctf.CtfType.CTF_WIENER_FILTER) # sxprint wienerim.get_attr_dict() # display(wienerim) # sxprint ctf.to_string() # plot(ctf.background) # plot(ctf.snr) # plot(ctf.compute_1d(ys,Ctf.CtfType.CTF_WIENER_FILTER)) fft1.mult(wienerim) out = fft1.do_ift() out["ctf"] = ctf out.clip_inplace( Region(int(ys2 * (oversamp - 1) / 2.0), int(ys2 * (oversamp - 1) / 2.0), ys2, ys2)) if invert: out.mult(-1.0) out.process("normalize.edgemean") try: out.write_image(wiener, i) except: sxprint(wiener, i) try: out.write_image(wiener, i) except: sxprint("!!! ", wiener, i) out.write_image("error.hed", -1) lctf = ctf #if wiener and wiener[:4]=="bdb:" : db_close_dict(wiener) #if phaseflip and phaseflip[:4]=="bdb:" : db_close_dict(phaseflip) return
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " stack outdir --phase=1 --ou=outer_radius|sxconsistency.py --phase=3 newlocal/main000 --ou=133 --thresherr=3.0 --params=paramsa outgrouparms" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option( "--phase", type="int", default=1, help= "Phase =1 prepares resampled stacks, =2 analyzes consistency of orientation parameters" ) parser.add_option("--ou", type="int", default=-1, help="outer radius for calculation of pixel error") parser.add_option("--sym", type="string", default="c1", help="symmetry of the refined structure") parser.add_option( "--thresherr", type="float", default=1.0, help="Threshold for accpetable orientation errors (in pixels)") parser.add_option( "--ndigits", type="int", default=1, help="Accuracy for checking whether parameters are identical") parser.add_option("--chunk", type="string", default="", help="Root of of four chunk files with indeces") parser.add_option( "--params", type="string", default="", help="Root of of six parameter file names with refinement results") (options, args) = parser.parse_args(arglist[1:]) sp_global_def.BATCH = True if options.phase == 1 and len(args) == 2: inputbdb = args[0] outdir = args[1] nn = EMUtil.get_image_count(inputbdb) t = list(range(nn)) shuffle(t) chunks = [] for i in range(4): # I use the MPI function here just to easily get the balanced load j, k = MPI_start_end(nn, 4, i) chunks.append(t[j:k]) chunks[i].sort() write_text_file(chunks[i], os.path.join(outdir, 'chunk%01d.txt' % i)) del t write_text_file([len(chunks[i]) for i in range(4)], os.path.join(outdir, 'chunklengths.txt')) """ pt = [[None]]*6 ll=0 for i in xrange(3): for j in xrange(i+1,4): pt[ll] = chunks[i]+chunks[j] ll+=1 for i in xrange(6): listfile = os.path.join(outdir,'lili%01d.txt'%i) write_text_file(pt[i],listfile) outbdb = "bdb:"+ os.path.join(outdir,"X%01d"%i) cmd = '{} {} {} {}'.format('e2bdb.py', inputbdb, '--makevstack='+outbdb, '--list='+listfile) subprocess.call(cmd, shell=True) """ # Run 6 programs elif options.phase == 2 and len(args) == 2: outdir = args[0] howmanythesame = args[1] ndigits = options.ndigits #for chc5 1, for ribo 4#4.0 prms = [] for i in range(6): prms.append( read_text_row( os.path.join(outdir, options.params + "%01d.txt" % i))) for j in range(len(prms[-1])): for k in range(5): prms[-1][j][k] = round(prms[-1][j][k], ndigits) nn = 2 * len(prms[0]) n4 = nn // 4 qt = [[[-1.0]] * nn for i in range(6)] ll = 0 for i in range(3): for j in range(i + 1, 4): qt[ll][i * n4:(i + 1) * n4] = prms[ll][:n4] qt[ll][j * n4:(j + 1) * n4] = prms[ll][n4:] ll += 1 thesame = 0 for ll in range(len(qt[0])): rw = [] for j in range(6): if (len(qt[j][ll]) > 1): rw.append(qt[j][ll]) isame = True for j in range(3): if (rw[0][j] != rw[1][j]): isame = False #print ll,rw[0][j], rw[1][j] break if (rw[0][j] != rw[2][j]): isame = False #print ll,rw[0][j], rw[2][j] break if isame: thesame += 1 qt = float(thesame) / nn sxprint("Proportion of the same orientations ", qt) write_text_file([qt], os.path.join(outdir, howmanythesame)) ######### PHASE 3 elif options.phase == 3 and len(args) == 2: outdir = args[0] outgrouparms = args[1] radius = options.ou thresherr = options.thresherr #for chc5 1, for ribo 4#4.0 sym = int(options.sym[1:]) qsym = 360.0 / sym blocks = ['A', 'B', 'C', 'D'] params = {} ll = 0 for i in range(3): for j in range(i + 1, 4): params[chr(65 + i) + chr(48 + ll)] = [] params[chr(65 + j) + chr(48 + ll)] = [] ll += 1 chunks = {} chunklengths = {} for i in range(4): chunks[chr(65 + i)] = list( map( int, read_text_file( os.path.join(outdir, options.chunk + "%01d.txt" % i)))) chunklengths[chr(65 + i)] = len(chunks[chr(65 + i)]) for i in range(6): prms = read_text_row( os.path.join(outdir, options.params + "%01d.txt" % i)) for q in blocks: if q + chr(48 + i) in params: params[q + chr(48 + i)] = prms[:chunklengths[q]] del prms[:chunklengths[q]] pairs = [["A0", "A1"], ["B3", "B4"], ["C1", "C5"], ["D2", "D4"]] lefts = ["A2", "B0", "C3", "D5"] # Compute average projection params and pixel errors avgtrans = {} pixer = {} for i, q in enumerate(pairs): avgtrans[q[0][0]] = [0.0] * chunklengths[q[0][0]] pixer[q[0][0]] = [0.0] * chunklengths[q[0][0]] for j in range(chunklengths[q[0][0]]): fifi = [params[q[0]][j], params[q[1]][j]] nas = [0.0, 0.0, 0.0] if (sym == 1): pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in range(2): n1, n2, n3 = getfvec(fifi[i][0], fifi[i][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(fifi[0][0], fifi[0][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 #if(k == 2): # print "XXXX" # print fifi[0],nas for i in range(1, 2): qnom = -1.e10 for j in range(-1, 2, 1): t1, t2, t3 = getfvec(fifi[i][0] + j * qsym, fifi[i][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # print ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 sxprint(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #print "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform([fifi[0][2:], fifi[1][2:]]) avgtrans[q[0][0]][j] = [ nphi, ntheta, twod[0], twod[1], twod[2] ] perr = {} tgood = 0 for q in blocks: perr[q] = [True] * chunklengths[q] for k in range(chunklengths[q]): if (pixer[q][k] > thresherr): perr[q][k] = False if perr[q][k]: tgood += 1 if (tgood < 4): ERROR( "No good images within the pixel error threshold specified" ) return sxprint(" tgood ", tgood) hi = hist_list( [pixer[q][k] for q in blocks for k in range(chunklengths[q])], 16) for i in range(len(hi[0])): sxprint("%4d %12.3f %12.0f " % (i, hi[0][i], hi[1][i])) # Finished, store average orientation params and table of good images # store lists of good images for each group # blocks is indexed by first letter good = [[] for i in range(4)] bad = [[] for i in range(4)] for i, q in enumerate(blocks): for k in range(chunklengths[q]): #deprt = max_3D_pixel_error(params[lefts[i]][k],avgtrans[q][k],r=radius) if perr[q][k]: good[i].append(chunks[q][k]) #[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k] ]) else: bad[i].append(chunks[q][k]) #[chunks[q][k],pixer[q][k],deprt, params[pairs[i][0]][k],params[pairs[i][1]][k],params[lefts[i]][k],avgtrans[q][k]]) write_text_file(good[i], os.path.join(outdir, "newgood%01d.txt" % i)) write_text_file(bad[i], os.path.join(outdir, "newbad%01d.txt" % i)) # write out parameters, for those in pairs write out average, for leftouts leave them as they were # These parameters refer to the original X files. ll = 0 for m in range(6): if q + chr(48 + m) in params: prmsgood = [] prmsbad = [] try: j = lefts.index(q + chr(48 + m)) for k in range(chunklengths[q]): if perr[q][k]: prmsgood.append(params[q + chr(48 + m)][k]) else: prmsbad.append(params[q + chr(48 + m)][k]) except: for k in range(chunklengths[q]): if perr[q][k]: prmsgood.append(avgtrans[q][k]) else: prmsbad.append(avgtrans[q][k]) write_text_row( prmsgood, os.path.join(outdir, "params-newgood%01d%01d.txt" % (i, ll))) write_text_row( prmsbad, os.path.join(outdir, "params-newbad%01d%01d.txt" % (i, ll))) ll += 1 # Generate newlili files from newgood, these contain original numbering of the total single file ll = 0 for i in range(3): for j in range(i + 1, 4): write_text_file(good[i] + good[j], os.path.join(outdir, "newlili%01d.txt" % ll)) ll += 1 # write out parameters, for those in pairs write out average, for leftouts leave them as they were # These parameters refer to the original X files. for i in range(6): prms = [] for q in blocks: if q + chr(48 + i) in params: try: j = lefts.index(q + chr(48 + i)) prms += params[q + chr(48 + i)] except: prms += avgtrans[q] write_text_row(prms, os.path.join(outdir, outgrouparms + "%01d.txt" % i)) # Write chunklengths chunklengths = [len(good[i]) for i in range(4)] write_text_file(chunklengths, os.path.join(outdir, "chunklengths.txt")) """ # We do not use consecutive numbering anymore # Generate newx files from newgood, these contain consecutive (with gaps) numbering that allows to generate truncated X files from the previous X files ll = 0 for i in xrange(3): firstblock = [] l = len(perr[blocks[i]]) for k in xrange(l): if perr[blocks[i]][k]: firstblock.append(k) for j in xrange(i+1,4): secondblock = [] for k in xrange(len(perr[blocks[j]])): if perr[blocks[j]][k]: secondblock.append(k+l) write_text_file( firstblock + secondblock, os.path.join(outdir,"goodX%01d.txt"%ll) ) ll += 1 del good,bad,firstblock,secondblock,perr """ """ 0 1 2 3 4 5 A A A = = = B = = B B = = C = C = C = = D = D D """ elif options.phase == 4 and len(args) == 1: outdir = args[0] bp = 'badparams' #outgrouparms= args[1] radius = options.ou thresherr = options.thresherr sym = int(options.sym[1:]) qsym = 360.0 / sym #params = [[None for i in xrange(3)] for j in xrange(4)] ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general. for jj in range(4): params = [None for ii in range(ll)] newbad = list( map(int, read_text_file(options.params + "%01d.txt" % jj))) nn = len(newbad) for ii in range(ll): params[ii] = read_text_row( os.path.join(outdir, bp + "%01d%01d.txt" % (jj, ii))) assert (nn == len(params[ii])) # Compute average projection params and pixel errors avgtrans = [None] * nn pixer = [0.0] * nn for j in range(nn): nas = [0.0, 0.0, 0.0] if (sym == 1): #pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in range(ll): n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 for i in range(1, ll): qnom = -1.e10 for j in range(-1, 2, 1): t1, t2, t3 = getfvec(params[i][j][0] + j * qsym, params[i][j][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # sxprint ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 sxprint(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #sxprint "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform( [params[ii][j][2:] for ii in range(ll)]) avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]] perr = errors_per_image(params, avgtrans, thresherr, radius) rescued = [] rejects = [] for j in range(nn): #sxprint chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)] if (perr[j][0] <= thresherr): rescued.append([newbad[j], j]) else: rejects.append([newbad[j], j]) if (len(rescued) == 0): write_text_row([-1, -1], os.path.join(outdir, "rescued%01d.txt" % jj)) else: write_text_row(rescued, os.path.join(outdir, "rescued%01d.txt" % jj)) # We also have to write params. if (len(rescued) != 0): for ii in range(ll): write_text_row( [ params[ii][rescued[k][1]] for k in range(len(rescued)) ], os.path.join(outdir, "params-rescued%01d%01d.txt" % (jj, ii))) if (len(rejects) == 0): write_text_row([-1, -1], os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) else: write_text_row( rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) if (len(rejects) != 0): for ii in range(ll): write_text_row( [ params[ii][rejects[k][1]] for k in range(len(rejects)) ], os.path.join(outdir, "params-rejects%01d%01d.txt" % (jj, ii))) hi = hist_list([perr[j][0] for j in range(nn)], 16) sxprint("Pixel errors for BAD GROUP ", chr(65 + jj)) for ii in range(len(hi[0])): sxprint("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii])) elif options.phase == 5 and len(args) == 1: # This version is for meridien refinement. There are simply three full sets of params. outdir = args[0] bp = 'badparams' #outgrouparms= args[1] radius = options.ou thresherr = options.thresherr sym = int(options.sym[1:]) qsym = 360.0 / sym #params = [[None for i in xrange(3)] for j in xrange(4)] ll = 3 # this is hardwired as we have three groups. however, I would like to keep the code general. for jj in range(1): params = [None for ii in range(ll)] #newbad = map(int, read_text_file(options.params+"%01d.txt"%jj) ) #nn = len(newbad) for ii in range(ll): params[ii] = read_text_row( os.path.join(outdir, "params%01d.txt" % (ii))) #assert(nn == len(params[ii]) ) nn = len(params[0]) newbad = list(range(nn)) # Compute average projection params and pixel errors avgtrans = [None] * nn pixer = [0.0] * nn for j in range(nn): nas = [0.0, 0.0, 0.0] if (sym == 1): #pixer[q[0][0]][j] = max_3D_pixel_error(fifi[0], fifi[1], r=radius) for i in range(ll): n1, n2, n3 = getfvec(params[i][j][0], params[i][j][1]) nas[0] += n1 nas[1] += n2 nas[2] += n3 else: m1, m2, m3 = getfvec(params[0][j][0], params[0][j][1]) nas[0] = m1 nas[1] = m2 nas[2] = m3 for i in range(1, ll): qnom = -1.e10 for j in range(-1, 2, 1): t1, t2, t3 = getfvec(params[i][j][0] + j * qsym, params[i][j][1]) nom = t1 * m1 + t2 * m2 + t3 * m3 if (nom > qnom): qnom = nom n1 = t1 n2 = t2 n3 = t3 #if(k == 2): # sxprint ' t1,t2,t3 ',fifi[i][0]+j*qsym,fifi[i][1],t1,t2,t3,nom,qnom nas[0] += n1 nas[1] += n2 nas[2] += n3 sxprint(qnom, n1, n2, n3, nas) # To get the correct pixer phi angle has to be taken from the above!! nom = sqrt(nas[0]**2 + nas[1]**2 + nas[2]**2) if (nom < 1.e-6): nphi = 0.0 ntheta = 0.0 else: ntheta = degrees(acos(nas[2] / nom)) % 360.0 if (sym > 1 and ntheta > 90.0): nphi = (degrees(atan2(nas[1], nas[0])) - 180.0) % qsym + 180.0 else: nphi = degrees(atan2(nas[1], nas[0])) % qsym #sxprint "FIFI %4d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f"%(k,fifi[0][0],fifi[0][1],fifi[1][0],fifi[1][1],fifi[2][0],fifi[2][1],nphi,ntheta) twod = average2dtransform( [params[ii][j][2:] for ii in range(ll)]) avgtrans[j] = [nphi, ntheta, twod[0], twod[1], twod[2]] write_text_row(avgtrans, os.path.join(outdir, "avgtrans.txt")) perr = errors_per_image(params, avgtrans, thresherr, radius) rescued = [] rejects = [] for j in range(nn): #sxprint chr(65+jj),j,perr[j][0],[[params[m][j][i] for i in xrange(2)] for m in xrange(ll)] if (perr[j][0] <= thresherr): rescued.append([newbad[j], j]) else: rejects.append([newbad[j], j]) if (len(rescued) == 0): write_text_row([-1, -1], os.path.join(outdir, "rescued%01d.txt" % jj)) else: write_text_row(rescued, os.path.join(outdir, "rescued%01d.txt" % jj)) # We also have to write params. if (len(rescued) != 0): for ii in range(ll): write_text_row( [ params[ii][rescued[k][1]] for k in range(len(rescued)) ], os.path.join(outdir, "params-rescued%01d%01d.txt" % (jj, ii))) if (len(rejects) == 0): write_text_row([-1, -1], os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) else: write_text_row( rejects, os.path.join(outdir, 'rejects' + "%01d.txt" % jj)) if (len(rejects) != 0): for ii in range(ll): write_text_row( [ params[ii][rejects[k][1]] for k in range(len(rejects)) ], os.path.join(outdir, "params-rejects%01d%01d.txt" % (jj, ii))) hi = hist_list([perr[j][0] for j in range(nn)], 16) sxprint("Pixel errors for BAD GROUP ", chr(65 + jj)) for ii in range(len(hi[0])): sxprint("%4d %12.3f %12.0f " % (ii, hi[0][ii], hi[1][ii])) else: sxprint("Usage: ") sxprint(""" Phase 1: sxconsistency.py --phase=1 bdb:data outdir output files are: in directory outdir: lili0.txt to lili5.txt contain indices of images in resampled six groups bdb:dataX0 to bdb:dataX5 are metafiles containing six resampled groups of images derived from bdb:data Phase 2 sxconsistency.py --phase=2 outdir --ndigits=1 --params=paramsb howmanythesame.txt outdir - directory containing files lili0.txt to lili5.txt produced in phase 1 --params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be master/main/params0.txt to master/main/params5.txt output files: howmanythesame.txt - contains one number, a ratio of number of images that did not change orientations to the total number of images Phase 3: sxconsistency.py --phase=3 outdir --ou=133 --thresherr=3.0 --params=paramsa outgrouparms input files: outdir - directory containing files lili0.txt to lili5.txt produced in phase 1 --params=master/main/params - Root of of six parameter file names with refinement results, the actual names should be master/main/params0.txt to master/main/params5.txt output files: outgrouparms*.txt - Root of of six parameters files with average 3D orientation parameters computed from six runs, can be imported into bdb:data The next two files contain the original image numbers refering to the top bdb. outdir/newgood.txt - Text file with indices of images whose orientation parameters arrors are below specified thresherr (to be kept) outdir/bad.txt - Text file with indices of images whose orientation parameters arrors are above specified thresherr (to be rejected) Phase 4: sxconsistency.py --phase=4 outdir --ou=133 --thresherr=3.0 --params=master/main001/newbad input files: outdir - directory containing files badparamsij.txt i= 0,3, j=0,2, which resulted from three alignments of four badchunk images --params= - Root of four files with original indices of bad images, they will be read and a subset corresponding to rescued ones will be outputed output files: rescued*.txt - four files with indices of accepted images from badchunk. rejects*.txt - four files with indices of rejected images from badchunk. There are two columns: [number in original bdb:chunk, number in bdb:newbad] """) sxprint("Please run '" + progname + " -h' for detailed options") sp_global_def.BATCH = False
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = optparse.os.path.basename(arglist[0]) usage = progname + """ inputvolume locresvolume maskfile outputfile --radius --falloff --MPI Locally filer a volume based on local resolution volume (sxlocres.py) within area outlined by the maskfile """ parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION) parser.add_option( "--radius", type="int", default=-1, help= "if there is no maskfile, sphere with r=radius will be used, by default the radius is nx/2-1" ) parser.add_option("--falloff", type="float", default=0.1, help="falloff of tanl filter (default 0.1)") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args(arglist[1:]) if len(args) < 3 or len(args) > 4: sp_global_def.sxprint("See usage " + usage) sp_global_def.ERROR( "Wrong number of parameters. Please see usage information above.") return if sp_global_def.CACHE_DISABLE: pass #IMPORTIMPORTIMPORT from sp_utilities import disable_bdb_cache sp_utilities.disable_bdb_cache() if options.MPI: number_of_proc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD) main_node = 0 if (myid == main_node): #print sys.argv vi = sp_utilities.get_im(sys.argv[1]) ui = sp_utilities.get_im(sys.argv[2]) #print Util.infomask(ui, None, True) radius = options.radius nx = vi.get_xsize() ny = vi.get_ysize() nz = vi.get_zsize() dis = [nx, ny, nz] else: falloff = 0.0 radius = 0 dis = [0, 0, 0] vi = None ui = None dis = sp_utilities.bcast_list_to_all(dis, myid, source_node=main_node) if (myid != main_node): nx = int(dis[0]) ny = int(dis[1]) nz = int(dis[2]) radius = sp_utilities.bcast_number_to_all(radius, main_node) if len(args) == 3: if (radius == -1): radius = min(nx, ny, nz) // 2 - 1 m = sp_utilities.model_circle(radius, nx, ny, nz) outvol = args[2] elif len(args) == 4: if (myid == main_node): m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5) else: m = sp_utilities.model_blank(nx, ny, nz) outvol = args[3] sp_utilities.bcast_EMData_to_all(m, myid, main_node) pass #IMPORTIMPORTIMPORT from sp_filter import filterlocal filteredvol = sp_filter.filterlocal(ui, vi, m, options.falloff, myid, main_node, number_of_proc) if (myid == 0): filteredvol.write_image(outvol) else: vi = sp_utilities.get_im(args[0]) ui = sp_utilities.get_im( args[1] ) # resolution volume, values are assumed to be from 0 to 0.5 nn = vi.get_xsize() falloff = options.falloff if len(args) == 3: radius = options.radius if (radius == -1): radius = nn // 2 - 1 m = sp_utilities.model_circle(radius, nn, nn, nn) outvol = args[2] elif len(args) == 4: m = sp_morphology.binarize(sp_utilities.get_im(args[2]), 0.5) outvol = args[3] sp_fundamentals.fftip(vi) # this is the volume to be filtered # Round all resolution numbers to two digits for x in range(nn): for y in range(nn): for z in range(nn): ui.set_value_at_fast(x, y, z, round(ui.get_value_at(x, y, z), 2)) st = EMAN2_cppwrap.Util.infomask(ui, m, True) filteredvol = sp_utilities.model_blank(nn, nn, nn) cutoff = max(st[2] - 0.01, 0.0) while (cutoff < st[3]): cutoff = round(cutoff + 0.01, 2) pt = EMAN2_cppwrap.Util.infomask( sp_morphology.threshold_outside(ui, cutoff - 0.00501, cutoff + 0.005), m, True) if (pt[0] != 0.0): vovo = sp_fundamentals.fft( sp_filter.filt_tanl(vi, cutoff, falloff)) for x in range(nn): for y in range(nn): for z in range(nn): if (m.get_value_at(x, y, z) > 0.5): if (round(ui.get_value_at(x, y, z), 2) == cutoff): filteredvol.set_value_at_fast( x, y, z, vovo.get_value_at(x, y, z)) sp_global_def.write_command(optparse.os.path.dirname(outvol)) filteredvol.write_image(outvol)
def main(): progname = os.path.basename(sys.argv[0]) usage = """%prog [options] input.pdb output.hdf Converts a pdb file into an electron density map. 0,0,0 in PDB space will map to the center of the volume.""" parser = OptionParser(usage=usage, version=EMANVERSION) parser.add_option("--apix", "-A", type="float", help="Angstrom/voxel", default=1.0) parser.add_option("--box", "-B", type="string", help="Box size in pixels, <xyz> or <x,y,z>") parser.add_option("--het", action="store_true", help="Include HET atoms in the map", default=False) parser.add_option( "--chains", type="string", help= "String list of chain identifiers to include, e.g. 'ABEFG'; default: include all chains", default='') parser.add_option( "--center", type="string", default="a", help= "center: c - coordinates; a (default) - center of gravity; <x,y,z> - vector (in Angstrom) to subtract from all coordinates; n - none" ) parser.add_option("--O", action="store_true", default=False, help="use O system of coordinates") parser.add_option("--quiet", action="store_true", default=False, help="Verbose is the default") parser.add_option("--tr0", type="string", default="none", help="Filename of initial 3x4 transformation matrix") parser.add_option("--set_apix_value", action="store_true", help="Set apix value in header of the ouput map", default=False) (options, args) = parser.parse_args() # if len(args) < 2: ERROR("Input and output files required") return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() chains = options.chains if chains == '': chains = None try: infile = open(args[0], "r") except: ERROR("Cannot open input file") return aavg = [0, 0, 0] # calculate atomic center asig = [0, 0, 0] # to compute radius of gyration natm = 0 atoms = [] # we store a list of atoms to process to avoid multiple passes nelec = 0 mass = 0 # read in initial-transformation file: if (options.tr0 != "none"): cols = read_text_file(options.tr0, -1) txlist = [] for i in range(3): txlist.append(cols[0][i]) txlist.append(cols[1][i]) txlist.append(cols[2][i]) txlist.append(cols[3][i]) tr0 = Transform(txlist) # parse the pdb file and pull out relevant atoms for line in infile: if (line[:4] == 'ATOM' or (line[:6] == 'HETATM' and options.het)): if chains and (line[21] not in chains): continue try: a = line[12:14].strip() aseq = int(line[6:11].strip()) res = int(line[22:26].strip()) if (options.O): x = float(line[38:46]) y = float(line[30:38]) z = -float(line[46:54]) else: x = float(line[30:38]) y = float(line[38:46]) z = float(line[46:54]) except: sxprint( "PDB Parse error:\n%s\n'%s','%s','%s' '%s','%s','%s'\n" % (line, line[12:14], line[6:11], line[22:26], line[30:38], line[38:46], line[46:54])) sxprint(a, aseq, res, x, y, z) try: nelec += atomdefs[a.upper()][0] mass += atomdefs[a.upper()][1] except: sxprint(("Unknown atom %s ignored at %d" % (a, aseq))) atoms.append([a, x, y, z]) natm += 1 if (options.center == "a"): aavg[0] += x * atomdefs[a.upper()][1] aavg[1] += y * atomdefs[a.upper()][1] aavg[2] += z * atomdefs[a.upper()][1] asig[0] += x**2 * atomdefs[a.upper()][1] asig[1] += y**2 * atomdefs[a.upper()][1] asig[2] += z**2 * atomdefs[a.upper()][1] else: aavg[0] += x aavg[1] += y aavg[2] += z asig[0] += x**2 asig[1] += y**2 asig[2] += z**2 infile.close() if (options.center == "a"): rad_gyr = sqrt((asig[0] + asig[1] + asig[2]) / mass - (aavg[0] / mass)**2 - (aavg[1] / mass)**2 - (aavg[2] / mass)**2) else: rad_gyr = sqrt((asig[0] + asig[1] + asig[2]) / natm - (aavg[0] / natm)**2 - (aavg[1] / natm)**2 - (aavg[2] / natm)**2) if not options.quiet: sxprint( "%d atoms; total charge = %d e-; mol mass = %.2f kDa; radius of gyration = %.2f A" % (natm, nelec, mass / 1000.0, rad_gyr)) # center PDB according to option: if (options.center == "a"): if not options.quiet: sxprint( "center of gravity at %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)" % (aavg[0] / mass, aavg[1] / mass, aavg[2] / mass)) for i in range(len(atoms)): atoms[i][1] -= aavg[0] / mass atoms[i][2] -= aavg[1] / mass atoms[i][3] -= aavg[2] / mass if (options.center == "c"): if not options.quiet: sxprint( "atomic center at %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)" % (aavg[0] / natm, aavg[1] / natm, aavg[2] / natm)) for i in range(len(atoms)): atoms[i][1] -= aavg[0] / natm atoms[i][2] -= aavg[1] / natm atoms[i][3] -= aavg[2] / natm spl = options.center.split(',') if len(spl) == 3: # substract the given vector from all coordinates if not options.quiet: sxprint( "vector to substract: %1.1f,%1.1f,%1.1f (center of volume at 0,0,0)" % (float(spl[0]), float(spl[1]), float(spl[2]))) for i in range(len(atoms)): atoms[i][1] -= float(spl[0]) atoms[i][2] -= float(spl[1]) atoms[i][3] -= float(spl[2]) # apply given initial transformation (this used to be done before centering, # thereby loosing the translation. This is the right place to apply tr0): if (options.tr0 != "none"): if not options.quiet: sxprint("Applying initial transformation to PDB coordinates... ") for i in range(len(atoms)): atom_coords = Vec3f(atoms[i][1], atoms[i][2], atoms[i][3]) new_atom_coords = tr0 * atom_coords atoms[i][1] = new_atom_coords[0] atoms[i][2] = new_atom_coords[1] atoms[i][3] = new_atom_coords[2] if not options.quiet: sxprint("done.\n") # bounding box: amin = [atoms[0][1], atoms[0][2], atoms[0][3]] amax = [atoms[0][1], atoms[0][2], atoms[0][3]] for i in range(1, len(atoms)): for k in range(3): amin[k] = min(atoms[i][k + 1], amin[k]) amax[k] = max(atoms[i][k + 1], amax[k]) if not options.quiet: sxprint("Range of coordinates [A]: x: %7.2f - %7.2f" % (amin[0], amax[0])) sxprint(" y: %7.2f - %7.2f" % (amin[1], amax[1])) sxprint(" z: %7.2f - %7.2f" % (amin[2], amax[2])) # find the output box size, either user specified or from bounding box box = [0, 0, 0] try: spl = options.box.split(',') if len(spl) == 1: box[0] = box[1] = box[2] = int(spl[0]) else: box[0] = int(spl[0]) box[1] = int(spl[1]) box[2] = int(spl[2]) except: for i in range(3): box[i] = int(2 * max(fabs(amax[i]), fabs(amin[i])) / options.apix) # Increase the box size by 1/4. box[i] += box[i] // 4 if not options.quiet: sxprint("Bounding box [pixels]: x: %5d " % box[0]) sxprint(" y: %5d " % box[1]) sxprint(" z: %5d " % box[2]) # figure oversampled box size #bigb = max(box[0],box[1],box[2]) fcbig = 1 """Multiline Comment0""" #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 #MULTILINEMULTILINEMULTILINE 0 if not options.quiet: sxprint("Box size: %d x %d x %d" % (box[0], box[1], box[2]), ", oversampling ", fcbig) # Calculate working dimensions pixelbig = options.apix / fcbig bigbox = [] for i in range(3): bigbox.append(box[i] * fcbig) # initialize the final output volume outmap = EMData(bigbox[0], bigbox[1], bigbox[2], True) nc = [] for i in range(3): nc.append(bigbox[i] // 2) # fill in the atoms for i in range(len(atoms)): #print "Adding %d '%s'"%(i,atoms[i][0]) if not options.quiet and i % 1000 == 0: sxprint('\r %d' % i, end=' ') sys.stdout.flush() try: elec = atomdefs[atoms[i][0].upper()][0] #outmap[int(atoms[i][1]/pixelbig+bigbox[0]//2),int(atoms[i][2]/pixelbig+bigbox[1]//2),int(atoms[i][3]/pixelbig+bigbox[2]//2)] += elec for k in range(2): pz = atoms[i][3] / pixelbig + nc[2] dz = pz - int(pz) uz = ((1 - k) + (2 * k - 1) * dz) * elec for l in range(2): py = atoms[i][2] / pixelbig + nc[1] dy = py - int(py) uy = ((1 - l) + (2 * l - 1) * dy) * uz for m in range(2): px = atoms[i][1] / pixelbig + nc[0] dx = px - int(px) outmap[int(px) + m, int(py) + l, int(pz) + k] += ((1 - m) + (2 * m - 1) * dx) * uy except: sxprint("Skipping %d '%s'" % (i, atoms[i][0])) if not options.quiet: sxprint('\r %d\nConversion complete.' % len(atoms)) #," Now shape atoms." """Multiline Comment1""" #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 #MULTILINEMULTILINEMULTILINE 1 (filename_path, filextension) = os.path.splitext(args[1]) if filextension == ".hdf": if options.set_apix_value: outmap.set_attr("apix_x", options.apix) outmap.set_attr("apix_y", options.apix) outmap.set_attr("apix_z", options.apix) outmap.set_attr("pixel_size", options.apix) else: sxprint("Pixel_size is not set in the header!") outmap.write_image(args[1], 0, EMUtil.ImageType.IMAGE_HDF) elif filextension == ".spi": outmap.write_image(args[1], 0, EMUtil.ImageType.IMAGE_SINGLE_SPIDER) else: ERROR("Unknown image type") return
def _main_(): _, child_argparser = setup_argparser() # These global variables are kind of ugly, but are necessary in python2. Will be removed in # python 3. global STACK_FILE_PATH global PROJECTION_PARAMETERS global VOLUME1 global VOLUME2 global DEFOCUS_SEARCH_RANGE global DEFOCUS_STEP_SIZE global MASK_VOLUME global RESOLUTION global PIXEL_SIZE global HALF_MAP_ASSIGNEMENTS args = child_argparser.parse_args() if "meridien" in sys.argv[1]: meridien_path = args.meridien_path files = sp_ctf_refine_io.read_meridien_data(meridien_path) volume1_file_path = files["first_halfmap"] volume2_file_path = files["second_halfmap"] chunk_file_path = files["chunk1"] params_file_path = files["final_params"] else: volume1_file_path = args.volume volume2_file_path = args.volume2 params_file_path = args.params_path chunk_file_path = args.chunk if volume2_file_path is None and chunk_file_path is not None: sp_global_def.ERROR( "If chunk file is specified, you need to specify a second volume (-v2)" ) STACK_FILE_PATH = args.inputstack # "bdb:name" mask_file_path = args.mask DEFOCUS_SEARCH_RANGE = args.range DEFOCUS_STEP_SIZE = args.delta output_folder = args.outputdir if os.path.exists(output_folder): sp_global_def.ERROR("Output folder already exists. Stop execution.") else: os.makedirs(output_folder) sp_global_def.write_command(output_folder) output_virtual_stack_path = "bdb:" + os.path.join(output_folder, "ctf_refined") output_stats_path = os.path.join(output_folder, "statistics") number_of_particles_to_read = args.number_particles volume_nominal_resolution = args.resolution RESOLUTION = volume_nominal_resolution PIXEL_SIZE = args.apix PROJECTION_PARAMETERS = sp_ctf_refine_io.read_meridien_params(params_file_path) num_cpu = multiprocessing.cpu_count() - 1 volume1, volume2, MASK_VOLUME = sp_ctf_refine_io.read_volume( path_vol_1=volume1_file_path, path_vol_2=volume2_file_path, path_mask=mask_file_path, resolution=volume_nominal_resolution, pixel_size=PIXEL_SIZE, ) VOLUME1 = volume1 VOLUME2 = volume2 particle_chunks, number_of_particles = indices_to_chunks( STACK_FILE_PATH, chunk_size=100, number_of_particles_to_read=number_of_particles_to_read, ) if chunk_file_path: HALF_MAP_ASSIGNEMENTS = get_half_map_assigments_per_particle( stack_path=STACK_FILE_PATH, chunk_path=chunk_file_path, number_of_particles_to_read=number_of_particles_to_read, ) sp_global_def.sxprint("####Start refinement####") start = time.time() # for chunk in particle_chunks: # refine_set(chunk) ##################################################################### # Python 2 workaround to get rid of a memory leak. # Pool eats up memory and only give it back after it returns. # So let it return more often by deviding chunks into chunks :-) ##################################################################### num_chunks = len(particle_chunks) refinement_results = [] with tqdm(total=num_chunks, file=sys.stdout) as pbar: for i in range(0, num_chunks, num_cpu): subset_chunk = particle_chunks[i: (i + num_cpu)] pool = multiprocessing.Pool(num_cpu) refinement_result = pool.map_async(refine_set, subset_chunk, chunksize=1) pool.close() # print_progress(refinement_result) pool.join() for res in refinement_result.get(): refinement_results.append(res) pbar.update(len(subset_chunk)) ##################################################################### end = time.time() sp_global_def.sxprint("Time for ", number_of_particles, " Particles:", end - start) # Ouput results refined_ctfs_as_list, refinement_results_per_micrograph = merge_ctf_refinement_results( refinement_results=refinement_results ) sp_ctf_refine_io.write_virtual_bdb_stack( output_stack_path=output_virtual_stack_path, origin_stack_path=STACK_FILE_PATH, refined_ctfs=refined_ctfs_as_list, number_of_particles=number_of_particles, ) sp_global_def.sxprint("Write statistics...") # WRITE STATISTICS if not os.path.exists(output_stats_path): os.makedirs(output_stats_path) refinement_stats_per_micrograh = calc_statistics(refinement_results_per_micrograph) sp_ctf_refine_io.write_statistics(output_stats_path, refinement_stats_per_micrograh) # Estimate error Range min_max_error, min_max_ratio = calculate_result_ranges( refinement_results_per_micrograph ) # Save particle plots sp_global_def.sxprint("Write images...") path_output_img = os.path.join(output_stats_path, "img/") if not os.path.exists(path_output_img): os.makedirs(path_output_img) sp_ctf_refine_plotting.create_and_save_particle_plots( path_output_img=path_output_img, stack_file_path=STACK_FILE_PATH, refinement_results_per_micrograph=refinement_results_per_micrograph, min_max_error=min_max_error, min_max_ratio=min_max_ratio, ) sp_global_def.sxprint("Write other...") refinement_results_matrix = get_refinement_results_matrix( refinement_results_per_micrograph ) particle_error_path = os.path.join(output_stats_path, "particle_results.txt") refinement_results_matrix = refinement_results_matrix[ refinement_results_matrix[:, 0].argsort() ] np.savetxt( particle_error_path, refinement_results_matrix, delimiter=",", fmt=["%d", "%f", "%f", "%f"], ) sp_global_def.sxprint("Done")
def main(): """ Main function. Arguments: None Returns: None """ command_args = parse_command_line() # Import volume sp_global_def.sxprint("Import volume.") input_vol = sp_utilities.get_im(command_args.input_volume) # Sanity checks sanity_checks(command_args, input_vol) try: os.makedirs(command_args.output_dir) except OSError: sp_global_def.sxprint( "Output directory already exists. No need to create it.") else: sp_global_def.sxprint("Created output directory.") sp_global_def.write_command(command_args.output_dir) output_prefix = os.path.join(command_args.output_dir, command_args.prefix) # Filter volume if specified if command_args.low_pass_filter_resolution is not None: sp_global_def.sxprint("Filter volume to {0}A.".format( command_args.low_pass_filter_resolution)) input_vol = sp_filter.filt_tanl( input_vol, old_div(command_args.pixel_size, command_args.low_pass_filter_resolution), command_args.low_pass_filter_falloff, ) input_vol.write_image(output_prefix + "_filtered_volume.hdf") else: sp_global_def.sxprint("Skip filter volume.") # Create a mask based on the filtered volume sp_global_def.sxprint("Create mask") density_threshold = -9999.0 nsigma = 1.0 if command_args.mol_mass: density_threshold = input_vol.find_3d_threshold( command_args.mol_mass, command_args.pixel_size) sp_global_def.sxprint( "Mask molecular mass translated into binary threshold: ", density_threshold) elif command_args.threshold: density_threshold = command_args.threshold elif command_args.nsigma: nsigma = command_args.nsigma else: assert False if command_args.edge_type == "cosine": mode = "C" elif command_args.edge_type == "gaussian": mode = "G" else: assert False mask_first = sp_morphology.adaptive_mask_scipy( input_vol, nsigma=nsigma, threshold=density_threshold, ndilation=command_args.ndilation, nerosion=command_args.nerosion, edge_width=command_args.edge_width, allow_disconnected=command_args.allow_disconnected, mode=mode, do_approx=command_args.do_old, do_fill=command_args.fill_mask, do_print=True, ) # Create a second mask based on the filtered volume s_mask = None s_density_threshold = 1 s_nsigma = 1.0 if command_args.second_mask is not None: sp_global_def.sxprint("Prepare second mask") s_mask = sp_utilities.get_im(command_args.second_mask) s_density_threshold = -9999.0 s_nsigma = 1.0 if command_args.s_mol_mass: s_density_threshold = input_vol.find_3d_threshold( command_args.s_mol_mass, command_args.s_pixel_size) sp_global_def.sxprint( "Second mask molecular mass translated into binary threshold: ", s_density_threshold, ) elif command_args.s_threshold: s_density_threshold = command_args.s_threshold elif command_args.s_nsigma: s_nsigma = command_args.s_nsigma else: assert False elif command_args.second_mask_shape is not None: sp_global_def.sxprint("Prepare second mask") nx = mask_first.get_xsize() ny = mask_first.get_ysize() nz = mask_first.get_zsize() if command_args.second_mask_shape == "cube": s_nx = command_args.s_nx s_ny = command_args.s_ny s_nz = command_args.s_nz s_mask = sp_utilities.model_blank(s_nx, s_ny, s_nz, 1) elif command_args.second_mask_shape == "cylinder": s_radius = command_args.s_radius s_nx = command_args.s_nx s_ny = command_args.s_ny s_nz = command_args.s_nz try: s_mask = sp_utilities.model_cylinder(s_radius, s_nx, s_ny, s_nz) except RuntimeError as e: sp_global_def.sxprint( "An error occured! Please check the error log") raise elif command_args.second_mask_shape == "sphere": s_radius = command_args.s_radius s_nx = command_args.s_nx s_ny = command_args.s_ny s_nz = command_args.s_nz try: s_mask = sp_utilities.model_circle(s_radius, s_nx, s_ny, s_nz) except RuntimeError as e: sp_global_def.sxprint( "An error occured! Please check the error log") raise else: assert False s_mask = sp_utilities.pad(s_mask, nx, ny, nz, 0) if s_mask is not None: sp_global_def.sxprint("Create second mask") if command_args.s_edge_type == "cosine": mode = "C" elif command_args.s_edge_type == "gaussian": mode = "G" else: assert False s_mask = sp_morphology.adaptive_mask_scipy( s_mask, nsigma=s_nsigma, threshold=s_density_threshold, ndilation=command_args.s_ndilation, nerosion=command_args.s_nerosion, edge_width=command_args.s_edge_width, allow_disconnected=command_args.s_allow_disconnected, mode=mode, do_approx=command_args.s_do_old, do_fill=command_args.s_fill_mask, do_print=True, ) if command_args.s_invert: s_mask = 1 - s_mask sp_global_def.sxprint("Write outputs.") mask_first.write_image(output_prefix + "_mask_first.hdf") s_mask.write_image(output_prefix + "_mask_second.hdf") masked_combined = mask_first * s_mask masked_combined.write_image(output_prefix + "_mask.hdf") else: sp_global_def.sxprint("Write outputs.") mask_first.write_image(output_prefix + "_mask.hdf")
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack outdir <maskfile> --K=10 --trials=2 --debug --maxit=100 --rand_seed=10 --crit='all' --F=0.9 --T0=2.0 --init_method='rnd' --normalize --CTF --MPI --CUDA" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--K", type="int", default=2, help="Number of classes (default 2)") parser.add_option("--trials", type="int", default=1, help="Number of trials of K-means (default 1)") parser.add_option("--maxit", type="int", default=100, help="Maximum number of iterations within K-means") parser.add_option("--CTF", action="store_true", default=False, help="Perform classification using CTF information") parser.add_option("--rand_seed", type="int", default=-1, help="Random seed of initial (default random)") parser.add_option( "--crit", type="string", default="D", help= "Criterions: Coleman [C], Harabasz[H], Davies-Bouldin[D], All [all]") #parser.add_option("--F", type="float", default=0.0, help="Cooling in simulated annealing, ex.: 0.9") #parser.add_option("--T0", type="float", default=0.0, help="Initial temperature in simulated annealing, ex: 100") parser.add_option("--MPI", action="store_true", default=False, help="Use MPI version") parser.add_option("--debug", action="store_true", default=False, help="") parser.add_option("--normalize", action="store_true", default=False, help="Normalize images under the mask") parser.add_option( '--init_method', type='string', default='rnd', help= 'Method used to initialize partition: "rnd" randomize or "d2w" for d2 weighting initialization (default is rnd)' ) (options, args) = parser.parse_args() if len(args) < 2 or len(args) > 3: sxprint("usage: " + usage) sxprint("Please run '" + progname + " -h' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return elif options.trials < 1: ERROR("Number of trials should be at least 1") return else: if len(args) == 2: mask = None else: mask = args[2] if options.K < 2: ERROR("K must be > 1 group") return if options.CTF: ERROR("CTF option not implemented") return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import k_means_main sp_global_def.BATCH = True k_means_main(args[0], args[1], mask, "SSE", options.K, options.rand_seed, options.maxit, options.trials, options.crit, options.CTF, 0.0, 0.0, options.MPI, False, options.debug, options.normalize, options.init_method) sp_global_def.BATCH = False
def resample( prjfile, outdir, bufprefix, nbufvol, nvol, seedbase,\ delta, d, snr, CTF, npad,\ MPI, myid, ncpu, verbose = 0 ): from sp_utilities import even_angles from random import seed, jumpahead, shuffle import os from sys import exit nprj = EMUtil.get_image_count( prjfile ) if MPI: if myid == 0: if os.path.exists(outdir): nx = 1 else: nx = 0 else: nx = 0 ny = bcast_number_to_all(nx, source_node = 0) if ny == 1: ERROR('Output directory exists, please change the name and restart the program', "resample", 1,myid) mpi.mpi_barrier( mpi.MPI_COMM_WORLD ) if myid == 0: os.makedirs(outdir) sp_global_def.write_command(outdir) mpi.mpi_barrier( mpi.MPI_COMM_WORLD ) else: if os.path.exists(outdir): ERROR('Output directory exists, please change the name and restart the program', "resample", 1,0) os.makedirs(outdir) sp_global_def.write_command(outdir) if(verbose == 1): finfo=open( os.path.join(outdir, "progress%04d.txt" % myid), "w" ) else: finfo = None #print " before evenangles",myid from sp_utilities import getvec from numpy import array, reshape refa = even_angles(delta) nrefa = len(refa) refnormal = zeros((nrefa,3),'float32') tetref = [0.0]*nrefa for i in range(nrefa): tr = getvec( refa[i][0], refa[i][1] ) for j in range(3): refnormal[i][j] = tr[j] tetref[i] = refa[i][1] del refa vct = array([0.0]*(3*nprj),'float32') if myid == 0: sxprint(" will read ",myid) tr = EMUtil.get_all_attributes(prjfile,'xform.projection') tetprj = [0.0]*nprj for i in range(nprj): temp = tr[i].get_params("spider") tetprj[i] = temp["theta"] if(tetprj[i] > 90.0): tetprj[i] = 180.0 - tetprj[i] vct[3*i+0] = tr[i].at(2,0) vct[3*i+1] = tr[i].at(2,1) vct[3*i+2] = tr[i].at(2,2) del tr else: tetprj = [0.0]*nprj #print " READ ",myid if MPI: #print " will bcast",myid vct = mpi.mpi_bcast( vct, len(vct), mpi.MPI_FLOAT, 0, mpi.MPI_COMM_WORLD ) from sp_utilities import bcast_list_to_all tetprj = bcast_list_to_all(tetprj, myid, 0) #print " reshape ",myid vct = reshape(vct,(nprj,3)) assignments = [[] for i in range(nrefa)] dspn = 1.25*delta for k in range(nprj): best_s = -1.0 best_i = -1 for i in range( nrefa ): if(abs(tetprj[k] - tetref[i]) <= dspn): s = abs(refnormal[i][0]*vct[k][0] + refnormal[i][1]*vct[k][1] + refnormal[i][2]*vct[k][2]) if s > best_s: best_s = s best_i = i assignments[best_i].append(k) am = len(assignments[0]) mufur = 1.0/am for i in range(1,len(assignments)): ti = len(assignments[i]) am = min(am, ti) if(ti>0): mufur += 1.0/ti del tetprj,tetref dp = 1.0 - d # keep that many in each direction keep = int(am*dp +0.5) mufur = keep*nrefa/(1.0 - mufur*keep/float(nrefa)) if myid == 0: sxprint(" Number of projections ",nprj,". Number of reference directions ",nrefa,", multiplicative factor for the variance ",mufur) sxprint(" Minimum number of assignments ",am," Number of projections used per stratum ", keep," Number of projections in resampled structure ",int(am*dp +0.5)*nrefa) if am <2 or am == keep: sxprint("incorrect settings") exit() # FIX if(seedbase < 1): seed() jumpahead(17*myid+123) else: seed(seedbase) jumpahead(17*myid+123) volfile = os.path.join(outdir, "bsvol%04d.hdf" % myid) from random import randint niter = nvol/ncpu/nbufvol for kiter in range(niter): if(verbose == 1): finfo.write( "Iteration %d: \n" % kiter ) finfo.flush() iter_start = time() # the following has to be converted to resample mults=1 means take given projection., mults=0 means omit mults = [ [0]*nprj for i in range(nbufvol) ] for i in range(nbufvol): for l in range(nrefa): mass = assignments[l][:] shuffle(mass) mass = mass[:keep] mass.sort() #print l, " * ",mass for k in range(keep): mults[i][mass[k]] = 1 ''' lout = [] for l in xrange(len(mults[i])): if mults[i][l] == 1: lout.append(l) write_text_file(lout, os.path.join(outdir, "list%04d_%03d.txt" %(i, myid))) del lout ''' del mass rectors, fftvols, wgtvols = resample_prepare( prjfile, nbufvol, snr, CTF, npad ) resample_insert( bufprefix, fftvols, wgtvols, mults, CTF, npad, finfo ) del mults resample_finish( rectors, fftvols, wgtvols, volfile, kiter, nprj, finfo ) rectors = None fftvols = None wgtvols = None if(verbose == 1): finfo.write( "time for iteration: %10.3f\n" % (time() - iter_start) ) finfo.flush()
if not options.outdir: outdir = os.path.dirname(os.path.realpath(options.classavgs)) else: outdir = options.outdir if options.mode == 'viper': selectdoc = options.classselect elif options.mode == 'projmatch': selectdoc = None elif options.mode == 'meridien': selectdoc = options.partselect else: sp_global_def.ERROR( "\nERROR!! Valid mode not specified. Valid modes are: viper, projmatch, and meridien.", __file__, 1) sxprint('Type %s --help to see available options\n' % os.path.basename(__file__)) exit() main_proj_compare(options.classavgs, options.vol3d, outdir, options, mode=options.mode, prjmethod=options.prjmethod, classangles=options.classangles, partangles=options.partangles, selectdoc=selectdoc, verbose=options.verbose, displayYN=options.display) sp_global_def.print_timestamp("Finish")
def main(args): """ Main function contains the logic behind the program. Arguments: args - Command line argument options Returns: None """ sp_global_def.BATCH = True output_file = sanity_checks(args) output_dtype = [] array_list = [] create_stack = False particle_data = None partres_data = None params_2d_data = None params_3d_data = None params_3d_subset_data = None if args.particle_stack: sxprint("Import particle stack") particle_data, create_stack = import_particle_stack( args.particle_stack, args.output_directory, args.relion_project_dir) output_dtype.extend(particle_data.dtype.descr) if args.partres_file: sxprint("Import partres file") partres_data = import_partres_file(args.partres_file) output_dtype.extend(partres_data.dtype.descr) if args.params_2d_file: sxprint("Import params 2d file") params_2d_data = import_params(args.params_2d_file, dim="2d") output_dtype.extend(params_2d_data.dtype.descr) if args.params_3d_file: sxprint("Import params 3d file") params_3d_data = import_params(args.params_3d_file, dim="3d") output_dtype.extend(params_3d_data.dtype.descr) if args.params_3d_chunk_file_0 != None and args.params_3d_chunk_file_1 != None: sxprint("Import params 3d chunk files") params_3d_subset_data = np.empty(params_3d_data.shape[0], dtype=[("_rlnRandomSubset", "<i8")]) params_3d_subset_data.fill(np.nan) params_import = [] for idx, file_name in enumerate( [args.params_3d_chunk_file_0, args.params_3d_chunk_file_1]): chunk_import = np.genfromtxt(file_name, int) params_3d_subset_data["_rlnRandomSubset"][chunk_import] = idx + 1 params_import.extend(chunk_import.tolist()) output_dtype.extend(params_3d_subset_data.dtype.descr) assert params_3d_subset_data.shape[0] == params_3d_data.shape[0] assert np.unique(params_import).shape[0] == params_3d_data.shape[0] if args.params_3d_index_file: sxprint("Import params 3d index") params_index_data = np.genfromtxt(args.params_3d_index_file, dtype=int) assert params_3d_data.shape[0] == params_index_data.shape[0] assert np.unique(params_index_data).shape[0] == params_3d_data.shape[0] elif args.particle_stack: params_index_data = np.arange(particle_data.shape[0]) else: params_index_data = np.arange(partres_data.shape[0]) mask_array = np.ones(particle_data.shape[0], dtype=np.bool) if args.list or args.exlist: sxprint("Import list/exlist information") mask_array = create_particle_data_mask(args.list, args.exlist, particle_data.shape[0]) output_data = np.empty(params_index_data.shape[0], dtype=sorted(list(set(output_dtype)))) particle_data_params = particle_data[params_index_data] mask_array_params = mask_array[params_index_data] array_list.append(particle_data_params) array_list.append(params_2d_data) array_list.append(params_3d_data) array_list.append(params_3d_subset_data) sxprint("Adjust header") for array in array_list: if array is not None: for name in array.dtype.names: output_data[name] = array[name] if partres_data is not None: for row in partres_data: mask = output_data["_rlnMicrographName"] == row[ "_rlnMicrographName"] for name in partres_data.dtype.names: output_data[name][mask] = row[name] final_output = output_data[mask_array_params] sxprint("Write star file") header = ["", "data_", "", "loop_"] header.extend([ "{0} #{1}".format(name, idx + 1) for idx, name in enumerate(final_output.dtype.names) ]) dtype_dict = final_output.dtype.fields fmt = [] for name in final_output.dtype.names: max_length = (len( max([str(entry).split(".")[0] for entry in final_output[name]], key=len)) + 2) if "float" in str(dtype_dict[name][0]): fmt.append("%{0}.6f".format(max_length + 7)) elif "int" in str(dtype_dict[name][0]): fmt.append("%{0}d".format(max_length)) elif "|S" in str(dtype_dict[name][0]): fmt.append("%{0}s".format(max_length)) else: assert False np.savetxt( output_file, final_output, fmt=" ".join(fmt), header="\n".join(header), comments="", ) if create_stack: sxprint("Create particle stacks") create_particle_stack(args.particle_stack, args.output_directory, particle_data) sxprint("Done!") sp_global_def.BATCH = False
def import_params(params_file, dim): """ Import 2D or 3D parameters. The 2d params file has the format: angle, shift x, shift y, mirror The 3d params file has the format: angle_phi, angle_theta, angle_psi, tx, ty Arguments: params_2d_file - File name of the 2d parameter files Returns: parameter array """ if dim == "2d": dtype_import_list = [ ("angle_psi", float), ("shift_x", float), ("shift_y", float), ("mirror", int), ] sxprint("What happens with mirror?") elif dim == "3d": dtype_import_list = [ ("angle_rot", float), ("angle_theta", float), ("angle_psi", float), ("shift_x", float), ("shift_y", float), ] else: sp_global_def.ERROR( "Dimension {0} not supported. Only '2d' and '3d' are supported.". format(dim), "sp_sphire2relion", ) input_data = np.genfromtxt(params_file, dtype=dtype_import_list) dtype_output_list = [ ("_rlnOriginX", float), ("_rlnOriginY", float), ("_rlnAngleRot", float), ("_rlnAngleTilt", float), ("_rlnAnglePsi", float), ] params_array = np.empty(len(input_data), dtype=sorted(dtype_output_list)) params_array["_rlnOriginX"] = input_data["shift_x"] params_array["_rlnOriginY"] = input_data["shift_y"] params_array["_rlnAnglePsi"] = input_data["angle_psi"] if dim == "2d": params_array["_rlnAngleTilt"] = 0 params_array["_rlnAngleRot"] = 0 elif dim == "3d": params_array["_rlnAngleTilt"] = input_data["angle_theta"] params_array["_rlnAngleRot"] = input_data["angle_rot"] return params_array
def ctf_fit(im_1d, bg_1d, im_2d, bg_2d, voltage, cs, ac, apix, bgadj=0, autohp=False): """Determines CTF parameters given power spectra produced by powspec_with_bg() The bgadj option will result in adjusting the bg_1d curve to better match the zeroes of the CTF (in which case bg_1d is modified in place).""" # defocus estimation global debug ys = im_2d.get_ysize() ds = 1.0 / (apix * ys) ctf = EMAN2Ctf() ctf.from_dict({ "defocus": 1.0, "voltage": voltage, "bfactor": 0.0, "cs": cs, "ampcont": ac, "apix": apix, "dsbg": ds, "background": bg_1d }) sf = sfact([i * ds for i in range(ys)], "ribosome", "nono") #sxprint " SF ",sf if debug: dfout = open("ctf.df.txt", "w") dfbest1 = (0, -1.0e20) for dfi in range(5, 128): # loop over defocus ac = 10 df = dfi / 20.0 ctf.defocus = df ctf.ampcont = ac cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP) st = .04 / ds norm = 0 for fz in range(len(cc)): if cc[fz] < 0: break tot, totr = 0, 0 for s in range(int(st), ys / 2): tot += (cc[s]**2) * (im_1d[s] - bg_1d[s]) totr += cc[s]**4 #for s in range(int(ys/2)): tot+=(cc[s*ctf.CTFOS]**2)*ps1d[-1][s]/norm #for s in range(int(fz/ctf.CTFOS),ys/2): tot+=(cc[s*ctf.CTFOS]**2)*ps1d[-1][s] #for s in range(int(fz/ctf.CTFOS),ys/2): tot+=(cc[s*ctf.CTFOS]**2)*snr[s] tot /= sqrt(totr) #tot/=totr if tot > dfbest1[1]: dfbest1 = (df, tot) try: dfout.write("%1.2f\t%g\n" % (df, tot)) except: pass #out=file("bg1d2.txt","w") #for a,b in enumerate(bg2): out.write("%1.4f\t%1.5f\n"%(a*ds,b)) #out.close() dfbest = dfbest1 for dfi in range(-10, 10): # loop over defocus df = dfi / 100.0 + dfbest1[0] ctf.defocus = df cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP) st = .04 / ds norm = 0 for fz in range(len(cc)): #norm+=cc[fz]**2 if cc[fz] < 0: break tot, totr = 0, 0 for s in range(int(st), ys / 2): tot += (cc[s]**2) * (im_1d[s] - bg_1d[s]) totr += cc[s]**4 tot /= sqrt(totr) if tot > dfbest[1]: dfbest = (df, tot) if debug: dfout.write("%1.2f\t%g\n" % (df, tot)) ctf.defocus = dfbest[0] cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP) Util.save_data(0, ds, cc, "ctf.ctf.txt") if bgadj: # now we try to construct a better background based on the CTF zeroes being zero bg2 = bg_1d[:] last = 0, 1.0 for x in range(1, len(bg2) - 1): if cc[x] * cc[x + 1] < 0: # we search +-1 point from the zero for the minimum cur = (x, min(im_1d[x] / bg_1d[x], im_1d[x - 1] / bg_1d[x - 1], im_1d[x + 1] / bg_1d[x + 1])) # once we have a pair of zeros we adjust the background values between for xx in range(last[0], cur[0]): w = (xx - last[0]) / float(cur[0] - last[0]) bg_1d[xx] = bg2[xx] * (cur[1] * w + last[1] * (1.0 - w)) # sxprint xx,"\t",(cur[1]*w+last[1]*(1.0-w)) #,"\t",cur[1],last[1] last = cur # cover the area from the last zero crossing to the end of the curve for xx in range(last[0], len(bg2)): bg_1d[xx] = bg2[xx] * last[1] snr = [snr_safe(im_1d[i], bg_1d[i]) for i in range(len(im_1d))] # This will dramatically reduce the intensity of the initial sharp peak found in almost all single particle data # this applies to the SNR curve only, downweighting the importance of this section of the spectrum without actually # removing the information by filtering the image data. It will, of course also impact Wiener filters. if autohp: for x in range(2, len(snr) - 2): if snr[x] > snr[x + 1] and snr[x + 1] < snr[x + 2]: break # we find the first minimum snr1max = max(snr[1:x]) # find the intensity of the first peak snr2max = max(snr[x + 2:len(snr) / 2]) # find the next highest snr peak qtmp = 0.5 * snr2max / snr1max for xx in range(1, x + 1): snr[xx] *= qtmp # scale the initial peak to 50% of the next highest peak # store the final results ctf.snr = snr ctf.defocus = dfbest[0] # This smooths the SNR curve #snr=ctf.compute_1d(ys,ds,Ctf.CtfType.CTF_SNR_SMOOTH) #snr=snr[:len(ctf.snr)] #ctf.snr=snr # Last, we try to get a decent B-factor # This is a quick hack and not very efficiently coded bfs = [ 0.0, 50.0, 100.0, 200.0, 400.0, 600.0, 800.0, 1200.0, 1800.0, 2500.0, 4000.0 ] best = (0, 0) s0 = int(.05 / ds) + 1 s1 = min(int(0.14 / ds), len(bg_1d) - 1) sxprint(" FREQ RANGE", s0, s1) for b in range(1, len(bfs) - 1): ctf.bfactor = bfs[b] cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP) sf = sfact([ds * i for i in range(len(cc))], "ribosome", "original") cc = [sf[i] * cc[i]**2 for i in range(len(cc))] # adjust the amplitude to match well a0, a1 = 0, 0 for s in range(s0, s1): a0 += cc[s] a1 += fabs(im_1d[s] - bg_1d[s]) if a1 == 0: a1 = 1.0 a0 /= a1 cc = [i / a0 for i in cc] er = 0 # compute the error for s in range(s0, len(bg_1d) - 1): er += (cc[s] - im_1d[s] + bg_1d[s])**2 if best[0] == 0 or er < best[0]: best = (er, b) # Stupid replication here, in a hurry bb = best[1] best = (best[0], bfs[best[1]]) for b in range(20): ctf.bfactor = bfs[bb - 1] * (1.0 - b / 20.0) + bfs[bb + 1] * (b / 20.0) cc = ctf.compute_1d(ys, ds, Ctf.CtfType.CTF_AMP) sf = sfact([i * ds for i in range(len(cc))], "ribosome", "original") cc = [sf[i] * cc[i]**2 for i in range(len(cc))] # adjust the amplitude to match well a0, a1 = 0, 0 for s in range(s0, s1): a0 += cc[s] a1 += fabs(im_1d[s] - bg_1d[s]) if a1 == 0: a1 = 1.0 a0 /= a1 cc = [i / a0 for i in cc] er = 0 # compute the error for s in range(s0, len(bg_1d) - 1): er += (cc[s] - im_1d[s] + bg_1d[s])**2 if best[0] == 0 or er < best[0]: best = (er, ctf.bfactor) sxprint(" BEST ", best) #for s in xrange(s0,len(bg_1d)-1): # sxprint s,cc[s],im_1d[s]-bg_1d[s],sfact(ds*s) #for i in xrange(3*len(cc)): # sxprint ds*i," ",sfact(ds*i) # sxprint bfs[b],best ctf.bfactor = best[1] if 1: sxprint("Best DF = %1.3f B-factor = %1.0f" % (dfbest[0], ctf.bfactor)) return ctf
def main(): global debug, logid progname = os.path.basename(sys.argv[0]) usage = """%prog [options] <input stack/image> ... Various CTF-related operations on images, including automatic fitting. Note that automatic fitting is limited to 5 microns underfocus at most. Input particles should be unmasked and unfiltered. A minimum of ~20% padding around the particles is required for background extraction, even if this brings the edge of another particle into the box in some cases. Particles should be reasonably well centered. Can also optionally phase flip and Wiener filter particles. Wiener filtration comes after phase-flipping, so if phase flipping is performed Wiener filtered particles will also be phase-flipped. Note that both operations are performed on oversampled images if specified (though final real-space images are clipped back to their original size. Increasing padding during the particle picking process will improve the accuracy of phase-flipping, particularly for images far from focus.""" parser = OptionParser(usage=usage, version=EMANVERSION) parser.add_option("--gui", action="store_true", help="Start the GUI for interactive fitting", default=False) parser.add_option("--auto_fit", action="store_true", help="Runs automated CTF fitting on the input images", default=False) parser.add_option( "--bgmask", type="int", help= "Compute the background power spectrum from the edge of the image, specify a mask radius in pixels which would largely mask out the particles. Default is boxsize/2.", default=0) parser.add_option("--apix", type="float", help="Angstroms per pixel for all images", default=0) parser.add_option("--voltage", type="float", help="Microscope voltage in KV", default=0) parser.add_option("--cs", type="float", help="Microscope Cs (spherical aberation)", default=0) parser.add_option("--ac", type="float", help="Amplitude contrast (percentage, default=10)", default=10) parser.add_option( "--autohp", action="store_true", help= "Automatic high pass filter of the SNR only to remove initial sharp peak, phase-flipped data is not directly affected (default false)", default=False) #parser.add_option("--invert",action="store_true",help="Invert the contrast of the particles in output files (default false)",default=False) parser.add_option("--nonorm", action="store_true", help="Suppress per image real-space normalization", default=False) parser.add_option( "--nosmooth", action="store_true", help= "Disable smoothing of the background (running-average of the log with adjustment at the zeroes of the CTF)", default=False) #parser.add_option("--phaseflip",action="store_true",help="Perform phase flipping after CTF determination and writes to specified file.",default=False) #parser.add_option("--wiener",action="store_true",help="Wiener filter (optionally phaseflipped) particles.",default=False) parser.add_option("--oversamp", type="int", help="Oversampling factor", default=1) parser.add_option( "--sf", type="string", help= "The name of a file containing a structure factor curve. Can improve B-factor determination.", default=None) parser.add_option("--debug", action="store_true", default=False) (options, args) = parser.parse_args() if len(args) < 1: parser.error("Input image required") if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() if options.auto_fit: if options.voltage == 0: parser.error("Please specify voltage") if options.cs == 0: parser.error("Please specify Cs") if options.apix == 0: sxprint("Using A/pix from header") debug = options.debug global sfcurve if options.sf: sfcurve = XYData() sfcurve.read_file(options.sf) logid = E2init(sys.argv) # if options.oversamp>1 : options.apix/=float(options.oversamp) db_project = db_open_dict("bdb:project") db_parms = db_open_dict("bdb:e2ctf.parms") db_misc = db_open_dict("bdb:e2ctf.misc") options.filenames = args ### Power spectrum and CTF fitting if options.auto_fit: img_sets = pspec_and_ctf_fit( options, debug) # converted to a function so to work with the workflow ### This computes the intensity of the background subtracted power spectrum at each CTF maximum for all sets global envelopes # needs to be a global for the Simplex minimizer # envelopes is essentially a cache of information that could be useful at later stages of refinement # as according to Steven Ludtke for i in img_sets: envelopes.append(ctf_env_points(i[2], i[3], i[1])) # we use a simplex minimizer to try to rescale the individual sets to match as best they can scales = [1.0] * len(img_sets) if (len(img_sets) > 3): incr = [0.2] * len(img_sets) simp = Simplex(env_cmp, scales, incr) scales = simp.minimize(maxiters=1000)[0] # sxprint scales sxprint(" ") # apply the final rescaling envelope = [] for i in range(len(scales)): cur = envelopes[i] for j in range(len(cur)): envelope.append((cur[j][0], cur[j][1] * scales[i])) envelope.sort() envelope = [i for i in envelope if i[1] > 0] # filter out all negative peak values db_misc = db_open_dict("bdb:e2ctf.misc") db_misc["envelope"] = envelope #db_close_dict("bdb:e2ctf.misc") #out=file("envelope.txt","w") #for i in envelope: out.write("%f\t%f\n"%(i[0],i[1])) #out.close() ### GUI - user can update CTF parameters interactively if options.gui: img_sets = get_gui_arg_img_sets(options.filenames) if len(img_sets) == 0: E2end(logid) ERROR("img_sets == 0") return app = EMApp() gui = GUIctf(app, img_sets) gui.show_guis() app.exec_() sxprint("done execution") ### Process input files #if debug : print "Phase flipping / Wiener filtration" # write wiener filtered and/or phase flipped particle data to the local database #if options.phaseflip or options.wiener: # only put this if statement here to make the program flow obvious # write_e2ctf_output(options) # converted to a function so to work with the workflow E2end(logid)
def powspec_with_bg(stackfile, radius=0, edgenorm=True, oversamp=1): """This routine will read the images from the specified file, optionally edgenormalize, then apply a gaussian mask with the specified radius then compute the average 2-D power spectrum for the stack. It will also compute the average 2-D power spectrum using 1-mask + edge apotization to get an appoximate 'background' power spectrum. 2-D results returned as a 2-D FFT intensity/0 image. 1-D results returned as a list of floats. returns a 4-tuple with spectra for (1d particle,1d background,2d particle,2d background) """ global masks im = EMData() im.read_image(stackfile, 0) ys = im.get_ysize() * oversamp ys2 = im.get_ysize() n = EMUtil.get_image_count(stackfile) if (radius == 0): radius = ys2 // 2 - ys2 // 4 # set up the inner and outer Gaussian masks try: mask1, ratio1, mask2, ratio2 = masks[(ys, radius)] except: mask1 = EMData(ys2, ys2, 1) mask1.to_one() #mask1.process_inplace("mask.gaussian",{"outer_radius":radius}) mask1.process_inplace("mask.gaussian", { "inner_radius": radius, "outer_radius": ys2 // 10 }) mask2 = mask1.copy() * -1 + 1 # mask1.process_inplace("mask.decayedge2d",{"width":4}) #mask2.process_inplace("mask.decayedge2d",{"width":4}) mask1.clip_inplace( Region(-(ys2 * (oversamp - 1) / 2), -(ys2 * (oversamp - 1) / 2), ys, ys)) mask2.clip_inplace( Region(-(ys2 * (oversamp - 1) / 2), -(ys2 * (oversamp - 1) / 2), ys, ys)) ratio1 = mask1.get_attr("square_sum") / (ys * ys) #/1.035 ratio2 = mask2.get_attr("square_sum") / (ys * ys) masks[(ys, radius)] = (mask1, ratio1, mask2, ratio2) sxprint(" RATIOS ", ratio1, ratio2, " ", radius) #mask1.write_image("mask1.hdf",0) #mask2.write_image("mask2.hdf",0) pav1 = model_blank(ys2, ys2) pva1 = model_blank(ys2, ys2) pav2 = model_blank(ys2, ys2) pva2 = model_blank(ys2, ys2) pf = float(ys2 * ys2)**2 / 4.0 fofo = [] for i in range(n): im1 = EMData() im1.read_image(stackfile, i) # im1=EMData(stackfile,i) #info(im1) #if edgenorm : im1.process_inplace("normalize.edgemean") if oversamp > 1: im1.clip_inplace( Region(-(ys2 * (oversamp - 1) / 2), -(ys2 * (oversamp - 1) / 2), ys, ys)) im2 = im1.copy() im1 *= mask1 pp = periodogram(im1) * pf / ratio1 fofo.append(rot_avg_table(pp)) Util.add_img(pav1, pp) Util.add_img2(pva1, pp) imf = im1.do_fft() #sxprint_col(imf,0) imf.ri2inten() #info(imf) imf.set_complex(0) #sxprint_col(imf,0) #imf.write_image("imf.hdf",0) #pop = periodogram(im1) #sxprint_col(pop,90) #pop.write_image("pop.hdf",0) if i == 0: av1 = imf else: av1 += imf im2 *= mask2 pp = periodogram(im2) * pf / ratio2 Util.add_img(pav2, pp) Util.add_img2(pva2, pp) imf = im2.do_fft() imf.ri2inten() if i == 0: av2 = imf else: av2 += imf av1 /= (float(n) * av1.get_ysize() * av1.get_ysize() * ratio1) av1.set_value_at(0, 0, 0.0) av1.set_complex(1) av1.set_attr("is_intensity", 1) av2 /= (float(n) * av2.get_ysize() * av2.get_ysize() * ratio2) av2.set_value_at(0, 0, 0.0) av2.set_complex(1) av2.set_attr("is_intensity", 1) av1_1d = av1.calc_radial_dist(av1.get_ysize() / 2, 0.0, 1.0, 1) av2_1d = av2.calc_radial_dist(av2.get_ysize() / 2, 0.0, 1.0, 1) pav1 /= n pva1 = square_root((pva1 - pav1 * pav1 * n) / (n - 1) / n) pav2 /= n pva2 = square_root((pva2 - pav2 * pav2 * n) / (n - 1) / n) pav1.write_image("av1.hdf", 0) pav2.write_image("av2.hdf", 0) pva1.write_image("va1.hdf", 0) pva2.write_image("va2.hdf", 0) write_text_file([ list(range(ys2 // 2 + 1)), rot_avg_table(pav1), rot_avg_table(pva1), rot_avg_table(pav2), rot_avg_table(pva2) ], "pwsa.txt") #write_text_file([range(ys2//2+1), fofo[0], fofo[1], fofo[2], fofo[3], fofo[4], fofo[5]],"ipwsa.txt") return (av1_1d, av2_1d, av1, av2)
def multi_align_stability( ali_params, mir_stab_thld=0.0, grp_err_thld=10000.0, err_thld=1.732, print_individual=False, d=64, ): def sqerr(a): n = len(a) avg = sum(a) sq = 0.0 for i in range(n): sq += a[i]**2 return (sq - avg * avg / n) / n # args - G, data - [T, d] def func(args, data, return_avg_pixel_error=True): ali_params = data[0] d = data[1] L = len(ali_params) N = len(ali_params[0]) / 4 args_list = [0.0] * (L * 3) for i in range(L * 3 - 3): args_list[i] = args[i] cosa = [0.0] * L sina = [0.0] * L for i in range(L): cosa[i] = numpy.cos(numpy.radians(args_list[i * 3])) sina[i] = numpy.sin(numpy.radians(args_list[i * 3])) sqr_pixel_error = [0.0] * N ave_params = [] for i in range(N): sum_cosa = 0.0 sum_sina = 0.0 sx = [0.0] * L sy = [0.0] * L for j in range(L): if int(ali_params[j][i * 4 + 3]) == 0: sum_cosa += numpy.cos( numpy.radians(args_list[j * 3] + ali_params[j][i * 4])) sum_sina += numpy.sin( numpy.radians(args_list[j * 3] + ali_params[j][i * 4])) sx[j] = (args_list[j * 3 + 1] + ali_params[j][i * 4 + 1] * cosa[j] + ali_params[j][i * 4 + 2] * sina[j]) sy[j] = (args_list[j * 3 + 2] - ali_params[j][i * 4 + 1] * sina[j] + ali_params[j][i * 4 + 2] * cosa[j]) else: sum_cosa += numpy.cos( numpy.radians(-args_list[j * 3] + ali_params[j][i * 4])) sum_sina += numpy.sin( numpy.radians(-args_list[j * 3] + ali_params[j][i * 4])) sx[j] = (-args_list[j * 3 + 1] + ali_params[j][i * 4 + 1] * cosa[j] - ali_params[j][i * 4 + 2] * sina[j]) sy[j] = (args_list[j * 3 + 2] + ali_params[j][i * 4 + 1] * sina[j] + ali_params[j][i * 4 + 2] * cosa[j]) sqrtP = numpy.sqrt(sum_cosa**2 + sum_sina**2) sqr_pixel_error[i] = max( 0.0, d * d / 4.0 * (1 - sqrtP / L) + sqerr(sx) + sqerr(sy)) # Get ave transform params H = EMAN2_cppwrap.Transform({"type": "2D"}) H.set_matrix([ sum_cosa / sqrtP, sum_sina / sqrtP, 0.0, sum(sx) / L, -sum_sina / sqrtP, sum_cosa / sqrtP, 0.0, sum(sy) / L, 0.0, 0.0, 1.0, 0.0, ]) dd = H.get_params("2D") # We are using here mirror of the LAST SET. H = EMAN2_cppwrap.Transform({ "type": "2D", "alpha": dd["alpha"], "tx": dd["tx"], "ty": dd["ty"], "mirror": int(ali_params[L - 1][i * 4 + 3]), "scale": 1.0, }) dd = H.get_params("2D") ave_params.append([dd["alpha"], dd["tx"], dd["ty"], dd["mirror"]]) # Warning: Whatever I return here is squared pixel error, this is for the easy expression of derivative # Don't forget to square root it after getting the value if return_avg_pixel_error: return sum(sqr_pixel_error) / N else: return sqr_pixel_error, ave_params """Multiline Comment0""" # I decided not to use scipy in order to reduce the dependency, I wrote the C++ code instead # from scipy import array, int32 # from scipy.optimize.lbfgsb import fmin_l_bfgs_b # Find out the subset which is mirror stable over all runs all_part = [] num_ali = len(ali_params) nima = len(ali_params[0]) / 4 for i in range(num_ali): mirror0 = [] mirror1 = [] for j in range(nima): if ali_params[i][j * 4 + 3] == 0: mirror0.append(j) else: mirror1.append(j) mirror0 = numpy.array(mirror0, "int32") mirror1 = numpy.array(mirror1, "int32") all_part.append([mirror0, mirror1]) match, stab_part, CT_s, CT_t, ST, st = sp_statistics.k_means_stab_bbenum( all_part, T=0, nguesses=1) mir_stab_part = stab_part[0] + stab_part[1] mir_stab_rate = len(mir_stab_part) / float(nima) if mir_stab_rate <= mir_stab_thld: return [], mir_stab_rate, -1.0 mir_stab_part.sort() del all_part, match, stab_part, CT_s, CT_t, ST, st # for i in xrange(len(mir_stab_part)): print i, mir_stab_part[i] nima2 = len(mir_stab_part) # print " mirror stable ",nima2 # Keep the alignment parameters of mirror stable particles ali_params_mir_stab = [[] for i in range(num_ali)] for i in range(num_ali): for j in mir_stab_part: ali_params_mir_stab[i].extend(ali_params[i][j * 4:j * 4 + 4]) # Find out the alignment parameters for each iteration against the last one args = [] for i in range(num_ali - 1): alpha, sx, sy, mirror = align_diff_params( ali_params_mir_stab[i], ali_params_mir_stab[num_ali - 1]) args.extend([alpha, sx, sy]) # Do an initial analysis, purge all outlier particles, whose pixel error are larger than three times the threshold data = [ali_params_mir_stab, d] pixel_error_before, ave_params = func(numpy.array(args), data, return_avg_pixel_error=False) # We have to return mir_stab_rate (see above), even if the group does not survive it and the pixel error before cleaning, # see below, and in ISAC print the user the overall statistics (histograms?) # so one can get on overall idea how good/bad data is. PAP 01/25/2015 # print " >>> ",sqrt(sum(pixel_error_before)/nima2) ali_params_cleaned = [[] for i in range(num_ali)] cleaned_part = [] for j in range(nima2): pixel_error_before[j] = max(0.0, pixel_error_before[j]) # prevent sqrt of 0 if numpy.sqrt(pixel_error_before[j]) > 3 * err_thld: pass # print " removed ",3*err_thld,j,sqrt(pixel_error_before[j]) else: cleaned_part.append(mir_stab_part[j]) for i in range(num_ali): ali_params_cleaned[i].extend( ali_params_mir_stab[i][j * 4:j * 4 + 4]) nima3 = len(cleaned_part) prever = numpy.sqrt(sum(pixel_error_before) / nima2) if nima3 <= 1: return [], mir_stab_rate, prever # print " cleaned part ",nima3 # Use LBFGSB to minimize the sum of pixel errors data = [ali_params_cleaned, d] # Use Python code # ps_lp, val, d = fmin_l_bfgs_b(func, array(args), args=[data], fprime=dfunc, bounds=None, m=10, factr=1e3, pgtol=1e-4, iprint=-1, maxfun=100) # Use C++ code ali_params_cleaned_list = [] for params in ali_params_cleaned: ali_params_cleaned_list.extend(params) results = EMAN2_cppwrap.Util.multi_align_error(args, ali_params_cleaned_list, d) ps_lp = results[:-1] # Negative val can happen in some rare cases, it should be due to rounding errors, # because all results show the val is about 1e-13. # print "Strange results" # print "args =", args # print "ali_params_cleaned_list =", ali_params_cleaned_list # print "results = ", results val = max(0.0, results[-1]) del ali_params_cleaned_list if numpy.sqrt(val) > grp_err_thld: return [], mir_stab_rate, numpy.sqrt(val) pixel_error_after, ave_params = func(ps_lp, data, return_avg_pixel_error=False) stable_set = [] val = 0.0 for i in range(nima): if i in cleaned_part: j = cleaned_part.index(i) err = numpy.sqrt(pixel_error_after[j]) if err < err_thld: stable_set.append([err, i, ave_params[j]]) val += err if print_individual: sp_global_def.sxprint( "Particle %4d : pixel error = %18.4f" % (i, err)) else: if print_individual: sp_global_def.sxprint( "Particle %4d : pixel error = %18.4f unstable" % (i, err)) else: if print_individual: sp_global_def.sxprint("Particle %4d : Mirror unstable" % i) # return average pixel error before pruning as it is more informative return stable_set, mir_stab_rate, prever # sqrt(val/len(stable_set))
def main_proj_compare(classavgstack, reconfile, outdir, options, mode='viper', prjmethod='trilinear', classangles=None, partangles=None, selectdoc=None, verbose=False, displayYN=False): """ Main function overseeing various projection-comparison modes. Arguments: classavgstack : Input image stack reconfile : Map of which to generate projections (an optionally perform alignment) outdir : Output directory mode : Mode, viper (pre-existing angles for each input image), projmatch (angles from internal projection-matching) verbose : (boolean) Whether to write additional information to screen options : (list) Command-line options, run 'sxproj_compare.py -h' for an exhaustive list classangles : Angles and shifts for each input class average partangles : Angles and shifts for each particle (mode meridien) selectdoc : Selection file for included images prjmethod : Interpolation method to use displayYN : (boolean) Whether to automatically open montage """ # Expand path for outputs refprojstack = os.path.join(outdir, 'refproj.hdf') refanglesdoc = os.path.join(outdir, 'refangles.txt') outaligndoc = os.path.join(outdir, 'docalign2d.txt') # If not an input, will create an output, in modes projmatch if classangles == None: classangles = os.path.join(outdir, 'docangles.txt') # You need either input angles (mode viper) or to calculate them on the fly (mode projmatch) if mode == 'viper': sp_global_def.ERROR( "\nERROR!! Input alignment parameters not specified.", __file__, 1) sxprint('Type %s --help to see available options\n' % os.path.basename(__file__)) exit() # Check if inputs exist check(classavgstack, verbose=verbose) check(reconfile, verbose=verbose) if verbose: sxprint('') # Check that dimensions of images and volume agree (maybe rescale volume) voldim = EMAN2.EMData(reconfile).get_xsize() imgdim = EMAN2.EMData(classavgstack, 0).get_xsize() if voldim != imgdim: sp_global_def.ERROR( "\nERROR!! Dimension of input volume doesn't match that of image stack: %s vs. %s" % (voldim, imgdim), __file__, 1) scale = float( imgdim ) / voldim # only approximate, since full-sized particle radius is arbitrary msg = 'The command to resize the volume will be of the form:\n' msg += 'e2proc3d.py %s resized_vol.hdf --scale=%1.5f --clip=%s,%s,%s\n' % ( reconfile, scale, imgdim, imgdim, imgdim) msg += 'Check the file in the ISAC directory named "README_shrink_ratio.txt" for confirmation.\n' sxprint(msg) exit() # Here if you want to be fancy, there should be an option to chose the projection method, # the mechanism can be copied from sxproject3d.py PAP if prjmethod == 'trilinear': method_num = 1 elif prjmethod == 'gridding': method_num = -1 elif prjmethod == 'nn': method_num = 0 else: sp_global_def.ERROR( "\nERROR!! Valid projection methods are: trilinear (default), gridding, and nn (nearest neighbor).", __file__, 1) sxprint('Usage:\n%s' % USAGE) exit() # Set output directory and log file name log, verbose = prepare_outdir_log(outdir, verbose) # In case class averages include discarded images, apply selection file if mode == 'viper': if selectdoc: goodavgs, extension = os.path.splitext( os.path.basename(classavgstack)) newclasses = os.path.join(outdir, goodavgs + "_kept" + extension) # e2proc2d appends to existing files, so rename existing output if os.path.exists(newclasses): renamefile = newclasses + '.bak' print_log_msg( "Selected-classes stack %s exists, renaming to %s" % (newclasses, renamefile), log, verbose) print_log_msg("mv %s %s\n" % (newclasses, renamefile), log, verbose) os.rename(newclasses, renamefile) print_log_msg( 'Creating subset of %s to %s based on selection list %s' % (classavgstack, newclasses, selectdoc), log, verbose) cmd = "e2proc2d.py %s %s --list=%s" % (classavgstack, newclasses, selectdoc) print_log_msg(cmd, log, verbose) os.system(cmd) sxprint('') # Update class-averages classavgstack = newclasses # align de novo to reference map if mode == 'projmatch': # Generate reference projections print_log_msg( 'Projecting %s to output %s using an increment of %s degrees using %s symmetry' % (reconfile, refprojstack, options.delta, options.symmetry), log, verbose) cmd = 'sxproject3d.py %s %s --delta=%s --method=S --phiEqpsi=Minus --symmetry=%s' % ( reconfile, refprojstack, options.delta, options.symmetry) if options.prjmethod == 'trilinear': cmd += ' --trilinear' cmd += '\n' print_log_msg(cmd, log, verbose) project3d(reconfile, refprojstack, delta=options.delta, symmetry=options.symmetry) # Export projection angles print_log_msg( "Exporting projection angles from %s to %s" % (refprojstack, refanglesdoc), log, verbose) cmd = "sp_header.py %s --params=xform.projection --import=%s\n" % ( refprojstack, refanglesdoc) print_log_msg(cmd, log, verbose) header(refprojstack, 'xform.projection', fexport=refanglesdoc) # Perform multi-reference alignment if options.align == 'ali2d': projdir = os.path.join( outdir, 'Projdir') # used if input angles no provided if os.path.isdir(projdir): print_log_msg('Removing pre-existing directory %s' % projdir, log, verbose) print_log_msg('rm -r %s\n' % projdir, log, verbose) shutil.rmtree( projdir) # os.rmdir only removes empty directories # Zero out alignment parameters in header print_log_msg( 'Zeroing out alignment parameters in header of %s' % classavgstack, log, verbose) cmd = 'sxheader.py %s --params xform.align2d --zero\n' % classavgstack print_log_msg(cmd, log, verbose) header(classavgstack, 'xform.align2d', zero=True) # Perform multi-reference alignment msg = 'Aligning images in %s to projections %s with a radius of %s and a maximum allowed shift of %s' % ( classavgstack, refprojstack, options.matchrad, options.matchshift) print_log_msg(msg, log, verbose) cmd = 'sxmref_ali2d.py %s %s %s --ou=%s --xr=%s --yr=%s\n' % ( classavgstack, refprojstack, projdir, options.matchrad, options.matchshift, options.matchshift) print_log_msg(cmd, log, verbose) mref_ali2d(classavgstack, refprojstack, projdir, ou=options.matchrad, xrng=options.matchshift, yrng=options.matchshift) # Export alignment parameters print_log_msg( 'Exporting angles from %s into %s' % (classavgstack, classangles), log, verbose) cmd = "sp_header.py %s --params=xform.align2d --export=%s\n" % ( classavgstack, classangles) print_log_msg(cmd, log, verbose) header(classavgstack, 'xform.align2d', fexport=classangles) # By default, use AP SH else: apsh(refprojstack, classavgstack, outangles=classangles, refanglesdoc=refanglesdoc, outaligndoc=outaligndoc, outerradius=options.matchrad, maxshift=options.matchshift, ringstep=options.matchstep, log=log, verbose=verbose) # Diagnostic alignlist = read_text_row( classangles) # contain 2D alignment parameters nimg1 = EMAN2.EMUtil.get_image_count(classavgstack) assert len(alignlist) == nimg1, "MRK_DEBUG" # Get alignment parameters from MERIDIEN if mode == 'meridien': continueTF = True # Will proceed unless some information is missing if not partangles: sp_global_def.ERROR( "\nERROR!! Input alignment parameters not provided.", __file__, 1) continueTF = False if not continueTF: sxprint('Type %s --help to see available options\n' % os.path.basename(__file__)) exit() if not options.classdocs or options.outliers: classdir = os.path.join(outdir, 'Byclass') if not os.path.isdir(classdir): os.makedirs(classdir) if options.outliers: goodclassparttemplate = os.path.join( classdir, 'goodpartsclass{0:03d}.txt') else: goodclassparttemplate = None if not options.classdocs: classmap = os.path.join(classdir, 'classmap.txt') classdoc = os.path.join(classdir, 'docclass{0:03d}.txt') options.classdocs = os.path.join(classdir, 'docclass*.txt') # Separate particles by class vomq(classavgstack, classmap, classdoc, log=log, verbose=verbose) mode_meridien(reconfile, classavgstack, options.classdocs, partangles, selectdoc, options.refineshift, options.refinerad, classangles, outaligndoc, interpolation_method=method_num, outliers=options.outliers, goodclassparttemplate=goodclassparttemplate, alignopt=options.align, ringstep=options.refinestep, log=log, verbose=verbose) # Import Euler angles print_log_msg( "Importing parameter information into %s from %s" % (classavgstack, classangles), log, verbose) cmd = "sp_header.py %s --params=xform.projection --import=%s\n" % ( classavgstack, classangles) print_log_msg(cmd, log, verbose) header(classavgstack, 'xform.projection', fimport=classangles) # Make comparison stack between class averages (images 0,2,4,...) and re-projections (images 1,3,5,...) compstack = compare_projs(reconfile, classavgstack, classangles, outdir, interpolation_method=method_num, log=log, verbose=verbose) # Optionally pop up e2display if displayYN: sxprint('Opening montage') cmd = "e2display.py %s\n" % compstack sxprint(cmd) os.system(cmd) sxprint("Done!")
nsigma=s_nsigma, threshold=s_density_threshold, ndilation=command_args.s_ndilation, nerosion=command_args.s_nerosion, edge_width=command_args.s_edge_width, allow_disconnected=command_args.s_allow_disconnected, mode=mode, do_approx=command_args.s_do_old, do_fill=command_args.s_fill_mask, do_print=True, ) if command_args.s_invert: s_mask = 1 - s_mask sp_global_def.sxprint("Write outputs.") mask_first.write_image(output_prefix + "_mask_first.hdf") s_mask.write_image(output_prefix + "_mask_second.hdf") masked_combined = mask_first * s_mask masked_combined.write_image(output_prefix + "_mask.hdf") else: sp_global_def.sxprint("Write outputs.") mask_first.write_image(output_prefix + "_mask.hdf") if __name__ == "__main__": sp_global_def.print_timestamp("Start") sp_global_def.BATCH = True main() sp_global_def.BATCH = False sp_global_def.sxprint("Done!") sp_global_def.print_timestamp("Finish")
def run_unblur( unblur_path, input_image, input_dir, output_dir, corrected_path, uncorrected_path, shift_path, frc_path, temp_path, log_path, file_list, options, ): # Lists to write the text files later micrograph_list = [] shift_list = [] if options.save_frames: frames_list = [] # If micrograph list is provided just process the images in the list mic_list = options.selection_list if mic_list: # Import list file try: set_selection = numpy.genfromtxt(mic_list, dtype=None) except TypeError: sp_global_def.ERROR("no entrys in list file {0}".format(mic_list)) # List of files which are in pattern and list file_list = [ entry for entry in file_list if entry[len(input_dir):] in set_selection and os.path.exists(entry) ] # If no match is there abort if len(file_list) == 0: sp_global_def.ERROR( "no files in {0} matched the file pattern:\n".format(mic_list), 1) # Get the number of files nr_files = len(file_list) # Timeing stuff time_start = time.time() time_list = [] # Loop over all files for index, inputfile in enumerate(sorted(file_list)): # Check, if there is an prefix and suffix. # If there is more then one entry: the suffix is the last one. # Otherwhise its just the one after the dot. input_suffix = inputfile.split("/")[-1].split(".")[-1] # First output to introduce the programm if index == 0: sp_global_def.sxprint( "Progress: 0.0%; Time: --h:--m:--s/--h:--m:--s; Unblur started!" ) # Time begin t1 = time.time() # Get the output names file_name = inputfile[len(input_dir):-len(input_suffix) - 1] if options.skip_dose_filter: micrograph_name = "{0}/{1}{2}.mrc".format(uncorrected_path, file_name, options.sum_suffix) frames_name = "{0}/{1}{2}.mrc".format(uncorrected_path, file_name, options.frames_suffix) frc_name = "{0}/{1}{2}.txt".format(frc_path, file_name, options.frc_suffix) else: micrograph_name = "{0}/{1}{2}.mrc".format(corrected_path, file_name, options.sum_suffix) frames_name = "{0}/{1}{2}.mrc".format(corrected_path, file_name, options.frames_suffix) micrograph_name_skip = "{0}/{1}{2}.mrc".format( uncorrected_path, file_name, options.sum_suffix) frames_name_skip = "{0}/{1}{2}.mrc".format(uncorrected_path, file_name, options.frames_suffix) frc_name = "{0}/{1}{2}.txt".format(frc_path, file_name, options.frc_suffix) frc_summovie_name = "{0}/{1}_summovie{2}.txt".format( frc_path, file_name, options.frc_suffix) shift_name = "{0}/{1}{2}.txt".format(shift_path, file_name, options.shift_suffix) if not options.unblur_ready: temp_name = "{0}/{1}{2}.mrc".format(temp_path, file_name, options.sum_suffix) else: temp_name = inputfile log_name = "{0}/{1}.log".format(log_path, file_name) error_name = "{0}/{1}.err".format(log_path, file_name) # Append the names to the lists micrograph_list.append("{0}{1}.mrc".format(file_name, options.sum_suffix)) shift_list.append(shift_name) if options.save_frames: frames_list.append("{0}{1}.mrc".format(file_name, options.frames_suffix)) # First build the unblur/summovie command if not options.skip_dose_filter: unblur_command = create_unblur_command(temp_name, micrograph_name, shift_name, frames_name, frc_name, options) # Options for the summovie command options_summovie = { "first": 1, "last": -1, "nr_frames": options.nr_frames, "pixel_size": options.pixel_size, "exposure_per_frame": options.exposure_per_frame, "voltage": options.voltage, "pre_exposure": options.pre_exposure, "dont_restore_noise": options.dont_restore_noise, "apply_dose_filter": False, } summovie_command = sp_utilities.create_summovie_command( temp_name, micrograph_name_skip, shift_name, frc_summovie_name, options_summovie, ) else: unblur_command = create_unblur_command(temp_name, micrograph_name, shift_name, frc_name, frames_name, options) # Export the number of threads export_threads_command = [] # Export export_threads_command.append("export") # Nr of threads export_threads_command.append("OMP_NUM_THREADS={0}".format( options.nr_threads)) if not options.unblur_ready: # Do a e2proc3d.py e2proc3d_command = [] # e2proc3d e2proc3d_command.append("e2proc3d.py") # inputfile e2proc3d_command.append("{0}".format(inputfile)) # outputfile e2proc3d_command.append("{0}".format(temp_name)) # Translate the command to single strings if not options.unblur_ready: e2proc3d_command = r" ".join(e2proc3d_command) export_threads_command = r" ".join(export_threads_command) unblur_command = "\n".join(unblur_command) if not options.skip_dose_filter: summovie_command = "\n".join(summovie_command) # Build full command if not options.unblur_ready: if not options.skip_dose_filter: full_command = r'{0}; {1}; echo "{2}" | {3}'.format( export_threads_command, e2proc3d_command, unblur_command, unblur_path, ) full_command_summovie = r'{0}; echo "{1}" | {2}'.format( export_threads_command, summovie_command, options.summovie_path) else: full_command = r'{0}; {1}; echo "{2}" | {3}'.format( export_threads_command, e2proc3d_command, unblur_command, unblur_path, ) else: if not options.skip_dose_filter: full_command = r'{0}; echo "{1}" | {2}'.format( export_threads_command, unblur_command, unblur_path) full_command_summovie = r'{0}; echo "{1}" | {2}'.format( export_threads_command, summovie_command, options.summovie_path) else: full_command = r'{0}; echo "{1}" | {2}'.format( export_threads_command, unblur_command, unblur_path) # Remove temp unblur files temp_unblur_files = glob.glob(".UnBlur*") for entry in temp_unblur_files: os.remove(entry) # Remove temp summovie files temp_summovie_files = glob.glob(".SumMovie*") for entry in temp_summovie_files: os.remove(entry) with open(log_name, "w") as f: with open(error_name, "w") as e: # Execute Command if not options.skip_dose_filter: subprocess.Popen([full_command], shell=True, stdout=f, stderr=e).wait() # Remove temp unblur files temp_unblur_files = glob.glob(".UnBlur*") for entry in temp_unblur_files: os.remove(entry) # Remove temp summovie files temp_summovie_files = glob.glob(".SumMovie*") for entry in temp_summovie_files: os.remove(entry) subprocess.Popen([full_command_summovie], shell=True, stdout=f, stderr=e).wait() else: subprocess.Popen([full_command], shell=True, stdout=f, stderr=e).wait() # Remove temp unblur files temp_unblur_files = glob.glob(".UnBlur*") for entry in temp_unblur_files: os.remove(entry) # Remove temp summovie files temp_summovie_files = glob.glob(".SumMovie*") for entry in temp_summovie_files: os.remove(entry) if not options.unblur_ready: if os.path.exists(temp_name): # Remove temp file os.remove(temp_name) else: sp_global_def.sxprint( ("Error with file:\n{0}".format(inputfile))) # Check if SumMovie and UnBlur finished cleanly with open(log_name, "r") as r: clean_summovie = False clean_unblur = False for line in r: if "SumMovie finished cleanly." in line: clean_summovie = True if "UnBlur finished cleanly." in line: clean_unblur = True if clean_unblur: sp_global_def.sxprint("UnBlur finished cleanly.") else: sp_global_def.ERROR( "unblur error. check the logfile for more information: {0}". format(log_name), action=0, ) if clean_summovie: sp_global_def.sxprint("SumMovie finished cleanly.") else: sp_global_def.ERROR( "summovie error. check the logfile for more information: {0}". format(log_name), action=0, ) time_list.append(time.time() - t1) # Do progress output percent = round(100 * (index + 1) / float(nr_files), 2) estimated_time = nr_files * sum(time_list) / float(len(time_list)) estimated_time_h = estimated_time // 3600 estimated_time_m = (estimated_time - estimated_time_h * 3600) // 60 estimated_time_s = (estimated_time - estimated_time_h * 3600 - estimated_time_m * 60) current_time = time.time() - time_start current_time_h = current_time // 3600 current_time_m = (current_time - current_time_h * 3600) // 60 current_time_s = current_time - current_time_h * 3600 - current_time_m * 60 sp_global_def.sxprint(( "Progress: {0:.2f}%; Time: {1:.0f}h:{2:.0f}m:{3:.0f}s/{4:.0f}h:{5:.0f}m:{6:.0f}s; Micrograph done:{7}" .format( percent, current_time_h, current_time_m, current_time_s, estimated_time_h, estimated_time_m, estimated_time_s, file_name, ))) # Write micrograph and shift list with open("{0}/unblur_micrographs.txt".format(output_dir), "w") as f: for entry in sorted(micrograph_list): f.write("{0}\n".format(entry)) with open("{0}/unblur_shiftfiles.txt".format(output_dir), "w") as f: for entry in sorted(shift_list): f.write("{0}\n".format(entry)) if options.save_frames: with open("{0}/unblur_frames.txt".format(output_dir), "w") as f: for entry in sorted(frames_list): f.write("{0}\n".format(entry))
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " stack <maskfile> --search_rng=10 --maxit=max_iteration --CTF --snr=SNR --Fourvar=Fourier_variance --oneDx --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--search_rng", type="int", default=-1, help="Search range for x-shift") parser.add_option("--search_ang", type="int", default=-1, help="Search range for inplane rotation angle") parser.add_option( "--search_rng_y", type="int", default=-1, help= "Search range for x-shift. Not used for 1D search (oneDx flag set).") parser.add_option("--maxit", type="int", default=100, help="Maximum number of iterations program will perform") parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction") parser.add_option( "--snr", type="float", default=1.0, help="signal-to-noise ratio of the data (default is 1.0)") parser.add_option("--Fourvar", action="store_true", default=False, help="compute Fourier variance") parser.add_option("--oneDx", action="store_true", default=False, help="1D search along x-axis") parser.add_option("--MPI", action="store_true", default=False, help="use MPI") parser.add_option("--curvature", action="store_true", default=False, help="for curved filament alignment") (options, args) = parser.parse_args() if not (options.MPI): sxprint("Only MPI version is currently implemented.") sxprint("Please run '" + progname + " -h' for detailed options") return if len(args) < 1 or len(args) > 2: sxprint("usage: " + usage) sxprint("Please run '" + progname + " -h' for detailed options") else: if len(args) == 1: mask = None else: mask = args[1] if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True if options.oneDx: helicalshiftali_MPI(args[0], mask, options.maxit, options.CTF, options.snr, options.Fourvar, options.search_rng) else: shiftali_MPI(args[0], mask, options.maxit, options.CTF, options.snr, options.Fourvar, options.search_rng, options.oneDx, options.search_rng_y) sp_global_def.BATCH = False return
def main(): # Parse the Options progname = os.path.basename(sys.argv[0]) usage = (progname + """ unblur_path input_micrograph_pattern output_directory --summovie_path --selection_list --nr_frames=nr_frames --pixel_size=pixel_size --voltage=voltage --exposure_per_frame=exposure_per_frame --pre_exposure=pre_exposure --nr_threads --save_frames --skip_dose_filter --expert_mode --shift_initial=shift_initial --shift_radius=shift_radius --b_factor=b_factor --fourier_vertical=fourier_vertical --fourier_horizontal=fourier_horizontal --shift_threshold=shift_threshold --iterations=iterations --dont_restore_noise --verbose sxunblur exists only in non-MPI version. Perform unblur and with dose filtering and summovie without dose filtering. sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur --summovie_path=~/my_app/summovie --nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0 --voltage=300.0 --pre_exposure=0.0 --nr_threads=1 Perform unblur with dose filtering and summovie without dose filtering with selection list. sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur --summovie_path=~/my_app/summovie --selection_list=selected_micrograph_file --nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0 --voltage=300.0 --pre_exposure=0.0 --nr_threads=1 Perform unblur without dose filtering. sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur --nr_frames=25 --pixel_size=1.19 --skip_dose_filter --nr_threads=1 Perform unblur without dose filtering and save the frames. sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur --nr_frames=25 --pixel_size=1.19 --skip_dose_filter --save_frames --nr_threads=1 Perform unblur with dose filtering and summovie without dose filtering with all options. sxunblur.py ~/my_app/unblur 'movies/micrograph_*_frames.mrc' outdir_unblur --summovie_path=~/my_app/summovie --nr_frames=25 --pixel_size=1.19 --exposure_per_frame=1.0 --voltage=300.0 --pre_exposure=0.0 --save_frames --expert_mode --shift_initial=2.0 --shift_radius=200.0 --b_factor=1500.0 --fourier_vertical=1 --fourier_horizontal=1 --shift_threshold=0.1 --iterations=10 --verbose --nr_threads=1 """) parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION) parser.add_option( "--summovie_path", type="str", default="", help= "summovie executable path (SPHIRE specific): Specify the file path of summovie executable. (default none)", ) parser.add_option( "--selection_list", type="str", default="", help= "Micrograph selecting list (SPHIRE specific): Specify a name of micrograph selection list text file. The file extension must be '.txt'. If this is not provided, all files matched with the micrograph name pattern will be processed. (default none)", ) parser.add_option( "--nr_frames", type="int", default=3, help= "Number of movie frames: The number of movie frames in each input micrograph. (default 3)", ) parser.add_option("--sum_suffix", type="str", default="_sum", help=optparse.SUPPRESS_HELP) parser.add_option("--shift_suffix", type="str", default="_shift", help=optparse.SUPPRESS_HELP) parser.add_option( "--pixel_size", type="float", default=-1.0, help= "Pixel size [A]: The pixel size of input micrographs. (default required float)", ) parser.add_option( "--voltage", type="float", default=300.0, help= "Microscope voltage [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)", ) parser.add_option( "--exposure_per_frame", type="float", default=2.0, help= "Per frame exposure [e/A^2]: The electron dose per frame in e/A^2. (default 2.0)", ) parser.add_option( "--pre_exposure", type="float", default=0.0, help= "Pre-exposure [e/A^2]: The electron does in e/A^2 used for exposure prior to imaging .(default 0.0)", ) parser.add_option( "--nr_threads", type="int", default=1, help= "Number of threads: The number of threads unblur can use. The higher the faster, but it requires larger memory. (default 1)", ) parser.add_option( "--save_frames", action="store_true", default=False, help= "Save aligned movie frames: Save aligned movie frames. This option slows down the process. (default False)", ) parser.add_option("--frames_suffix", type="string", default="_frames", help=optparse.SUPPRESS_HELP) parser.add_option( "--skip_dose_filter", action="store_true", default=False, help= "Skip dose filter step: With this option, voltage, exposure per frame, and pre exposure will be ignored. (default False)", ) parser.add_option( "--expert_mode", action="store_true", default=False, help= "Use expert mode: Requires initial shift, shift radius, b-factor, fourier_vertical, fourier_horizontal, shift threshold, iterations, restore noise, and verbosity options. (default False)", ) parser.add_option("--frc_suffix", type="string", default="_frc", help=optparse.SUPPRESS_HELP) parser.add_option( "--shift_initial", type="float", default=2.0, help= "Minimum shift for initial search [A] (expert mode): Effective with unblur expert mode. (default 2.0)", ) parser.add_option( "--shift_radius", type="float", default=200.0, help= "Outer radius shift limit [A] (expert mode): Effective with unblur expert mode. (default 200.0)", ) parser.add_option( "--b_factor", type="float", default=1500.0, help= "Apply B-factor to images [A^2] (expert mode): Effective with unblur expert mode. (default 1500.0)", ) parser.add_option( "--fourier_vertical", type="int", default=1, help= "Vertical Fourier central mask size (expert mode): The half-width of central vertical line of Fourier mask. Effective with unblur expert mode. (default 1)", ) parser.add_option( "--fourier_horizontal", type="int", default=1, help= "Horizontal Fourier central mask size (expert mode): The half-width of central horizontal line of Fourier mask. Effective with unblur expert mode. (default 1)", ) parser.add_option( "--shift_threshold", type="float", default=0.1, help= "Termination shift threshold (expert mode): Effective with unblur expert mode. (default 0.1)", ) parser.add_option( "--iterations", type="int", default=10, help= "Maximum iterations (expert mode): Effective with unblur expert mode. (default 10)", ) parser.add_option( "--dont_restore_noise", action="store_true", default=False, help= "Do not restore noise power (expert mode): Effective with unblur expert mode. (default False)", ) parser.add_option( "--verbose", action="store_true", default=False, help= "Verbose (expert mode): Effective with unblur expert mode. (default False)", ) parser.add_option( "--unblur_ready", action="store_true", default=False, help=optparse.SUPPRESS_HELP, ) # list of the options and the arguments (options, args) = parser.parse_args(sys.argv[1:]) sp_global_def.BATCH = True # If there arent enough arguments, stop the script if len(args) != 3: sp_global_def.sxprint("Usage: " + usage) sp_global_def.ERROR( "Invalid number of parameters used. Please see usage information above." ) return # Convert the realtive parts to absolute ones unblur_path = os.path.realpath(args[0]) # unblur_path input_image = os.path.realpath(args[1]) # input_micrograph_pattern output_dir = os.path.realpath(args[2]) # output_directory # If the unblur executable file does not exists, stop the script if not os.path.exists(unblur_path): sp_global_def.ERROR( "Unblur directory does not exist, please change the name and restart the program" ) return # If the output directory exists, stop the script if os.path.exists(output_dir): sp_global_def.ERROR( "Output directory exists, please change the name and restart the program" ) return # If the input file does not exists, stop the script file_list = glob.glob(input_image) if not file_list: sp_global_def.ERROR( "Input file does not exist, please change the name and restart the program" ) return # If the skip_dose_filter option is false, the summovie path is necessary if not options.skip_dose_filter and not os.path.exists( options.summovie_path): sp_global_def.ERROR( "Path to the SumMovie executable is necessary when dose weighting is performed" ) return # Output paths corrected_path = "{:s}/corrsum_dose_filtered".format(output_dir) uncorrected_path = "{:s}/corrsum".format(output_dir) shift_path = "{:s}/shift".format(output_dir) frc_path = "{:s}/frc".format(output_dir) log_path = "{:s}/logfiles".format(output_dir) temp_path = "{0}/temp".format(output_dir) # Split the path of the image name at the "/" Characters. # The last entry contains the micrograph name. # Split the micrograph name at the wildcard character for the # prefix and suffix. input_split = input_image.split("/") input_name = input_split[-1].split("*") # Get the input directory if len(input_split) != 1: input_dir = input_image[:-len(input_split[-1])] else: input_dir = "" # Create output directorys if not os.path.exists(output_dir): os.makedirs(output_dir) sp_global_def.write_command(output_dir) if not os.path.exists(uncorrected_path): os.mkdir(uncorrected_path) if not os.path.exists(shift_path): os.mkdir(shift_path) if not os.path.exists(corrected_path): if not options.skip_dose_filter: os.mkdir(corrected_path) if not os.path.exists(frc_path): if options.expert_mode or not options.skip_dose_filter: os.mkdir(frc_path) if not os.path.exists(temp_path) and not options.unblur_ready: os.mkdir(temp_path) if not os.path.exists(log_path): os.mkdir(log_path) # Run unblur run_unblur( unblur_path=unblur_path, input_image=input_image, input_dir=input_dir, output_dir=output_dir, corrected_path=corrected_path, uncorrected_path=uncorrected_path, shift_path=shift_path, frc_path=frc_path, temp_path=temp_path, log_path=log_path, file_list=file_list, options=options, ) if not options.unblur_ready: # Remove temp folder for entry in glob.glob("{0}/*".format(temp_path)): os.remove(entry) os.rmdir(temp_path) sp_global_def.sxprint("All Done!") sp_global_def.BATCH = False
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " proj_stack output_averages --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--img_per_group", type="int", default=100, help="number of images per group") parser.add_option("--radius", type="int", default=-1, help="radius for alignment") parser.add_option( "--xr", type="string", default="2 1", help="range for translation search in x direction, search is +/xr") parser.add_option( "--yr", type="string", default="-1", help= "range for translation search in y direction, search is +/yr (default = same as xr)" ) parser.add_option( "--ts", type="string", default="1 0.5", help= "step size of the translation search in both directions, search is -xr, -xr+ts, 0, xr-ts, xr, can be fractional" ) parser.add_option( "--iter", type="int", default=30, help="number of iterations within alignment (default = 30)") parser.add_option( "--num_ali", type="int", default=5, help="number of alignments performed for stability (default = 5)") parser.add_option("--thld_err", type="float", default=1.0, help="threshold of pixel error (default = 1.732)") parser.add_option( "--grouping", type="string", default="GRP", help= "do grouping of projections: PPR - per projection, GRP - different size groups, exclusive (default), GEV - grouping equal size" ) parser.add_option( "--delta", type="float", default=-1.0, help="angular step for reference projections (required for GEV method)" ) parser.add_option( "--fl", type="float", default=0.3, help="cut-off frequency of hyperbolic tangent low-pass Fourier filter") parser.add_option( "--aa", type="float", default=0.2, help="fall-off of hyperbolic tangent low-pass Fourier filter") parser.add_option("--CTF", action="store_true", default=False, help="Consider CTF correction during the alignment ") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args() myid = mpi.mpi_comm_rank(MPI_COMM_WORLD) number_of_proc = mpi.mpi_comm_size(MPI_COMM_WORLD) main_node = 0 if len(args) == 2: stack = args[0] outdir = args[1] else: sp_global_def.ERROR("Incomplete list of arguments", "sxproj_stability.main", 1, myid=myid) return if not options.MPI: sp_global_def.ERROR("Non-MPI not supported!", "sxproj_stability.main", 1, myid=myid) return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True img_per_grp = options.img_per_group radius = options.radius ite = options.iter num_ali = options.num_ali thld_err = options.thld_err xrng = get_input_from_string(options.xr) if options.yr == "-1": yrng = xrng else: yrng = get_input_from_string(options.yr) step = get_input_from_string(options.ts) if myid == main_node: nima = EMUtil.get_image_count(stack) img = get_image(stack) nx = img.get_xsize() ny = img.get_ysize() else: nima = 0 nx = 0 ny = 0 nima = bcast_number_to_all(nima) nx = bcast_number_to_all(nx) ny = bcast_number_to_all(ny) if radius == -1: radius = nx / 2 - 2 mask = model_circle(radius, nx, nx) st = time() if options.grouping == "GRP": if myid == main_node: sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") proj_params = [] for i in range(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) # Here is where the grouping is done, I didn't put enough annotation in the group_proj_by_phitheta, # So I will briefly explain it here # proj_list : Returns a list of list of particle numbers, each list contains img_per_grp particle numbers # except for the last one. Depending on the number of particles left, they will either form a # group or append themselves to the last group # angle_list : Also returns a list of list, each list contains three numbers (phi, theta, delta), (phi, # theta) is the projection angle of the center of the group, delta is the range of this group # mirror_list: Also returns a list of list, each list contains img_per_grp True or False, which indicates # whether it should take mirror position. # In this program angle_list and mirror list are not of interest. proj_list_all, angle_list, mirror_list = group_proj_by_phitheta( proj_params, img_per_grp=img_per_grp) del proj_params sxprint(" B number of groups ", myid, " ", len(proj_list_all), time() - st) mpi_barrier(MPI_COMM_WORLD) # Number of groups, actually there could be one or two more groups, since the size of the remaining group varies # we will simply assign them to main node. n_grp = nima / img_per_grp - 1 # Divide proj_list_all equally to all nodes, and becomes proj_list proj_list = [] for i in range(n_grp): proc_to_stay = i % number_of_proc if proc_to_stay == main_node: if myid == main_node: proj_list.append(proj_list_all[i]) elif myid == main_node: mpi_send(len(proj_list_all[i]), 1, MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(proj_list_all[i], len(proj_list_all[i]), MPI_INT, proc_to_stay, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) elif myid == proc_to_stay: img_per_grp = mpi_recv(1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) img_per_grp = int(img_per_grp[0]) temp = mpi_recv(img_per_grp, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) proj_list.append(list(map(int, temp))) del temp mpi_barrier(MPI_COMM_WORLD) sxprint(" C ", myid, " ", time() - st) if myid == main_node: # Assign the remaining groups to main_node for i in range(n_grp, len(proj_list_all)): proj_list.append(proj_list_all[i]) del proj_list_all, angle_list, mirror_list # Compute stability per projection projection direction, equal number assigned, thus overlaps elif options.grouping == "GEV": if options.delta == -1.0: ERROR( "Angular step for reference projections is required for GEV method" ) return from sp_utilities import even_angles, nearestk_to_refdir, getvec refproj = even_angles(options.delta) img_begin, img_end = MPI_start_end(len(refproj), number_of_proc, myid) # Now each processor keeps its own share of reference projections refprojdir = refproj[img_begin:img_end] del refproj ref_ang = [0.0] * (len(refprojdir) * 2) for i in range(len(refprojdir)): ref_ang[i * 2] = refprojdir[0][0] ref_ang[i * 2 + 1] = refprojdir[0][1] + i * 0.1 sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") # the solution below is very slow, do not use it unless there is a problem with the i/O """ for i in xrange(number_of_proc): if myid == i: proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") mpi_barrier(MPI_COMM_WORLD) """ sxprint(" B ", myid, " ", time() - st) proj_ang = [0.0] * (nima * 2) for i in range(nima): dp = proj_attr[i].get_params("spider") proj_ang[i * 2] = dp["phi"] proj_ang[i * 2 + 1] = dp["theta"] sxprint(" C ", myid, " ", time() - st) asi = Util.nearestk_to_refdir(proj_ang, ref_ang, img_per_grp) del proj_ang, ref_ang proj_list = [] for i in range(len(refprojdir)): proj_list.append(asi[i * img_per_grp:(i + 1) * img_per_grp]) del asi sxprint(" D ", myid, " ", time() - st) #from sys import exit #exit() # Compute stability per projection elif options.grouping == "PPR": sxprint(" A ", myid, " ", time() - st) proj_attr = EMUtil.get_all_attributes(stack, "xform.projection") sxprint(" B ", myid, " ", time() - st) proj_params = [] for i in range(nima): dp = proj_attr[i].get_params("spider") phi, theta, psi, s2x, s2y = dp["phi"], dp["theta"], dp[ "psi"], -dp["tx"], -dp["ty"] proj_params.append([phi, theta, psi, s2x, s2y]) img_begin, img_end = MPI_start_end(nima, number_of_proc, myid) sxprint(" C ", myid, " ", time() - st) from sp_utilities import nearest_proj proj_list, mirror_list = nearest_proj( proj_params, img_per_grp, list(range(img_begin, img_begin + 1))) #range(img_begin, img_end)) refprojdir = proj_params[img_begin:img_end] del proj_params, mirror_list sxprint(" D ", myid, " ", time() - st) else: ERROR("Incorrect projection grouping option") return ########################################################################################################### # Begin stability test from sp_utilities import get_params_proj, read_text_file #if myid == 0: # from utilities import read_text_file # proj_list[0] = map(int, read_text_file("lggrpp0.txt")) from sp_utilities import model_blank aveList = [model_blank(nx, ny)] * len(proj_list) if options.grouping == "GRP": refprojdir = [[0.0, 0.0, -1.0]] * len(proj_list) for i in range(len(proj_list)): sxprint(" E ", myid, " ", time() - st) class_data = EMData.read_images(stack, proj_list[i]) #print " R ",myid," ",time()-st if options.CTF: from sp_filter import filt_ctf for im in range(len(class_data)): # MEM LEAK!! atemp = class_data[im].copy() btemp = filt_ctf(atemp, atemp.get_attr("ctf"), binary=1) class_data[im] = btemp #class_data[im] = filt_ctf(class_data[im], class_data[im].get_attr("ctf"), binary=1) for im in class_data: try: t = im.get_attr( "xform.align2d") # if they are there, no need to set them! except: try: t = im.get_attr("xform.projection") d = t.get_params("spider") set_params2D(im, [0.0, -d["tx"], -d["ty"], 0, 1.0]) except: set_params2D(im, [0.0, 0.0, 0.0, 0, 1.0]) #print " F ",myid," ",time()-st # Here, we perform realignment num_ali times all_ali_params = [] for j in range(num_ali): if (xrng[0] == 0.0 and yrng[0] == 0.0): avet = ali2d_ras(class_data, randomize=True, ir=1, ou=radius, rs=1, step=1.0, dst=90.0, maxit=ite, check_mirror=True, FH=options.fl, FF=options.aa) else: avet = within_group_refinement(class_data, mask, True, 1, radius, 1, xrng, yrng, step, 90.0, ite, options.fl, options.aa) ali_params = [] for im in range(len(class_data)): alpha, sx, sy, mirror, scale = get_params2D(class_data[im]) ali_params.extend([alpha, sx, sy, mirror]) all_ali_params.append(ali_params) #aveList[i] = avet #print " G ",myid," ",time()-st del ali_params # We determine the stability of this group here. # stable_set contains all particles deemed stable, it is a list of list # each list has two elements, the first is the pixel error, the second is the image number # stable_set is sorted based on pixel error #from utilities import write_text_file #write_text_file(all_ali_params, "all_ali_params%03d.txt"%myid) stable_set, mir_stab_rate, average_pix_err = multi_align_stability( all_ali_params, 0.0, 10000.0, thld_err, False, 2 * radius + 1) #print " H ",myid," ",time()-st if (len(stable_set) > 5): stable_set_id = [] members = [] pix_err = [] # First put the stable members into attr 'members' and 'pix_err' for s in stable_set: # s[1] - number in this subset stable_set_id.append(s[1]) # the original image number members.append(proj_list[i][s[1]]) pix_err.append(s[0]) # Then put the unstable members into attr 'members' and 'pix_err' from sp_fundamentals import rot_shift2D avet.to_zero() if options.grouping == "GRP": aphi = 0.0 atht = 0.0 vphi = 0.0 vtht = 0.0 l = -1 for j in range(len(proj_list[i])): # Here it will only work if stable_set_id is sorted in the increasing number, see how l progresses if j in stable_set_id: l += 1 avet += rot_shift2D(class_data[j], stable_set[l][2][0], stable_set[l][2][1], stable_set[l][2][2], stable_set[l][2][3]) if options.grouping == "GRP": phi, theta, psi, sxs, sy_s = get_params_proj( class_data[j]) if (theta > 90.0): phi = (phi + 540.0) % 360.0 theta = 180.0 - theta aphi += phi atht += theta vphi += phi * phi vtht += theta * theta else: members.append(proj_list[i][j]) pix_err.append(99999.99) aveList[i] = avet.copy() if l > 1: l += 1 aveList[i] /= l if options.grouping == "GRP": aphi /= l atht /= l vphi = (vphi - l * aphi * aphi) / l vtht = (vtht - l * atht * atht) / l from math import sqrt refprojdir[i] = [ aphi, atht, (sqrt(max(vphi, 0.0)) + sqrt(max(vtht, 0.0))) / 2.0 ] # Here more information has to be stored, PARTICULARLY WHAT IS THE REFERENCE DIRECTION aveList[i].set_attr('members', members) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', pix_err) else: sxprint(" empty group ", i, refprojdir[i]) aveList[i].set_attr('members', [-1]) aveList[i].set_attr('refprojdir', refprojdir[i]) aveList[i].set_attr('pixerr', [99999.]) del class_data if myid == main_node: km = 0 for i in range(number_of_proc): if i == main_node: for im in range(len(aveList)): aveList[im].write_image(args[1], km) km += 1 else: nl = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nl = int(nl[0]) for im in range(nl): ave = recv_EMData(i, im + i + 70000) nm = mpi_recv(1, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) nm = int(nm[0]) members = mpi_recv(nm, MPI_INT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('members', list(map(int, members))) members = mpi_recv(nm, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('pixerr', list(map(float, members))) members = mpi_recv(3, MPI_FLOAT, i, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) ave.set_attr('refprojdir', list(map(float, members))) ave.write_image(args[1], km) km += 1 else: mpi_send(len(aveList), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) for im in range(len(aveList)): send_EMData(aveList[im], main_node, im + myid + 70000) members = aveList[im].get_attr('members') mpi_send(len(members), 1, MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) mpi_send(members, len(members), MPI_INT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) members = aveList[im].get_attr('pixerr') mpi_send(members, len(members), MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) try: members = aveList[im].get_attr('refprojdir') mpi_send(members, 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) except: mpi_send([-999.0, -999.0, -999.0], 3, MPI_FLOAT, main_node, SPARX_MPI_TAG_UNIVERSAL, MPI_COMM_WORLD) sp_global_def.BATCH = False mpi_barrier(MPI_COMM_WORLD)
def generate_runscript(filename, seg_ny, ptcl_dst, fract): if ptcl_dst < 15: ERROR( "Distance in pixels between adjacent segments should be at least one rise!" ) return sxprint("Generating run script with the following parameters: \n") sxprint("y-dimension of segment used for refinement: %d" % seg_ny) sxprint("Distance in pixels between adjacent segments: %d" % ptcl_dst) sxprint("Fraction of structure used for applying helical symmetry: %.2f" % fract) if os.path.exists(filename): ERROR("File " + filename + " already exists. Please either remove or rename the file") return f = open(filename, 'w') f.write('#!/bin/csh\n') f.write('\n') f.write('set echo on\n') f.write('\n') f.write('#clean the previous outputs\n') f.write('rm *.hdf *.txt *.bck rm *.pdb\n') f.write('rm log*\n') #f.write('rm -rf outsymsearch \n') f.write('rm -rf mic result_* \n') f.write('rm -rf EMAN*\n') f.write('\n') f.write('# get the previous runned results\n') f.write('tar -zxvf answer.tar.gz\n') f.write('\n') f.write('#generate volume from pdb\n') f.write('tar -zxvf 3MFP_1SU.tar.gz\n') f.write('\n') f.write('# Helicise the Atom coordinates\n') f.write( '# Input: pdb file containing atom coordinates to helicise (3MFP_1SU.pdb)\n' ) f.write('# Output: pdb file containing helicised coordinates (rnew.pdb)\n') f.write( 'sxhelical_demo.py 3MFP_1SU.pdb rnew.pdb --heli --dp=27.6 --dphi=166.715\n' ) f.write('\n') f.write('#generate the density map\n') f.write('sxpdb2em.py rnew.pdb tmp.hdf --apix=1.84 --center=c \n') f.write('\n') f.write( '# Generate three micrographs, where each micrograph contains one projection of a long filament.\n' ) f.write( '# Input: Reference volume from which projections are calculated (tmp.hdf)\n' ) f.write( '# Output: Output directory in which micrographs will be written (mic)\n' ) f.write( 'sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84\n' ) f.write('\n') f.write('# generate initial volume for later helical refinement\n') f.write( '# Output: A noisy cylinder written to the output file name (ini.hdf).\n' ) f.write( 'sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35\n' ) f.write('\n') f.write( '# Estimate defocus value for each micrograph. This can be done either by GUI or pure command-line \n' ) f.write('# 1. To estimate defocus by GUI:\n') f.write('cd mic\n') f.write('sxhelixboxer.py mic0.hdf --gui &\n') f.write('\n') f.write( "# When the GUI starts up, there will be a checkbox at the bottom labelled 'CTF Estimation using CTER'. Check this box, and additional fields will appear in the GUI.\n" ) f.write( "# Under 'Parameters of CTF estimation,' enter 256 for 'Window size,' 2.0 for 'Cs, 10.0 for 'Amplitude Contrast,' 200 for 'Voltage', and 1.84 for Pixel size.\n" ) f.write( '# Click "Estimate CTF using CTER" button, and the estimated defocus will show up in the "Estimated defocus" box.\n' ) f.write( '# Now you can either estimate defocus for the remaining micrographs one by one using the GUI, or you can estimate defocus for the remaining micrographs in batch mode using \n' ) f.write('# the parameters you entered in the GUI for mic0.hdf. \n') f.write( '# Make sure to remove any output directories (pwrot and partres followed by underscore and \n' ) f.write( '# then name of the micrograph, e.g., pwrot_mic0 and partres_mic0) generated in the previous run of CTF estimation using CTER.\n' ) f.write('rm -r pwrot_mic0;rm -r partres_mic0\n') f.write('sxhelixboxer.py mic*.hdf --gui &\n') f.write('cd ..\n') f.write('\n') f.write('# 2. To estimate defocus by command-line\n') f.write('cd mic\n') f.write( 'sxhelixboxer.py pwrot partres --cter --indir=. --nameroot=mic --nx=200 --Cs=2.0 --voltage=200 --kboot=16 --apix=1.84 --ac=10.0 \n' ) f.write('cd ..\n') f.write('\n') f.write('cd mic\n') f.write('# have to open boxer\n') f.write( '# if want to save time, just close the window immediately and use saved coordinate\n' ) f.write('# otherwise draw the box carefully to get the coordinate \n') f.write('sxhelixboxer.py *.hdf --gui --helix-width=200 --ptcl-width=200\n') f.write('cd ..\n') f.write('\n') f.write('# if use saved coordinate, please use below commands\n') f.write('tar -zxvf saved_pos.tar.gz\n') f.write('cp saved_db.tar.gz mic/.\n') f.write('cp -r saved_pos/*box* mic/.\n') f.write('rm -rf saved_pos\n') f.write('cd mic\n') f.write('tar -zxvf saved_db.tar.gz\n') f.write('cd ..\n') f.write('\n') f.write('# Window segments from boxed helices\n') f.write( '# For more information on the windowing utility, see http://sparx-em.org/sparxwiki/windowallmic\n' ) f.write('# distance between segments is 1 rise exactly\n') f.write( 'sxhelixboxer.py houtdir --window --dirid=mic --micid=mic --micsuffix=hdf --dp=27.6 --apix=1.84 --boxsize=200 --outstacknameall=bdb:hadata --hcoords_suffix=_boxes.txt --ptcl-dst=%d --rmax=64\n' % ptcl_dst) f.write('\n') f.write('# Generate 2D mask for centering EM images\n') f.write( '# Output: 2D mask written to output file name provided (mask2d.hdf)\n' ) f.write( 'sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=70\n' ) f.write('\n') f.write('# center all the EM images\n') f.write( '# centering parameters will be saved in header, the input images will not be changed.\n' ) # horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1 # f.write('sxheader.py bdb:hadata --params=active --one\n') f.write('sxheader.py bdb:hadata --params=xform.align2d --zero\n') f.write( 'mpirun -np 2 sxshiftali.py bdb:hadata mask2d.hdf --oneDx --search_rng=10 --maxit=20 --CTF --MPI\n' ) f.write('\n') f.write( '# Apply the centering parameters stored in the header of each image in bdb:hadata to the image, \n' ) f.write( '# normalize using average and standard deviation outside the mask, and save the centered and normalized image to bdb:hdata\n' ) f.write( '# Input: Input stack containing 2D alignment parameters in the header (bdb:hadata)\n' ) f.write('# Name of 2D mask to use for normalization\n') f.write( '# Output: Stack of images (bdb:hdata) after applying 2D alignment parameters to the images in input stack. \n' ) f.write( 'sxhelical_demo.py bdb:hadata bdb:hdata mask2d.hdf --applyparams\n') f.write('\n') f.write('#Exhaustive search \n') f.write('sxheader.py bdb:hdata --params=xform.projection --zero\n') # horatio active_refactoring Jy51i1EwmLD4tWZ9_00000_1 # f.write('sxheader.py bdb:hdata --params=active --one\n') f.write( 'mpirun -np 3 sxhelicon.py bdb:hdata ini.hdf result_helicon --CTF --seg_ny=%d --fract=%.2f --psi_max=2.0 --delta=1.0 --maxit=7 --function=[.,nofunc,helical3c] --searchxshift=3.68 --xwobble=1.84 --ywobble=0 --dp=27.6 --dphi=166.715 --apix=1.84 --rmin=0.0 --rmax=64 --MPI\n' % (seg_ny, fract)) f.write('\n') f.write('#Local search \n') f.write( 'sxheader.py bdb:hdata --params=xform.projection --import=result_helicon/parameters0007.txt\n' ) f.write( 'mpirun -np 3 sxlocalhelicon.py bdb:hdata result_helicon/volf007.hdf result_local --CTF --seg_ny=%d --fract=%.2f --psi_max=2.0 --delta=1.0 --maxit=11 --function=[.,nofunc,helical3c] --boundaryavg --MA --MA_WRAP=0 --xr=3.68 --txs=1.84 --an=20 --ynumber=16 --dp=27.6 --dphi=166.715 --apix=1.84 --rmin=0.0 --rmax=64 --MPI\n' % (seg_ny, fract)) f.write('\n')
def refine_set(particle_indices): """ Runs the defocus refinement for a set of particles. :param particle_indices: Indicies of the particles inside the stack :return: List of refined CTFs. """ try: refined_ctfs = [] particle_micrographs = {} for particle_index in particle_indices: particle = sp_ctf_refine_io.read_particle(STACK_FILE_PATH, particle_index) # particle.write_image("/home/twagner/temp/refine/"+str(particle_index)+"particle.hdf") if RESOLUTION is not None and PIXEL_SIZE is not None: particle = particle.process( "filter.lowpass.gauss", {"cutoff_freq": 1.0 / RESOLUTION, "apix": PIXEL_SIZE}, ) # In case of phase plate a high pass might be necessary # if PARTICLE_DIAMETER is not None # particle = particle.process("filter.highpass.gauss",{"cutoff_freq":1.0/160}) particle_projection_params = PROJECTION_PARAMETERS[particle_index].tolist() # As i shift the volume and not the particle, multiply by -1 particle_projection_params[3] = -1 * particle_projection_params[3] particle_projection_params[4] = -1 * particle_projection_params[4] # particle_projection_params[3] = add_proj_error(particle_projection_params, 3, 2) # particle_projection_params[4] = add_proj_error(particle_projection_params, 4, 2) # particle_projection_params[2] = add_proj_error(particle_projection_params, 2, 2) particle_ctf = particle.get_attr("ctf") volume = VOLUME1 if HALF_MAP_ASSIGNEMENTS is not None: if HALF_MAP_ASSIGNEMENTS[particle_index] != 0: volume = VOLUME2 best_ctf, error, drratio = refine_defocus_with_error_est( particle_volume=volume, particle_image=particle, projection_angle_and_shift=particle_projection_params, current_ctf=particle_ctf, def_search_range=DEFOCUS_SEARCH_RANGE, def_step_size=DEFOCUS_STEP_SIZE, ) particle_micrograph_filename = particle.get_attr("ptcl_source_image") if particle_micrograph_filename in particle_micrographs: particle_micrographs[particle_micrograph_filename]["indices"].append( particle_index ) particle_micrographs[particle_micrograph_filename]["defocus"].append( best_ctf.defocus ) particle_micrographs[particle_micrograph_filename]["diff"].append( particle_ctf.defocus - best_ctf.defocus ) particle_micrographs[particle_micrograph_filename]["error"].append( error ) particle_micrographs[particle_micrograph_filename]["drratio"].append( drratio ) else: particle_micrographs[particle_micrograph_filename] = { "indices": [particle_index], "diff": [particle_ctf.defocus - best_ctf.defocus], "error": [error], "defocus": [best_ctf.defocus], "drratio": [drratio], } refined_ctfs.append(best_ctf) del particle_projection_params del volume del particle_ctf del particle except Exception as err: import traceback sp_global_def.sxprint( "Exception happend for particles in range ", np.min(particle_indices), "-", np.max(particle_indices), ) traceback.print_exc() raise err return refined_ctfs, particle_micrographs
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + """ Input Output [options] Generate three micrographs, each micrograph contains one projection of a long filament. Input: Reference Volume, output directory Output: Three micrographs stored in output directory sxhelical_demo.py tmp.hdf mic --generate_micrograph --CTF --apix=1.84 Generate noisy cylinder ini.hdf with radius 35 pixels and box size 100 by 100 by 200 sxhelical_demo.py ini.hdf --generate_noisycyl --boxsize="100,100,200" --rad=35 Generate rectangular 2D mask mask2d.hdf with width 60 pixels and image size 200 by 200 pixels sxhelical_demo.py mask2d.hdf --generate_mask --masksize="200,200" --maskwidth=60 Apply the centering parameters to bdb:adata, normalize using average and standard deviation outside the mask, and output the new images to bdb:data sxhelical_demo.py bdb:adata bdb:data mask2d.hdf --applyparams Generate run through example script for helicon sxhelical_demo.py --generate_script --filename=run --seg_ny=180 --ptcl_dist=15 --fract=0.35 """ parser = OptionParser(usage, version=SPARXVERSION) # helicise the Atom coordinates # generate micrographs of helical filament parser.add_option( "--generate_micrograph", action="store_true", default=False, help= "Generate three micrographs where each micrograph contains one projection of a long filament. \n Input: Reference Volume, output directory \n Output: Three micrographs containing helical filament projections stored in output directory" ) parser.add_option("--CTF", action="store_true", default=False, help="Use CTF correction") parser.add_option("--apix", type="float", default=-1, help="pixel size in Angstroms") parser.add_option( "--rand_seed", type="int", default=14567, help= "the seed used for generating random numbers (default 14567) for adding noise to the generated micrographs." ) parser.add_option("--Cs", type="float", default=2.0, help="Microscope Cs (spherical aberation)") parser.add_option("--voltage", type="float", default=200.0, help="Microscope voltage in KV") parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast (percentage, default=10)") parser.add_option("--nonoise", action="store_true", default=False, help="Do not add noise to the micrograph.") # generate initial volume parser.add_option("--generate_noisycyl", action="store_true", default=False, help="Generate initial volume of noisy cylinder.") parser.add_option( "--boxsize", type="string", default="100,100,200", help= "String containing x , y, z dimensions (separated by comma) in pixels") parser.add_option("--rad", type="int", default=35, help="Radius of initial volume in pixels") # generate 2D mask parser.add_option("--generate_mask", action="store_true", default=False, help="Generate 2D rectangular mask.") parser.add_option( "--masksize", type="string", default="200,200", help= "String containing x and y dimensions (separated by comma) in pixels") parser.add_option("--maskwidth", type="int", default=60, help="Width of rectangular mask") # Apply 2D alignment parameters to input stack and output new images to output stack parser.add_option( "--applyparams", action="store_true", default=False, help= "Apply the centering parameters to input stack, normalize using average and standard deviation outside the mask, and output the new images to output stack" ) # Generate run script parser.add_option("--generate_script", action="store_true", default=False, help="Generate script for helicon run through example") parser.add_option("--filename", type="string", default="runhelicon", help="Name of run script to generate") parser.add_option("--seg_ny", type="int", default=180, help="y-dimension of segment used for refinement") parser.add_option( "--ptcl_dist", type="int", default=15, help= "Distance in pixels between adjacent segments windowed from same filament" ) parser.add_option( "--fract", type="float", default=0.35, help="Fraction of the volume used for applying helical symmetry.") (options, args) = parser.parse_args() if len(args) > 3: sxprint("usage: " + usage) sxprint("Please run '" + progname + " -h' for detailed options") ERROR( "Invalid number of parameters. Please see usage information above." ) return else: if options.generate_script: generate_runscript(options.filename, options.seg_ny, options.ptcl_dist, options.fract) if options.generate_micrograph: if options.apix <= 0: ERROR("Please enter pixel size.") return generate_helimic(args[0], args[1], options.apix, options.CTF, options.Cs, options.voltage, options.ac, options.nonoise, options.rand_seed) if options.generate_noisycyl: from sp_utilities import model_cylinder, model_gauss_noise outvol = args[0] boxdims = options.boxsize.split(',') if len(boxdims) < 1 or len(boxdims) > 3: ERROR( "Enter box size as string containing x , y, z dimensions (separated by comma) in pixels. E.g.: --boxsize=\'100,100,200\'" ) return nx = int(boxdims[0]) if len(boxdims) == 1: ny = nx nz = nx else: ny = int(boxdims[1]) if len(boxdims) == 3: nz = int(boxdims[2]) (model_cylinder(options.rad, nx, ny, nz) * model_gauss_noise(1.0, nx, ny, nz)).write_image(outvol) if options.generate_mask: from sp_utilities import model_blank, pad outvol = args[0] maskdims = options.masksize.split(',') if len(maskdims) < 1 or len(maskdims) > 2: ERROR( "Enter box size as string containing x , y dimensions (separated by comma) in pixels. E.g.: --boxsize=\'200,200\'" ) return nx = int(maskdims[0]) if len(maskdims) == 1: ny = nx else: ny = int(maskdims[1]) mask = pad(model_blank(options.maskwidth, ny, 1, 1.0), nx, ny, 1, 0.0) mask.write_image(outvol) if options.applyparams: from sp_utilities import get_im, get_params2D, set_params2D from sp_fundamentals import cyclic_shift stack = args[0] newstack = args[1] mask = get_im(args[2]) nima = EMUtil.get_image_count(stack) for im in range(nima): prj = get_im(stack, im) alpha, sx, sy, mirror, scale = get_params2D(prj) prj = cyclic_shift(prj, int(sx)) set_params2D(prj, [0.0, 0., 0.0, 0, 1]) stat = Util.infomask(prj, mask, False) prj = (prj - stat[0]) / stat[1] ctf_params = prj.get_attr("ctf") prj.set_attr('ctf_applied', 0) prj.write_image(newstack, im)
def main(): progname = os.path.basename(sys.argv[0]) usage = progname + " averages1 averages2 --th_grp" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--T", type="int", default=0, help=" Threshold for matching") parser.add_option("--J", type="int", default=50, help=" J") parser.add_option("--max_branching", type="int", default=40, help=" maximum branching") parser.add_option("--verbose", action="store_true", default=False, help=" Threshold for matching") parser.add_option("--timing", action="store_true", default=False, help=" Get the timing") (options, args) = parser.parse_args() if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() sp_global_def.BATCH = True from numpy import array from sp_statistics import k_means_stab_bbenum R = len(args) Parts = [] mem = [0] * R avg = [0] * R for r in range(R): data = EMData.read_images(args[r]) avg[r] = len(data) part = [] for k in range(len(data)): lid = data[k].get_attr('members') mem[r] += len(lid) lid = array(lid, 'int32') lid.sort() part.append(lid.copy()) Parts.append(part) if options.timing: from time import time time1 = time() MATCH, STB_PART, CT_s, CT_t, ST, st = k_means_stab_bbenum( Parts, T=options.T, J=options.J, max_branching=options.max_branching, stmult=0.1, branchfunc=2) if options.verbose: sxprint(MATCH) sxprint(STB_PART) sxprint(CT_s) sxprint(CT_t) sxprint(ST) sxprint(st) sxprint(" ") for i in range(len(MATCH)): u = MATCH[i][0] # u is the group in question in partition 1 assert len(STB_PART[u]) == CT_s[u] sxprint("Group ", end=' ') for r in range(R): sxprint("%3d " % (MATCH[i][r]), end=' ') sxprint(" matches: group size = ", end=' ') for r in range(R): sxprint(" %3d" % len(Parts[r][MATCH[i][r]]), end=' ') sxprint(" matched size = %3d" % (CT_s[u]), end=' ') if options.verbose: sxprint(" matched group = %s" % (STB_PART[u])) else: sxprint("") sxprint("\nNumber of averages = ", end=' ') for r in range(R): sxprint("%3d" % (avg[r]), end=' ') sxprint("\nTotal number of particles = ", end=' ') for r in range(R): sxprint("%3d" % (mem[r]), end=' ') sxprint(" number of matched particles = %5d" % (sum(CT_s))) if options.timing: sxprint("Elapsed time = ", time() - time1) sp_global_def.BATCH = False
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " stack ref_vol outdir <maskfile> --ir=inner_radius --ou=outer_radius --rs=ring_step --xr=x_range --ynumber=y_numbers --txs=translational_search_stepx --delta=angular_step --an=angular_neighborhood --center=1 --maxit=max_iter --CTF --snr=1.0 --ref_a=S --sym=c1 --datasym=symdoc --new" parser = OptionParser(usage, version=SPARXVERSION) #parser.add_option("--ir", type="float", default= -1, help="inner radius for rotational correlation > 0 (set to 1) (Angstroms)") parser.add_option( "--ou", type="float", default=-1, help= "outer radius for rotational 2D correlation < int(nx/2)-1 (set to the radius of the particle) (Angstroms)" ) parser.add_option( "--rs", type="int", default=1, help="step between rings in rotational correlation >0 (set to 1)") parser.add_option( "--xr", type="string", default=" 4 2 1 1 1", help= "range for translation search in x direction, search is +/-xr (Angstroms) " ) parser.add_option( "--txs", type="string", default="1 1 1 0.5 0.25", help= "step size of the translation search in x directions, search is -xr, -xr+ts, 0, xr-ts, xr (Angstroms)" ) parser.add_option( "--y_restrict", type="string", default="-1 -1 -1 -1 -1", help= "range for translational search in y-direction, search is +/-y_restrict in Angstroms. This only applies to local search, i.e., when an is not -1. If y_restrict < 0, then for ihrsrlocalcons (option --localcons local search with consistency), the y search range is set such that it is the same ratio to dp as angular search range is to dphi. For regular ihrsr, y search range is the full range when y_restrict< 0. Default is -1." ) parser.add_option( "--ynumber", type="string", default="4 8 16 32 32", help= "even number of the translation search in y direction, search is (-dpp/2,-dpp/2+dpp/ny,,..,0,..,dpp/2-dpp/ny dpp/2]" ) parser.add_option("--delta", type="string", default=" 10 6 4 3 2", help="angular step of reference projections") parser.add_option( "--an", type="string", default="-1", help= "angular neighborhood for local searches (default -1, meaning do exhaustive search)" ) parser.add_option( "--maxit", type="int", default=30, help= "maximum number of iterations performed for each angular step (default 30) " ) parser.add_option("--CTF", action="store_true", default=False, help="CTF correction") parser.add_option("--snr", type="float", default=1.0, help="Signal-to-Noise Ratio of the data (default 1)") parser.add_option("--MPI", action="store_true", default=True, help="use MPI version") #parser.add_option("--fourvar", action="store_true", default=False, help="compute Fourier variance") parser.add_option("--apix", type="float", default=-1.0, help="pixel size in Angstroms") parser.add_option("--dp", type="float", default=-1.0, help="delta z - translation in Angstroms") parser.add_option("--dphi", type="float", default=-1.0, help="delta phi - rotation in degrees") parser.add_option( "--ndp", type="int", default=12, help= "In symmetrization search, number of delta z steps equals to 2*ndp+1") parser.add_option( "--ndphi", type="int", default=12, help="In symmetrization search,number of dphi steps equas to 2*ndphi+1" ) parser.add_option("--dp_step", type="float", default=0.1, help="delta z (Angstroms) step for symmetrization") parser.add_option("--dphi_step", type="float", default=0.1, help="dphi step for symmetrization") parser.add_option( "--psi_max", type="float", default=10.0, help= "maximum psi - how far rotation in plane can can deviate from 90 or 270 degrees (default 10)" ) parser.add_option("--rmin", type="float", default=0.0, help="minimal radius for hsearch (Angstroms)") parser.add_option("--rmax", type="float", default=80.0, help="maximal radius for hsearch (Angstroms)") parser.add_option("--fract", type="float", default=0.7, help="fraction of the volume used for helical search") parser.add_option("--sym", type="string", default="c1", help="symmetry of the structure") parser.add_option("--function", type="string", default="helical", help="name of the reference preparation function") parser.add_option("--datasym", type="string", default="datasym.txt", help="symdoc") parser.add_option( "--nise", type="int", default=200, help="start symmetrization after nise steps (default 200)") parser.add_option("--npad", type="int", default=2, help="padding size for 3D reconstruction, (default 2)") parser.add_option("--debug", action="store_true", default=False, help="debug") parser.add_option("--new", action="store_true", default=False, help="use rectangular recon and projection version") parser.add_option( "--initial_theta", type="float", default=90.0, help="intial theta for reference projection (default 90)") parser.add_option( "--delta_theta", type="float", default=1.0, help="delta theta for reference projection (default 1.0)") parser.add_option("--WRAP", type="int", default=1, help="do helical wrapping (default 1, meaning yes)") (options, args) = parser.parse_args(arglist[1:]) if len(args) < 1 or len(args) > 5: sxprint("usage: " + usage + "\n") sxprint("Please run '" + progname + " -h' for detailed options") ERROR( "Invalid number of parameters used. please see usage information above." ) return else: # Convert input arguments in the units/format as expected by ihrsr_MPI in applications. if options.apix < 0: ERROR("Please enter pixel size") return rminp = int((float(options.rmin) / options.apix) + 0.5) rmaxp = int((float(options.rmax) / options.apix) + 0.5) from sp_utilities import get_input_from_string, get_im xr = get_input_from_string(options.xr) txs = get_input_from_string(options.txs) y_restrict = get_input_from_string(options.y_restrict) irp = 1 if options.ou < 0: oup = -1 else: oup = int((options.ou / options.apix) + 0.5) xrp = '' txsp = '' y_restrict2 = '' for i in range(len(xr)): xrp += " " + str(float(xr[i]) / options.apix) for i in range(len(txs)): txsp += " " + str(float(txs[i]) / options.apix) # now y_restrict has the same format as x search range .... has to change ihrsr accordingly for i in range(len(y_restrict)): y_restrict2 += " " + str(float(y_restrict[i]) / options.apix) if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_applications import ihrsr sp_global_def.BATCH = True if len(args) < 4: mask = None else: mask = args[3] ihrsr(args[0], args[1], args[2], mask, irp, oup, options.rs, xrp, options.ynumber, txsp, options.delta, options.initial_theta, options.delta_theta, options.an, options.maxit, options.CTF, options.snr, options.dp, options.ndp, options.dp_step, options.dphi, options.ndphi, options.dphi_step, options.psi_max, rminp, rmaxp, options.fract, options.nise, options.npad, options.sym, options.function, options.datasym, options.apix, options.debug, options.MPI, options.WRAP, y_restrict2) sp_global_def.BATCH = False
def main(): arglist = [] for arg in sys.argv: arglist.append(arg) progname = os.path.basename(arglist[0]) usage = progname + " stack ref_vol outdir --dp=rise --dphi=rotation --apix=pixel_size --phistep=phi_step --zstep=z_step --fract=helicising_fraction --rmax=maximum_radius --rmin=min_radius --CTF --sym=c1 --function=user_function --maxit=max_iter --MPI" parser = OptionParser(usage, version=SPARXVERSION) parser.add_option("--dp", type="float", default=1.0, help="delta z - translation in Angstroms") parser.add_option("--dphi", type="float", default=1.0, help="delta phi - rotation in degrees") parser.add_option("--apix", type="float", default=1.84, help="pixel size in Angstroms") parser.add_option("--rmin", type="int", default=0, help="minimal radial extent of structure") parser.add_option("--rmax", type="int", default=70, help="maximal radial extent of structure") parser.add_option("--fract", type="float", default=0.66, help="fraction of the volume used for helical search") parser.add_option("--sym", type="string", default="c1", help="symmetry of the structure") parser.add_option("--function", type="string", default="helical", help="name of the reference preparation function") parser.add_option("--zstep", type="int", default=1, help="Step size for translational search along z") parser.add_option("--CTF", action="store_true", default=False, help="CTF correction") parser.add_option("--maxit", type="int", default=5, help="maximum number of iterations performed") parser.add_option("--MPI", action="store_true", default=False, help="use MPI version") (options, args) = parser.parse_args(arglist[1:]) if len(args) != 3: sxprint("Usage: " + usage) sxprint("Please run \'" + progname + " -h\' for detailed options") ERROR( "Invalid number of parameters used. Please see usage information above." ) return else: if not options.MPI: ERROR( "There is only MPI version of sxfilrecons3d.py. See SPARX wiki page for downloading MyMPI details." ) return if sp_global_def.CACHE_DISABLE: from sp_utilities import disable_bdb_cache disable_bdb_cache() from sp_development import filrecons3D_MPI sp_global_def.BATCH = True filrecons3D_MPI(args[0], args[1], args[2], options.dp, options.dphi, options.apix, options.function, options.zstep, options.fract, options.rmax, options.rmin, options.CTF, options.maxit, options.sym) sp_global_def.BATCH = False
def align_diff_params(ali_params1, ali_params2): """ This function determines the relative angle, shifts and mirrorness between two sets of alignment parameters. """ nima = len(ali_params1) nima2 = len(ali_params2) if nima2 != nima: sp_global_def.sxprint("Error: Number of images do not agree!") return 0.0, 0.0, 0.0, 0 else: nima /= 4 del nima2 # Read the alignment parameters and determine the relative mirrorness mirror_same = 0 for i in range(nima): if ali_params1[i * 4 + 3] == ali_params2[i * 4 + 3]: mirror_same += 1 if mirror_same > nima / 2: mirror = 0 else: mirror_same = nima - mirror_same mirror = 1 # Determine the relative angle cosi = 0.0 sini = 0.0 angle1 = [] angle2 = [] for i in range(nima): mirror1 = ali_params1[i * 4 + 3] mirror2 = ali_params2[i * 4 + 3] if abs(mirror1 - mirror2) == mirror: alpha1 = ali_params1[i * 4] alpha2 = ali_params2[i * 4] if mirror1 == 1: alpha1 = -alpha1 alpha2 = -alpha2 angle1.append(alpha1) angle2.append(alpha2) alphai = angle_diff(angle1, angle2) # Determine the relative shift sxi = 0.0 syi = 0.0 for i in range(nima): mirror1 = ali_params1[i * 4 + 3] mirror2 = ali_params2[i * 4 + 3] if abs(mirror1 - mirror2) == mirror: alpha1 = ali_params1[i * 4] # alpha2 = ali_params2[i*4] sx1 = ali_params1[i * 4 + 1] sx2 = ali_params2[i * 4 + 1] sy1 = ali_params1[i * 4 + 2] sy2 = ali_params2[i * 4 + 2] alpha12, sx12, sy12, mirror12 = sp_utilities.combine_params2( alpha1, sx1, sy1, int(mirror1), alphai, 0.0, 0.0, 0) if mirror1 == 0: sxi += sx2 - sx12 else: sxi -= sx2 - sx12 syi += sy2 - sy12 sxi /= mirror_same syi /= mirror_same return alphai, sxi, syi, mirror
def helicalshiftali_MPI(stack, maskfile=None, maxit=100, CTF=False, snr=1.0, Fourvar=False, search_rng=-1): nproc = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD) myid = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD) main_node = 0 ftp = file_type(stack) if myid == main_node: print_begin_msg("helical-shiftali_MPI") max_iter = int(maxit) if (myid == main_node): infils = EMUtil.get_all_attributes(stack, "filament") ptlcoords = EMUtil.get_all_attributes(stack, 'ptcl_source_coord') filaments = ordersegments(infils, ptlcoords) total_nfils = len(filaments) inidl = [0] * total_nfils for i in range(total_nfils): inidl[i] = len(filaments[i]) linidl = sum(inidl) nima = linidl tfilaments = [] for i in range(total_nfils): tfilaments += filaments[i] del filaments else: total_nfils = 0 linidl = 0 total_nfils = bcast_number_to_all(total_nfils, source_node=main_node) if myid != main_node: inidl = [-1] * total_nfils inidl = bcast_list_to_all(inidl, myid, source_node=main_node) linidl = bcast_number_to_all(linidl, source_node=main_node) if myid != main_node: tfilaments = [-1] * linidl tfilaments = bcast_list_to_all(tfilaments, myid, source_node=main_node) filaments = [] iendi = 0 for i in range(total_nfils): isti = iendi iendi = isti + inidl[i] filaments.append(tfilaments[isti:iendi]) del tfilaments, inidl if myid == main_node: print_msg("total number of filaments: %d" % total_nfils) if total_nfils < nproc: ERROR( 'number of CPUs (%i) is larger than the number of filaments (%i), please reduce the number of CPUs used' % (nproc, total_nfils), myid=myid) # balanced load temp = chunks_distribution([[len(filaments[i]), i] for i in range(len(filaments))], nproc)[myid:myid + 1][0] filaments = [filaments[temp[i][1]] for i in range(len(temp))] nfils = len(filaments) #filaments = [[0,1]] #print "filaments",filaments list_of_particles = [] indcs = [] k = 0 for i in range(nfils): list_of_particles += filaments[i] k1 = k + len(filaments[i]) indcs.append([k, k1]) k = k1 data = EMData.read_images(stack, list_of_particles) ldata = len(data) sxprint("ldata=", ldata) nx = data[0].get_xsize() ny = data[0].get_ysize() if maskfile == None: mrad = min(nx, ny) // 2 - 2 mask = pad(model_blank(2 * mrad + 1, ny, 1, 1.0), nx, ny, 1, 0.0) else: mask = get_im(maskfile) # apply initial xform.align2d parameters stored in header init_params = [] for im in range(ldata): t = data[im].get_attr('xform.align2d') init_params.append(t) p = t.get_params("2d") data[im] = rot_shift2D(data[im], p['alpha'], p['tx'], p['ty'], p['mirror'], p['scale']) if CTF: from sp_filter import filt_ctf from sp_morphology import ctf_img ctf_abs_sum = EMData(nx, ny, 1, False) ctf_2_sum = EMData(nx, ny, 1, False) else: ctf_2_sum = None ctf_abs_sum = None from sp_utilities import info for im in range(ldata): data[im].set_attr('ID', list_of_particles[im]) st = Util.infomask(data[im], mask, False) data[im] -= st[0] if CTF: ctf_params = data[im].get_attr("ctf") qctf = data[im].get_attr("ctf_applied") if qctf == 0: data[im] = filt_ctf(fft(data[im]), ctf_params) data[im].set_attr('ctf_applied', 1) elif qctf != 1: ERROR('Incorrectly set qctf flag', myid=myid) ctfimg = ctf_img(nx, ctf_params, ny=ny) Util.add_img2(ctf_2_sum, ctfimg) Util.add_img_abs(ctf_abs_sum, ctfimg) else: data[im] = fft(data[im]) del list_of_particles if CTF: reduce_EMData_to_root(ctf_2_sum, myid, main_node) reduce_EMData_to_root(ctf_abs_sum, myid, main_node) if CTF: if myid != main_node: del ctf_2_sum del ctf_abs_sum else: temp = EMData(nx, ny, 1, False) tsnr = 1. / snr for i in range(0, nx + 2, 2): for j in range(ny): temp.set_value_at(i, j, tsnr) temp.set_value_at(i + 1, j, 0.0) #info(ctf_2_sum) Util.add_img(ctf_2_sum, temp) #info(ctf_2_sum) del temp total_iter = 0 shift_x = [0.0] * ldata for Iter in range(max_iter): if myid == main_node: start_time = time() print_msg("Iteration #%4d\n" % (total_iter)) total_iter += 1 avg = EMData(nx, ny, 1, False) for im in range(ldata): Util.add_img(avg, fshift(data[im], shift_x[im])) reduce_EMData_to_root(avg, myid, main_node) if myid == main_node: if CTF: tavg = Util.divn_filter(avg, ctf_2_sum) else: tavg = Util.mult_scalar(avg, 1.0 / float(nima)) else: tavg = model_blank(nx, ny) if Fourvar: bcast_EMData_to_all(tavg, myid, main_node) vav, rvar = varf2d_MPI(myid, data, tavg, mask, "a", CTF) if myid == main_node: if Fourvar: tavg = fft(Util.divn_img(fft(tavg), vav)) vav_r = Util.pack_complex_to_real(vav) # normalize and mask tavg in real space tavg = fft(tavg) stat = Util.infomask(tavg, mask, False) tavg -= stat[0] Util.mul_img(tavg, mask) tavg.write_image("tavg.hdf", Iter) # For testing purposes: shift tavg to some random place and see if the centering is still correct #tavg = rot_shift3D(tavg,sx=3,sy=-4) if Fourvar: del vav bcast_EMData_to_all(tavg, myid, main_node) tavg = fft(tavg) sx_sum = 0.0 nxc = nx // 2 for ifil in range(nfils): """ # Calculate filament average avg = EMData(nx, ny, 1, False) filnima = 0 for im in xrange(indcs[ifil][0], indcs[ifil][1]): Util.add_img(avg, data[im]) filnima += 1 tavg = Util.mult_scalar(avg, 1.0/float(filnima)) """ # Calculate 1D ccf between each segment and filament average nsegms = indcs[ifil][1] - indcs[ifil][0] ctx = [None] * nsegms pcoords = [None] * nsegms for im in range(indcs[ifil][0], indcs[ifil][1]): ctx[im - indcs[ifil][0]] = Util.window(ccf(tavg, data[im]), nx, 1) pcoords[im - indcs[ifil][0]] = data[im].get_attr( 'ptcl_source_coord') #ctx[im-indcs[ifil][0]].write_image("ctx.hdf",im-indcs[ifil][0]) #print " CTX ",myid,im,Util.infomask(ctx[im-indcs[ifil][0]], None, True) # search for best x-shift cents = nsegms // 2 dst = sqrt( max((pcoords[cents][0] - pcoords[0][0])**2 + (pcoords[cents][1] - pcoords[0][1])**2, (pcoords[cents][0] - pcoords[-1][0])**2 + (pcoords[cents][1] - pcoords[-1][1])**2)) maxincline = atan2(ny // 2 - 2 - float(search_rng), dst) kang = int(dst * tan(maxincline) + 0.5) #print " settings ",nsegms,cents,dst,search_rng,maxincline,kang # ## C code for alignment. @ming results = [0.0] * 3 results = Util.helixshiftali(ctx, pcoords, nsegms, maxincline, kang, search_rng, nxc) sib = int(results[0]) bang = results[1] qm = results[2] #print qm, sib, bang # qm = -1.e23 # # for six in xrange(-search_rng, search_rng+1,1): # q0 = ctx[cents].get_value_at(six+nxc) # for incline in xrange(kang+1): # qt = q0 # qu = q0 # if(kang>0): tang = tan(maxincline/kang*incline) # else: tang = 0.0 # for kim in xrange(cents+1,nsegms): # dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) # xl = dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # #print " A ", ifil,six,incline,kim,xl,ixl,dxl # qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # xl = -dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # for kim in xrange(cents): # dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) # xl = -dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qt += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # xl = dst*tang+six+nxc # ixl = int(xl) # dxl = xl - ixl # qu += (1.0-dxl)*ctx[kim].get_value_at(ixl) + dxl*ctx[kim].get_value_at(ixl+1) # if( qt > qm ): # qm = qt # sib = six # bang = tang # if( qu > qm ): # qm = qu # sib = six # bang = -tang #if incline == 0: print "incline = 0 ",six,tang,qt,qu #print qm,six,sib,bang #print " got results ",indcs[ifil][0], indcs[ifil][1], ifil,myid,qm,sib,tang,bang,len(ctx),Util.infomask(ctx[0], None, True) for im in range(indcs[ifil][0], indcs[ifil][1]): kim = im - indcs[ifil][0] dst = sqrt((pcoords[cents][0] - pcoords[kim][0])**2 + (pcoords[cents][1] - pcoords[kim][1])**2) if (kim < cents): xl = -dst * bang + sib else: xl = dst * bang + sib shift_x[im] = xl # Average shift sx_sum += shift_x[indcs[ifil][0] + cents] # #print myid,sx_sum,total_nfils sx_sum = mpi.mpi_reduce(sx_sum, 1, mpi.MPI_FLOAT, mpi.MPI_SUM, main_node, mpi.MPI_COMM_WORLD) if myid == main_node: sx_sum = float(sx_sum[0]) / total_nfils print_msg("Average shift %6.2f\n" % (sx_sum)) else: sx_sum = 0.0 sx_sum = 0.0 sx_sum = bcast_number_to_all(sx_sum, source_node=main_node) for im in range(ldata): shift_x[im] -= sx_sum #print " %3d %6.3f"%(im,shift_x[im]) #exit() # combine shifts found with the original parameters for im in range(ldata): t1 = Transform() ##import random ##shix=random.randint(-10, 10) ##t1.set_params({"type":"2D","tx":shix}) t1.set_params({"type": "2D", "tx": shift_x[im]}) # combine t0 and t1 tt = t1 * init_params[im] data[im].set_attr("xform.align2d", tt) # write out headers and STOP, under MPI writing has to be done sequentially mpi.mpi_barrier(mpi.MPI_COMM_WORLD) par_str = ["xform.align2d", "ID"] if myid == main_node: from sp_utilities import file_type if (file_type(stack) == "bdb"): from sp_utilities import recv_attr_dict_bdb recv_attr_dict_bdb(main_node, stack, data, par_str, 0, ldata, nproc) else: from sp_utilities import recv_attr_dict recv_attr_dict(main_node, stack, data, par_str, 0, ldata, nproc) else: send_attr_dict(main_node, data, par_str, 0, ldata) if myid == main_node: print_end_msg("helical-shiftali_MPI")
def update_plot(self): val = self.curset ctf = self.data[val][1] ds = self.data[val][1].dsbg s = [ds * i for i in range(len(ctf.background))] if self.plotmode == 1: self.guiplot.set_data((s, self.data[val][2]), "fg", True, True) self.guiplot.set_data((s, self.data[val][3]), "bg") self.guiplot.setAxisParms("s (1/A)", "Intensity (a.u)") elif self.plotmode == 0: bgsub = [ self.data[val][2][i] - self.data[val][3][i] for i in range(len(self.data[val][2])) ] self.guiplot.set_data((s, bgsub), "fg-bg", True, True) fit = ctf.compute_1d(len(s) * 2, ds, Ctf.CtfType.CTF_AMP) # The fit curve sf = sfact(s, "ribosome", "nono") fit = [sf[i] * fit[i]**2 for i in range(len(s)) ] # squared * a generic structure factor # auto-amplitude for b-factor adjustment rto, nrto = 0, 0 for i in range(int(.04 / ds) + 1, min(int(0.15 / ds), len(s) - 1)): if bgsub[i] > 0: #rto+=fit[i]**2/fabs(bgsub[i]) #nrto+=fit[i] #rto+=fit[i]**2 #nrto+=bgsub[i]**2 rto += fit[i] nrto += fabs(bgsub[i]) if nrto == 0: rto = 1.0 else: rto /= nrto fit = [fit[i] / rto for i in range(len(s))] self.guiplot.set_data((s, fit), "fit") self.guiplot.setAxisParms("s (1/A)", "Intensity (a.u)") elif self.plotmode == 2: snr = ctf.compute_1d(len(s) * 2, ds, Ctf.CtfType.CTF_SNR) # The snr curve self.guiplot.set_data((s, snr[:len(s)]), "snr", True) self.guiplot.setAxisParms("s (1/A)", "SNR (intensity ratio)") elif self.plotmode == 3: snr = ctf.compute_1d(len(s) * 2, ds, Ctf.CtfType.CTF_SNR) # The snr curve self.guiplot.set_data((s, snr[:len(s)]), "snr", True) ssnr = ctf.compute_1d(len(s) * 2, ds, Ctf.CtfType.CTF_SNR_SMOOTH) # The fit curve self.guiplot.set_data((s, ssnr[:len(s)]), "ssnr") self.guiplot.setAxisParms("s (1/A)", "SNR (intensity ratio)") elif self.plotmode == 4: snr = ctf.compute_1d(len(s) * 2, ds, Ctf.CtfType.CTF_SNR) # The snr curve for i in range(1, len(snr)): snr[i] = snr[i] * i + snr[i - 1] # integrate SNR*s # for i in range(1,len(snr)): snr[i]/=snr[-1] # normalize for i in range(1, len(snr)): snr[i] /= len( snr ) # this way the relative quality of images can be compared self.guiplot.set_data((s, snr[:len(s)]), "snr", True) self.guiplot.setAxisParms("s (1/A)", "Integrated SNR") elif self.plotmode == 5: inten = [ fabs(i) for i in ctf.compute_1d(len(s) * 2, ds, Ctf.CtfType.CTF_AMP) ] # The snr curve self.guiplot.set_data((s, inten[:len(s)]), "single", True) all = [0 for i in inten] for st in self.data: sxprint(st) inten = [ fabs(i) for i in st[1].compute_1d( len(s) * 2, ds, Ctf.CtfType.CTF_AMP) ] for i in range(len(all)): all[i] += inten[i] self.guiplot.set_data((s, all[:len(s)]), "total")