def takeoverHeaders(filename, numpart, boxsize, keepfiles=False):
### better workaround than copyFile ... still a workaround though
imagicroot = checkImagicExecutablePath()
basedir = os.path.split(filename)[0]
basename = os.path.split(filename)[1]
batchfile = os.path.join(basedir, "takeoverHeaders.batch")
if basename[-4:] == ".img" or basename[-4:] == ".hed":
stripped_file = basename[:-4]
else:
stripped_file = basename
f = open(batchfile, 'w')
f.write("#!/bin/csh -f\n")
f.write("setenv IMAGIC_BATCH 1\n")
f.write("cd %s\n" % (basedir))
f.write(str(imagicroot)+"/stand/testim.e <<EOF\n")
f.write("test,1,%d\n" % (numpart))
f.write("%d,%d\n" % (boxsize, boxsize))
f.write("REAL\n")
f.write("BLOBS\n")
f.write("EOF\n")
f.close()
proc = subprocess.Popen('chmod 755 '+batchfile, shell=True)
proc.wait()
apParam.runCmd(batchfile, "IMAGIC")
shutil.move(os.path.join(basedir, "test.hed"), os.path.join(basedir, stripped_file+".hed"))
os.remove(os.path.join(basedir, "test.img"))
if keepfiles is not True:
os.remove(batchfile)
def scaleTemplates(self):
reffile = os.path.join(self.params['rundir'], "references.hed")
if self.params['apix'] != self.templatestack['apix']:
scalefactor = float(
self.templatestack['apix']) / self.params['apix']
templates = apImagicFile.readImagic(reffile)
scaledtemplates = []
for templatearray in templates['images']:
newarray = apTemplate.scaleTemplate(templatearray, scalefactor)
scaledtemplates.append(newarray)
apImagicFile.writeImagic(scaledtemplates, reffile)
refbox = apFile.getBoxSize(reffile)[0]
stbox = self.params['boxsize']
### now clip the references to get identical boxsizes
if stbox != refbox:
while os.path.isfile(reffile + ".new.img"):
apFile.removeStack(reffile + ".new.img")
emancmd = "proc2d " + reffile + " " + reffile + ".new.hed clip=" + str(
stbox) + " edgenorm"
apParam.runCmd(emancmd, "EMAN")
os.rename(reffile + ".new.hed", reffile)
os.rename(reffile + ".new.img", reffile[:-4] + ".img")
return
def takeoverHeaders(filename, numpart, boxsize, keepfiles=False):
### better workaround than copyFile ... still a workaround though
imagicroot = checkImagicExecutablePath()
basedir = os.path.split(filename)[0]
basename = os.path.split(filename)[1]
batchfile = os.path.join(basedir, "takeoverHeaders.batch")
if basename[-4:] == ".img" or basename[-4:] == ".hed":
stripped_file = basename[:-4]
else:
stripped_file = basename
f = open(batchfile, 'w')
f.write("#!/bin/csh -f\n")
f.write("setenv IMAGIC_BATCH 1\n")
f.write("cd %s\n" % (basedir))
f.write(str(imagicroot) + "/stand/testim.e <<EOF\n")
f.write("test,1,%d\n" % (numpart))
f.write("%d,%d\n" % (boxsize, boxsize))
f.write("REAL\n")
f.write("BLOBS\n")
f.write("EOF\n")
f.close()
proc = subprocess.Popen('chmod 755 ' + batchfile, shell=True)
proc.wait()
apParam.runCmd(batchfile, "IMAGIC")
shutil.move(os.path.join(basedir, "test.hed"),
os.path.join(basedir, stripped_file + ".hed"))
os.remove(os.path.join(basedir, "test.img"))
if keepfiles is not True:
os.remove(batchfile)
def start(self):
### database entry parameters
package_table = 'ApXmippRefineIterData|xmippParams'
### set projection-matching path
self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir']))
# self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], self.runparams['package_params']['WorkingDir']))
### check for variable root directories between file systems
apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['rundir'], self.runparams['remoterundir'], self.runparams['rundir'])
### determine which iterations to upload
lastiter = self.findLastCompletedIteration()
uploadIterations = self.verifyUploadIterations(lastiter)
### upload each iteration
for iteration in uploadIterations:
apDisplay.printColor("uploading iteration %d" % iteration, "cyan")
### set package parameters, as they will appear in database entries
package_database_object = self.instantiateProjMatchParamsData(iteration)
### move FSC file to results directory
oldfscfile = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_resolution.fsc" % iteration)
newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.fsc" % (self.params['timestamp'],iteration))
if os.path.exists(oldfscfile):
shutil.copyfile(oldfscfile, newfscfile)
### create a stack of class averages and reprojections (optional)
self.compute_stack_of_class_averages_and_reprojections(iteration)
### create a text file with particle information
self.createParticleDataFile(iteration)
if not self.params['euleronly']:
### create mrc file of map for iteration and reference number
oldvol = os.path.join(self.projmatchpath, "Iter_%d" % iteration, "Iter_%d_reconstruction.vol" % iteration)
newvol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol001.mrc" % (self.params['timestamp'], iteration))
mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix'])
apParam.runCmd(mrccmd, "EMAN")
### make chimera snapshot of volume
self.createChimeraVolumeSnapshot(newvol, iteration)
### instantiate database objects
self.insertRefinementRunData(iteration)
self.insertRefinementIterationData(iteration, package_table, package_database_object)
### calculate Euler jumps
if self.runparams['numiter'] > 1:
self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)
### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists
### e.g. {1: [5, 4, 3, 2, 1]} means 5 iters completed for refine 1
complete_refinements = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=False)
if self.params['cleanup_files'] is True:
self.cleanupFiles(complete_refinements)
def start(self):
### database entry parameters
package_table = 'ApXmippML3DRefineIterData|xmippML3DParams'
### set ml3d path
self.ml3dpath = os.path.abspath(os.path.join(self.params['rundir'], "recon", self.runparams['package_params']['WorkingDir'], "RunML3D"))
### check for variable root directories between file systems
if os.path.split(self.runparams['remoterundir'])[1] == "recon":
self.runparams['remoterundir'] = os.path.split(self.runparams['remoterundir'])[0]
apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(self.params['rundir'], self.runparams['remoterundir'], self.params['rundir'])
### determine which iterations to upload
lastiter = self.findLastCompletedIteration()
uploadIterations = self.verifyUploadIterations(lastiter)
### create ml3d_lib.doc file somewhat of a workaround, but necessary to make projections
total_num_2d_classes = self.createModifiedLibFile()
### upload each iteration
for iteration in uploadIterations:
### set package parameters, as they will appear in database entries
package_database_object = self.instantiateML3DParamsData(iteration)
for j in range(self.runparams['NumberOfReferences']):
### calculate FSC for each iteration using split selfile (selfile requires root directory change)
self.calculateFSCforIteration(iteration, j+1)
### create a stack of class averages and reprojections (optional)
self.compute_stack_of_class_averages_and_reprojections(iteration, j+1)
### create a text file with particle information
self.createParticleDataFile(iteration, j+1, total_num_2d_classes)
### create mrc file of map for iteration and reference number
oldvol = os.path.join(self.ml3dpath, "ml3d_it%.6d_vol%.6d.vol" % (iteration, j+1))
newvol = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.mrc" % (self.params['timestamp'], iteration, j+1))
mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol, self.runparams['apix'])
apParam.runCmd(mrccmd, "EMAN")
### make chimera snapshot of volume
self.createChimeraVolumeSnapshot(newvol, iteration, j+1)
### instantiate database objects
self.insertRefinementRunData(iteration, j+1)
self.insertRefinementIterationData(iteration, package_table, package_database_object, j+1)
### calculate Euler jumps
if self.runparams['numiter'] > 1:
self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)
### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists
### e.g. {1: [5, 4, 3, 2, 1], 2: [6, 5, 4, 3, 2, 1]} means 5 iters completed for refine 1 & 6 iters completed for refine 2
complete_refinements = self.verifyNumberOfCompletedRefinements(multiModelRefinementRun=True)
if self.params['cleanup_files'] is True:
self.cleanupFiles(complete_refinements)
def applyEnvelopeAndCTF(self, stack):
### get defocus lists
numpart = self.params['projcount']
cut = int(numpart/80.0)+1
apDisplay.printMsg("%d particles per dot"%(cut))
if len(self.deflist1) == 0:
self.getListOfDefoci(numpart)
### break up particles
partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc")
t0 = time.time()
apDisplay.printMsg("Applying CTF and Envelop to particles")
### apply CTF using ACE2
ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile)
### apply Envelop using ACE2
envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile)
### correct CTF using ACE2
if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True:
ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile)
else:
ctfcorrectdocfile = envelopdocfile
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
### write corrected particle list to doc file
ctfpartlist = []
ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst")
inf = open(ctfcorrectdocfile, 'r')
outf = open(ctfpartlistfile, "w")
for line in inf:
### get filename
filename = line.strip().split()[0]
if not os.path.isfile(filename):
apDisplay.printError("CTF and envelop apply failed")
ctfpartlist.append(filename)
outf.write(filename+"\t1\n")
inf.close()
outf.close()
### merge individual files into a common stack
ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed")
apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc")
if self.params['pad'] is True:
emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box'])
apParam.runCmd(emancmd, "EMAN")
shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4])
shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4])
return ctfstack, ctfpartlist
def applyEnvelopeAndCTF(self, stack):
### get defocus lists
numpart = self.params['projcount']
cut = int(numpart/80.0)+1
apDisplay.printMsg("%d particles per dot"%(cut))
if len(self.deflist1) == 0:
self.getListOfDefoci(numpart)
### break up particles
partlistdocfile = apXmipp.breakupStackIntoSingleFiles(stack, filetype="mrc")
t0 = time.time()
apDisplay.printMsg("Applying CTF and Envelop to particles")
### apply CTF using ACE2
ctfapplydocfile = self.applyCTFToDocFile(partlistdocfile)
### apply Envelop using ACE2
envelopdocfile = self.applyEnvelopToDocFile(ctfapplydocfile)
### correct CTF using ACE2
if self.params['ace2correct'] is True or self.params['ace2correct_rand'] is True:
ctfcorrectdocfile = self.correctCTFToDocFile(envelopdocfile)
else:
ctfcorrectdocfile = envelopdocfile
timeper = (time.time()-t0)/float(numpart)
apDisplay.printColor("Total time %s"%(apDisplay.timeString(time.time()-t0)), "green")
apDisplay.printColor("Time per particle %s"%(apDisplay.timeString(timeper)), "green")
### write corrected particle list to doc file
ctfpartlist = []
ctfpartlistfile = os.path.join(self.params['rundir'], "ctfpartlist.lst")
inf = open(ctfcorrectdocfile, 'r')
outf = open(ctfpartlistfile, "w")
for line in inf:
### get filename
filename = line.strip().split()[0]
if not os.path.isfile(filename):
apDisplay.printError("CTF and envelop apply failed")
ctfpartlist.append(filename)
outf.write(filename+"\t1\n")
inf.close()
outf.close()
### merge individual files into a common stack
ctfstack = os.path.join(self.params['rundir'], "ctfstack.hed")
apXmipp.gatherSingleFilesIntoStack(ctfpartlistfile, ctfstack, filetype="mrc")
if self.params['pad'] is True:
emancmd = "proc2d %s %s.clip.hed clip=%d,%d" % (ctfstack, ctfstack[:-4], self.params['box'], self.params['box'])
apParam.runCmd(emancmd, "EMAN")
shutil.move("%s.clip.hed" % ctfstack[:-4], "%s.hed" % ctfstack[:-4])
shutil.move("%s.clip.img" % ctfstack[:-4], "%s.img" % ctfstack[:-4])
return ctfstack, ctfpartlist
def addNoise(self, oldstack, noiselevel, SNR): ### create new image with modified SNR basename, extension = os.path.splitext(oldstack) formattedsnr = "%.3f" % (SNR) newstack = basename+"_snr"+formattedsnr+".hed" apFile.removeStack(newstack) emancmd = "proc2d "+oldstack+" "+newstack+" addnoise="+str(noiselevel) apParam.runCmd(emancmd, "EMAN") return newstack
def addNoise(self, oldstack, noiselevel, SNR): ### create new image with modified SNR basename, extension = os.path.splitext(oldstack) formattedsnr = "%.3f" % (SNR) newstack = basename+"_snr"+formattedsnr+".hed" apFile.removeStack(newstack) emancmd = "proc2d "+oldstack+" "+newstack+" addnoise="+str(noiselevel) apParam.runCmd(emancmd, "EMAN") return newstack
def start(self):
### get analysis paramteres
self.analysisdata = appiondata.ApAlignAnalysisRunData.direct_query(self.params['analysisId'])
pixelsize = self.analysisdata['alignstack']['pixelsize']
boxsize = self.analysisdata['alignstack']['boxsize']
bin = self.analysisdata['imagicMSArun']['bin']
self.params['apix'] = float(pixelsize) * int(bin)
self.params['boxsize'] = int(boxsize) / int(bin)
self.params['num_particles'] = self.analysisdata['alignstack']['num_particles']
starttime=time.time()
print self.params
print "... stack pixel size: "+str(self.params['apix'])
print "... stack box size: "+str(self.params['boxsize'])
apDisplay.printColor("Running IMAGIC .batch file: See imagicMSAcluster log file(s) corresponding to # of classes for details", "cyan")
### insert run into database
self.insertClusterRun()
### split the cluster numbers
numclasslist = self.params['num_classes'].split(",")
for item in numclasslist:
numclusters = int(item)
apDisplay.printColor("\n==========================\nprocessing class averages for "
+str(numclusters)+" classes\n==========================\n", "green")
### create IMAGIC batch file
batchfile = self.createImagicBatchFile(numclusters)
### execute IMAGIC batch file
clustertime0 = time.time()
proc = subprocess.Popen("chmod 775 "+str(batchfile), shell=True)
proc.wait()
apIMAGIC.executeImagicBatchFile(batchfile)
logfile = open(os.path.join(self.params['rundir'], "imagicMSAcluster_classes_"+str(numclusters)+".log"))
loglines = logfile.readlines()
for line in loglines:
if re.search("ERROR in program", line):
apDisplay.printError("ERROR IN IMAGIC SUBROUTINE, please check the logfile: imagicMSAcluster_classes_"\
+str(numclusters)+".log")
apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-clustertime0), "cyan")
### normalize
classfile = os.path.join(self.params['rundir'], self.params['classumfile'])
emancmd = "proc2d "+classfile+" "+classfile+".norm.hed norm"
while os.path.isfile(classfile+".norm.img"):
apStack.removeStack(alignstack+".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(classfile+".norm.hed", classfile)
os.rename(classfile+".norm.img", classfile[:-4]+".img")
### connect to database
self.insertClusterStack(numclusters)
def start(self):
### get analysis paramteres
self.analysisdata = appiondata.ApAlignAnalysisRunData.direct_query(self.params['analysisId'])
pixelsize = self.analysisdata['alignstack']['pixelsize']
boxsize = self.analysisdata['alignstack']['boxsize']
bin = self.analysisdata['imagicMSArun']['bin']
self.params['apix'] = float(pixelsize) * int(bin)
self.params['boxsize'] = int(boxsize) / int(bin)
self.params['num_particles'] = self.analysisdata['alignstack']['num_particles']
starttime=time.time()
print self.params
print "... stack pixel size: "+str(self.params['apix'])
print "... stack box size: "+str(self.params['boxsize'])
apDisplay.printColor("Running IMAGIC .batch file: See imagicMSAcluster log file(s) corresponding to # of classes for details", "cyan")
### insert run into database
self.insertClusterRun()
### split the cluster numbers
numclasslist = self.params['num_classes'].split(",")
for item in numclasslist:
numclusters = int(item)
apDisplay.printColor("\n==========================\nprocessing class averages for "
+str(numclusters)+" classes\n==========================\n", "green")
### create IMAGIC batch file
batchfile = self.createImagicBatchFile(numclusters)
### execute IMAGIC batch file
clustertime0 = time.time()
proc = subprocess.Popen("chmod 775 "+str(batchfile), shell=True)
proc.wait()
apIMAGIC.executeImagicBatchFile(batchfile)
logfile = open(os.path.join(self.params['rundir'], "imagicMSAcluster_classes_"+str(numclusters)+".log"))
loglines = logfile.readlines()
for line in loglines:
if re.search("ERROR in program", line):
apDisplay.printError("ERROR IN IMAGIC SUBROUTINE, please check the logfile: imagicMSAcluster_classes_"\
+str(numclusters)+".log")
apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-clustertime0), "cyan")
### normalize
classfile = os.path.join(self.params['rundir'], self.params['classumfile'])
emancmd = "proc2d "+classfile+" "+classfile+".norm.hed norm"
while os.path.isfile(classfile+".norm.img"):
apStack.removeStack(alignstack+".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(classfile+".norm.hed", classfile)
os.rename(classfile+".norm.img", classfile[:-4]+".img")
### connect to database
self.insertClusterStack(numclusters)
def unpackResults(self):
""" untar results, if this hasn't been done yet """
if os.path.exists(os.path.join(self.params["rundir"], "recon_results.tar.gz")):
apParam.runCmd("tar -xvzf recon_results.tar.gz", "SHELL")
apParam.runCmd("rm recon_results.tar.gz", "SHELL")
if os.path.exists(os.path.join(self.params["rundir"], "volumes.tar.gz")):
apParam.runCmd("tar -xvzf volumes.tar.gz", "SHELL")
apParam.runCmd("rm volumes.tar.gz", "SHELL")
def _boxParticlesFromImage(self, imgdata, parttree, imgstackfile):
'''
Box Particles From the manipulated full size image file on disk
'''
### make corrected integrated frame image
imgpath = self.getOriginalImagePath(imgdata)
t0 = time.time()
if self.params['phaseflipped'] is True:
if self.params['fliptype'] == 'emanimage':
### ctf correct whole image using EMAN
imgpath = self.phaseFlipWholeImage(imgpath, imgdata)
elif self.params['fliptype'] == "spiderimage":
imgpath = self.phaseFlipSpider(imgpath, imgdata)
elif self.params['fliptype'][:9] == "ace2image":
### ctf correct whole image using Ace 2
imgpath = self.phaseFlipAceTwo(imgpath, imgdata)
self.ctftimes.append(time.time()-t0)
if imgpath is None:
return None, None, None
### run apBoxer
apDisplay.printMsg("boxing "+str(len(parttree))+" particles into temp file: "+imgstackfile)
### method to align helices
t0 = time.time()
if self.params['rotate'] is True:
apBoxer.boxerRotate(imgpath, parttree, imgstackfile, self.boxsize)
if self.params['finealign'] is True:
from appionlib import apXmipp
apXmipp.breakupStackIntoSingleFiles(imgstackfile, filetype="mrc")
rotcmd = "s_finealign %s %i" %(self.params['rundir'], self.boxsize)
apParam.runCmd(rotcmd, "HIP", verbose=True)
# read in text file containing refined angles
anglepath = os.path.join(self.params['rundir'], 'angles.out')
f = open(anglepath, 'r')
angles = f.readlines()
# loop through parttree and add refined angle to rough angle
for i in range(len(parttree)):
partdict = parttree[i]
fineangle = float(angles[i])
newangle = float(partdict['angle'])-fineangle
partdict['angle'] = newangle
# rerun apBoxer.boxerRotate with the new parttree containing final angles
apBoxer.boxerRotate(imgpath, parttree, imgstackfile, self.boxsize, pixlimit=self.params['pixlimit'])
else:
apBoxer.boxer(imgpath, parttree, imgstackfile, self.boxsize, pixlimit=self.params['pixlimit'])
self.batchboxertimes.append(time.time()-t0)
def shift_images(self, filename):
### random shifts and rotations to a stack
shiftstackname = filename[:-4]+"_rand.hed"
apFile.removeStack(shiftstackname)
# shiftfile = os.path.join(self.params['rundir'], "shift_rotate.lst")
shiftfile = os.path.join(self.params['rundir'], "shift.lst")
if os.path.isfile(shiftfile):
apFile.removeFile(shiftfile)
f = open(shiftfile, "a")
# apDisplay.printMsg("Now randomly shifting and rotating particles")
apDisplay.printMsg("Now randomly shifting particles")
for i in range(self.params['projcount']):
# if self.params['rotang'] != 0:
# randrot = random.uniform(-1*self.params['rotang'], self.params['rotang'])
randx = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad'])
randy = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad'])
if self.params['flip'] is True:
flip = random.choice([0,1])
else:
flip = 0
# if self.params['rotang'] != 0:
# emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" rot="+str(randrot)+" trans="+str(randx)+","+str(randy)
# else:
# emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy)
emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy)
if flip == 0:
if self.params['pad'] is False:
emancmd = emancmd+" clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm"
else:
emancmd = emancmd+" edgenorm"
else:
if self.params['pad'] is False:
emancmd = emancmd+" flip clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm"
else:
emancmd = emancmd+" flip edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=False)
# if self.params['rotang'] != 0:
# f.write("%.3f,"%(randrot))
# else:
# f.write(",")
f.write("%.3f,"%(randx))
f.write("%.3f,"%(randy))
f.write(str(flip)+"\n")
f.close()
return shiftstackname
def shift_images(self, filename):
### random shifts and rotations to a stack
shiftstackname = filename[:-4]+"_rand.hed"
apFile.removeStack(shiftstackname)
# shiftfile = os.path.join(self.params['rundir'], "shift_rotate.lst")
shiftfile = os.path.join(self.params['rundir'], "shift.lst")
if os.path.isfile(shiftfile):
apFile.removeFile(shiftfile)
f = open(shiftfile, "a")
# apDisplay.printMsg("Now randomly shifting and rotating particles")
apDisplay.printMsg("Now randomly shifting particles")
for i in range(self.params['projcount']):
# if self.params['rotang'] != 0:
# randrot = random.uniform(-1*self.params['rotang'], self.params['rotang'])
randx = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad'])
randy = random.uniform(-1*self.params['shiftrad'], self.params['shiftrad'])
if self.params['flip'] is True:
flip = random.choice([0,1])
else:
flip = 0
# if self.params['rotang'] != 0:
# emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" rot="+str(randrot)+" trans="+str(randx)+","+str(randy)
# else:
# emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy)
emancmd = "proc2d "+filename+" "+shiftstackname+" first="+str(i)+" last="+str(i)+" trans="+str(randx)+","+str(randy)
if flip == 0:
if self.params['pad'] is False:
emancmd = emancmd+" clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm"
else:
emancmd = emancmd+" edgenorm"
else:
if self.params['pad'] is False:
emancmd = emancmd+" flip clip="+str(self.params['box'])+","+str(self.params['box'])+" edgenorm"
else:
emancmd = emancmd+" flip edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=False)
# if self.params['rotang'] != 0:
# f.write("%.3f,"%(randrot))
# else:
# f.write(",")
f.write("%.3f,"%(randx))
f.write("%.3f,"%(randy))
f.write(str(flip)+"\n")
f.close()
return shiftstackname
def unpackResults(self):
''' untar results, if this hasn't been done yet '''
if os.path.exists(
os.path.join(self.params['rundir'], "recon_results.tar.gz")):
apParam.runCmd("tar -xvzf recon_results.tar.gz", "SHELL")
apParam.runCmd("rm recon_results.tar.gz", "SHELL")
if os.path.exists(os.path.join(self.params['rundir'],
"volumes.tar.gz")):
apParam.runCmd("tar -xvzf volumes.tar.gz", "SHELL")
apParam.runCmd("rm volumes.tar.gz", "SHELL")
def post_process_stack(self, filename, pad, invert):
### pad out and/or invert projections
if self.params['invert'] is True or pad is True:
emancmd = "proc2d %s %s.process.hed " % (filename, filename)
if self.params['invert'] is True:
emancmd+="invert "
if pad is True:
emancmd+="clip=%d " % (self.params['box']*self.params['padF'])
apParam.runCmd(emancmd, "EMAN")
shutil.move("%s.process.hed" % filename, "%s.hed" % filename[:-4])
shutil.move("%s.process.img" % filename, "%s.img" % filename[:-4])
# self.params['projcount'] = int(split[-1][0]) ### last row, first value is last projection
# self.params['projcount'] = self.numProj(ang=self.params['projinc'], sym='c1') ### for some reasons did not work for prop=20
# numpart = apFile.numImagesInStack(stackfile)
return
def post_process_stack(self, filename, pad, invert):
### pad out and/or invert projections
if self.params['invert'] is True or pad is True:
emancmd = "proc2d %s %s.process.hed " % (filename, filename)
if self.params['invert'] is True:
emancmd+="invert "
if pad is True:
emancmd+="clip=%d " % (self.params['box']*self.params['padF'])
apParam.runCmd(emancmd, "EMAN")
shutil.move("%s.process.hed" % filename, "%s.hed" % filename[:-4])
shutil.move("%s.process.img" % filename, "%s.img" % filename[:-4])
# self.params['projcount'] = int(split[-1][0]) ### last row, first value is last projection
# self.params['projcount'] = self.numProj(ang=self.params['projinc'], sym='c1') ### for some reasons did not work for prop=20
# numpart = apFile.numImagesInStack(stackfile)
return
def start(self):
self.writeParams()
stackdata = apStack.getOnlyStackData(self.params["stackid"], msg=False)
stackpath = os.path.abspath(stackdata["path"]["path"])
stackfile = os.path.join(stackpath, "start.hed")
self.params["localstack"] = os.path.join(stackpath, self.timestamp + ".hed")
proccmd = "proc2d " + stackfile + " " + self.params["localstack"] + " apix=" + str(self.params["step"])
if self.params["bin"] > 1:
proccmd += " shrink=%d" % int(self.params["bin"])
proccmd += " last=" + str(self.params["numpart"] - 1)
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params["numpart"] != apFile.numImagesInStack(self.params["localstack"]):
apDisplay.printError("Missing particles in stack")
apXmipp.breakupStackIntoSingleFiles(self.params["localstack"], filetype="mrc")
if self.params["prehip"] == "yes":
if self.params["rise"] is not None:
self.writeLLBO1()
elif self.params["ll1"] is not None:
self.writeLLBO2()
elif self.params["rise"] is None and self.params["ll1"] is None:
apDisplay.printError(
"You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP"
)
apDisplay.printMsg("now running s_prehip to set up input files")
apDisplay.printMsg(
"Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables."
)
cmd = "s_prehip"
proc = subprocess.Popen(cmd)
proc.wait()
else:
apDisplay.printMsg("now running s_hip2: Phoelix programs")
cmd = "s_hip2"
proc = subprocess.Popen(cmd)
proc.wait()
# apParam.runCmd(cmd, package="Phoelix")
self.putFilesInStack()
self.insertHipRunData()
self.insertHipParamsData()
self.insertHipIterData()
self.insertHipParticleData()
def prealignClassAverages(rundir, avgs):
'''function to iteratively align class averages to each other prior to
input into angular reconstitution (optional) '''
imagicroot = checkImagicExecutablePath()
batchfile = os.path.join(rundir, "prealignClassAverages.batch")
f = open(batchfile, 'w')
f.write("#!/bin/csh -f\n")
f.write("setenv IMAGIC_BATCH 1\n")
f.write("cd " + rundir + "/\n")
### this is the actual alignment
f.write(
str(imagicroot) +
"/align/alirefs.e <<EOF >> prealignClassAverages.log\n")
f.write("ALL\n")
f.write("CCF\n")
f.write(str(os.path.basename(avgs)[:-4]) + "\n")
f.write("NO\n")
f.write("0.99\n")
f.write(str(os.path.basename(avgs)[:-4]) + "_aligned\n")
f.write("-999.\n")
f.write("0.2\n")
f.write("-180,180\n")
f.write("NO\n")
f.write("5\n")
f.write("NO\n")
f.write("EOF\n")
f.close()
avgs = avgs[:-4] + "_aligned.img"
proc = subprocess.Popen('chmod 755 ' + batchfile, shell=True)
proc.wait()
apParam.runCmd(batchfile, "IMAGIC")
return avgs
def applyBfactor(infile, fscfile, apix, mass=None, outfile=None):
embfactorfile = "embfactor64.exe"
embfactorexe = apParam.getExecPath("embfactor64.exe")
if embfactorexe is None:
apDisplay.printWarning("Could not find %s"%(embfactorfile))
return infile
if outfile is None:
outfile = os.path.splitext(infile)[0]+"-bfactor.mrc"
cmd = embfactorexe
cmd += " -FSC %s"%(fscfile)
cmd += " -sampling %.3f"%(apix)
### this option always failed for me -neil
#if mass is not None:
# cmd += " -molweight %d"%(mass*1000)
cmd += " %s"%(infile)
cmd += " %s"%(outfile)
apParam.runCmd(cmd, package="B-factor", verbose=True, showcmd=True)
if not apVolume.isValidVolume(outfile):
apDisplay.printWarning("B-factor correction failed %s"%(embfactorfile))
return infile
return outfile
def start(self):
self.writeParams()
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
stackpath = os.path.abspath(stackdata['path']['path'])
stackfile = os.path.join(stackpath, "start.hed")
self.params['localstack'] = os.path.join(stackpath, self.timestamp+".hed")
proccmd = "proc2d "+stackfile+" "+self.params['localstack']+" apix="+str(self.params['step'])
if self.params['bin'] > 1:
proccmd += " shrink=%d"%int(self.params['bin'])
proccmd += " last="+str(self.params['numpart']-1)
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
apXmipp.breakupStackIntoSingleFiles(self.params['localstack'], filetype="mrc")
if self.params['prehip'] == "yes":
if self.params['rise'] is not None:
self.writeLLBO1()
elif self.params['ll1'] is not None:
self.writeLLBO2()
elif self.params['rise'] is None and self.params['ll1'] is None:
apDisplay.printError("You must specify either rise and twist or (1,0) and (0,1) layer lines to run preHIP")
apDisplay.printMsg("now running s_prehip to set up input files")
apDisplay.printMsg("Make sure the layer line/bessel order assignment is correct. If it is not, you may need to adjust some of the input variables.")
cmd = "s_prehip"
proc = subprocess.Popen(cmd)
proc.wait()
else:
apDisplay.printMsg("now running s_hip2: Phoelix programs")
cmd = "s_hip2"
proc = subprocess.Popen(cmd)
proc.wait()
#apParam.runCmd(cmd, package="Phoelix")
self.putFilesInStack()
self.insertHipRunData()
self.insertHipParamsData()
self.insertHipIterData()
self.insertHipParticleData()
def scaleTemplates(self): reffile = os.path.join(self.params['rundir'], "references.hed") if self.params['apix'] != self.templatestack['apix']: scalefactor = float(self.templatestack['apix']) / self.params['apix'] templates = apImagicFile.readImagic(reffile) scaledtemplates = [] for templatearray in templates['images']: newarray = apTemplate.scaleTemplate(templatearray, scalefactor) scaledtemplates.append(newarray) apImagicFile.writeImagic(scaledtemplates, reffile) refbox = apFile.getBoxSize(reffile)[0] stbox = self.params['boxsize'] ### now clip the references to get identical boxsizes if stbox != refbox: while os.path.isfile(reffile+".new.img"): apFile.removeStack(reffile+".new.img") emancmd = "proc2d "+reffile+" "+reffile+".new.hed clip="+str(stbox)+" edgenorm" apParam.runCmd(emancmd, "EMAN") os.rename(reffile+".new.hed", reffile) os.rename(reffile+".new.img", reffile[:-4]+".img") return
def prealignClassAverages(rundir, avgs):
'''function to iteratively align class averages to each other prior to
input into angular reconstitution (optional) '''
imagicroot = checkImagicExecutablePath()
batchfile = os.path.join(rundir, "prealignClassAverages.batch")
f = open(batchfile, 'w')
f.write("#!/bin/csh -f\n")
f.write("setenv IMAGIC_BATCH 1\n")
f.write("cd "+rundir+"/\n")
### this is the actual alignment
f.write(str(imagicroot)+"/align/alirefs.e <<EOF >> prealignClassAverages.log\n")
f.write("ALL\n")
f.write("CCF\n")
f.write(str(os.path.basename(avgs)[:-4])+"\n")
f.write("NO\n")
f.write("0.99\n")
f.write(str(os.path.basename(avgs)[:-4])+"_aligned\n")
f.write("-999.\n")
f.write("0.2\n")
f.write("-180,180\n")
f.write("NO\n")
f.write("5\n")
f.write("NO\n")
f.write("EOF\n")
f.close()
avgs = avgs[:-4]+"_aligned.img"
proc = subprocess.Popen('chmod 755 '+batchfile, shell=True)
proc.wait()
apParam.runCmd(batchfile, "IMAGIC")
return avgs
def applyBfactor(infile, fscfile, apix, mass=None, outfile=None):
embfactorfile = "embfactor64.exe"
embfactorexe = apParam.getExecPath("embfactor64.exe")
if embfactorexe is None:
apDisplay.printWarning("Could not find %s" % (embfactorfile))
return infile
if outfile is None:
outfile = os.path.splitext(infile)[0] + "-bfactor.mrc"
cmd = embfactorexe
cmd += " -FSC %s" % (fscfile)
cmd += " -sampling %.3f" % (apix)
### this option always failed for me -neil
#if mass is not None:
# cmd += " -molweight %d"%(mass*1000)
cmd += " %s" % (infile)
cmd += " %s" % (outfile)
apParam.runCmd(cmd, package="B-factor", verbose=True, showcmd=True)
if not apVolume.isValidVolume(outfile):
apDisplay.printWarning("B-factor correction failed %s" %
(embfactorfile))
return infile
return outfile
def runRelionPreprocess(self, newstackroot):
'''
1. Use stackIntoPicks.py to extract the particle locations from the selected stack.
2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run.
3. Run relion_preprocess with rescale and norm. Outputs .mrcs file.
Neil: most of these steps could be done more generally
'''
apDisplay.printWarning("Making a new stack from original images")
# Build the stackIntoPicks command
apDisplay.printMsg('Extracting the particle locations from your selected stack.')
newstackrunname = self.params['runname']+"_particles"
newstackrundir = self.params['rundir']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
sessionid = int(self.params['expid'])
cmd = "stackIntoPicks.py "
cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s "%
(stackid,projectid,newstackrunname,newstackrundir))
cmd += (" --commit --expId=%d --jobtype=stackintopicks "%
(sessionid))
# Run the command
logfilepath = os.path.join(newstackrundir,'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Build the makestack2 command
'''
This is the command we want to make a stack from a stackIntoPicks run
makestack2.py
--single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2
--description="made from stackrun1 using stackintopicks"
--runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66
--commit --preset=en --projectid=303 --session=zz07jul25b
--no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2
--ppn=1 --nodes=1 --walltime=240 --jobid=2016
'''
apDisplay.printMsg('Using selected stack particle locations to make a Relion ready stack....')
# Get the ID of the stackIntoPicks run we just created
#apParticle.getSelectionIdFromName(runname, sessionname)
runq = appiondata.ApSelectionRunData()
runq['name'] = newstackrunname
runq['session'] = leginondata.SessionData.direct_query(self.params['expid'])
rundatas = runq.query(results=1)
print rundatas
if rundatas:
selectionid = rundatas[0].dbid
else:
apDisplay.printError("Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n")
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(self.params['last'],totalpart)
stackpathname = os.path.basename( self.originalStackData.path )
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir,'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText( 'lowpass', ( self.params['lowpass'], self.originalStackData.lowpass ), False)
highpasstext = setArgText( 'highpass', ( self.params['highpass'], self.originalStackData.highpass ), True)
partlimittext = setArgText('partlimit',(numpart,),False)
xmipp_normtext = setArgText('xmipp-normalize', (self.params['xmipp-norm'],), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = "makestack2.py "
cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d "%
(os.path.basename(newstackimagicfile),selectionid,inverttext,
unbinnedboxsize,bin))
cmd += (" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "%
(stackpathname,stackid,projectid))
cmd += (" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d "%
(presetname,newstackrunname,newstackrundir,sessionname,sessionid))
cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 "
# Run the command
logfilepath = os.path.join(newstackrundir,'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd,logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Clean up
boxfiles = glob.glob("*.box")
for boxfile in boxfiles:
apFile.removeFile(boxfile)
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'],self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot+'.hed'
# Run Relion pre-process command
'''
Setup the follow relion preprocess command to normalize the new stack:
relion_preprocess \
--o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \
--operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed
'''
apDisplay.printMsg('Running Relion preprocessing to normalize your Relion Ready stack....')
bg_radius = math.floor((self.stack['boxsize'] / 2) - 1)
relioncmd = "relion_preprocess "
relioncmd += " --o particles --norm --bg_radius %d "%(bg_radius)
relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s "%(newstackimagicfile)
apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True)
self.stack['file'] = 'particles.mrcs'
def runAlignmentGPU(self, stackfile, templatestack,
origstackfile,
xysearch, xystep,
firstring=2, lastring=100,
dataext=".spi",
iternum=1, oldpartlist=None):
"""
inputs:
stack
template
search params
outputs:
aligned stack
rotation/shift params
"""
### setup
if dataext in templatestack:
templatestack = templatestack[:-4]
if dataext in stackfile:
stackfile = stackfile[:-4]
if dataext in origstackfile:
origstackfile = origstackfile[:-4]
t0 = time.time()
rundir = "alignments"
apParam.createDirectory(rundir)
### remove previous iterations
apFile.removeFile(rundir+"/paramdoc%02d%s" % (iternum, dataext))
docfile = rundir+("/paramdoc%02d" % (iternum))+dataext
### required for apsh like commands
#inputanglesfile = rundir+("/angles%02d" % (iternum))+dataext
cmd = "/home/ryan/alignment_gpu/bin/alignment_gpu "
cmd += "'%s' "%(stackfile+dataext)
cmd += "'%s' "%(templatestack+dataext)
cmd += "%d "%(xysearch)
cmd += "%d "%(xystep)
cmd += "%d "%(1)
cmd += "%d "%(firstring)
cmd += "%d "%(lastring)
cmd += "'%s' "%(docfile)
apParam.runCmd(cmd, package="Alignment_GPU", verbose=True, showcmd=True)
### convert spider rotation, shift data to python
picklefile = rundir+("/paramdoc%02d" % (iternum))+".pickle"
if oldpartlist is not None and iternum > 1:
apDisplay.printMsg("updating particle doc info")
partlist = alignment.updateRefBasedDocFile(oldpartlist, docfile, picklefile)
elif iternum == 1:
apDisplay.printMsg("reading initial particle doc info")
partlist = self.readRefBasedDocFile(docfile, picklefile)
else:
apDisplay.printError("reading (not updating) particle doc info on iteration "+str(iternum))
### write aligned stack -- with python loop
alignedstack = rundir+("/alignedstack%02d" % (iternum))
alignment.alignStack(origstackfile, alignedstack, partlist, dataext)
### average stack
emancmd = ( "proc2d "+alignedstack+dataext+" "
+rundir+("/avgimg%02d" % (iternum))+".mrc "
+" average")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
td1 = time.time()-t0
apDisplay.printMsg("completed alignment of particles in "+apDisplay.timeString(td1))
if len(partlist) < 1:
apDisplay.printError("Failed to find any particles")
return alignedstack+dataext, partlist
def runAlignmentGPU(self, stackfile, templatestack,
origstackfile,
xysearch, xystep,
firstring=2, lastring=100,
dataext=".spi",
iternum=1, oldpartlist=None):
"""
inputs:
stack
template
search params
outputs:
aligned stack
rotation/shift params
"""
### setup
if dataext in templatestack:
templatestack = templatestack[:-4]
if dataext in stackfile:
stackfile = stackfile[:-4]
if dataext in origstackfile:
origstackfile = origstackfile[:-4]
t0 = time.time()
rundir = "alignments"
apParam.createDirectory(rundir)
### remove previous iterations
apFile.removeFile(rundir+"/paramdoc%02d%s" % (iternum, dataext))
docfile = rundir+("/paramdoc%02d" % (iternum))+dataext
### required for apsh like commands
#inputanglesfile = rundir+("/angles%02d" % (iternum))+dataext
cmd = "/home/ryan/alignment_gpu/bin/alignment_gpu "
cmd += "'%s' "%(stackfile+dataext)
cmd += "'%s' "%(templatestack+dataext)
cmd += "%d "%(xysearch)
cmd += "%d "%(xystep)
cmd += "%d "%(1)
cmd += "%d "%(firstring)
cmd += "%d "%(lastring)
cmd += "'%s' "%(docfile)
apParam.runCmd(cmd, package="Alignment_GPU", verbose=True, showcmd=True)
### convert spider rotation, shift data to python
picklefile = rundir+("/paramdoc%02d" % (iternum))+".pickle"
if oldpartlist is not None and iternum > 1:
apDisplay.printMsg("updating particle doc info")
partlist = alignment.updateRefBasedDocFile(oldpartlist, docfile, picklefile)
elif iternum == 1:
apDisplay.printMsg("reading initial particle doc info")
partlist = self.readRefBasedDocFile(docfile, picklefile)
else:
apDisplay.printError("reading (not updating) particle doc info on iteration "+str(iternum))
### write aligned stack -- with python loop
alignedstack = rundir+("/alignedstack%02d" % (iternum))
alignment.alignStack(origstackfile, alignedstack, partlist, dataext)
### average stack
emancmd = ( "proc2d "+alignedstack+dataext+" "
+rundir+("/avgimg%02d" % (iternum))+".mrc "
+" average")
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
td1 = time.time()-t0
apDisplay.printMsg("completed alignment of particles in "+apDisplay.timeString(td1))
if len(partlist) < 1:
apDisplay.printError("Failed to find any particles")
return alignedstack+dataext, partlist
def compute_stack_of_class_averages_and_reprojections(self, iteration, reference_number):
''' takes Xmipp single files, doc and sel files associated with ML3D, creates a stack of class averages in the results directory '''
'''
### make projections, and put them back into resultspath
selfile = "ml3d_it%.6d_vol%.6d.sel" % (iteration, reference_number)
refvolume = "ml3d_it%.6d_vol%.6d.vol\n" % (iteration, reference_number)
docfile = "ml3d_it%.6d_vol%.6d.doc" % (iteration, reference_number)
apXmipp.compute_stack_of_class_averages_and_reprojections(self.ml3dpath, selfile, refvolume, docfile, \
self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration, reference_number, extract=True)
return
'''
os.chdir(self.ml3dpath)
### remove "RunML3D/" from selfile (created by ML3D program), then extract header information to docfile
selfile = "ml3d_it%.6d_vol%.6d.sel" % (iteration, reference_number)
f = open(selfile, "r")
lines = f.readlines()
newlines = [re.sub("RunML3D/", "", line) for line in lines]
f.close()
f = open(selfile, "w")
f.writelines(newlines)
f.close()
extractcmd = "xmipp_header_extract -i ml3d_it%.6d_vol%.6d.sel -o ml3d_it%.6d_vol%.6d.doc" \
% (iteration, reference_number, iteration, reference_number)
apParam.runCmd(extractcmd, "Xmipp")
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("ml3d_it%.6d_vol%.6d.vol\n" % (iteration, reference_number))
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" % (self.runparams['boxsize'], self.runparams['boxsize']))
f.write("ml3d_it%.6d_vol%.6d.doc rot tilt psi\n" % (iteration, reference_number))
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
docfile = "ml3d_it%.6d_vol%.6d.doc" % (iteration, reference_number)
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else: pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning("number of projections does not match number of classes for model %d, iteration %d")
stackarray = []
stackname = os.path.join(self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (self.params['timestamp'], iteration, reference_number))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(self.params['rundir'])
return
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackid'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(
self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
clipsize = int(self.clipsize) * self.params['bin']
if clipsize % 2 == 1:
clipsize += 1 ### making sure that clipped boxsize is even
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
clipsize, clipsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
if self.params['highpass'] is not None and self.params['highpass'] > 1:
proccmd += " hp=" + str(self.params['highpass'])
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
proccmd += " lp=" + str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
if self.params['numpart'] != apFile.numImagesInStack(
self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(
self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = (
" " + " -i " +
os.path.join(self.params['rundir'], self.partlistdocfile) +
" -codes " + str(self.params['numrefs']) + " -iter " +
str(self.params['maxiter']) + " -o " + os.path.join(
self.params['rundir'], "part" + self.params['timestamp']))
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun + " -np " + str(
self.params['nproc']) + " " + xmippexe + " " + xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd,
package="Xmipp",
verbose=True,
showcmd=True,
logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: " +
apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(
self.params, "cl2d-" + self.params['timestamp'] + "-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-" +
self.params['timestamp'] +
"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(
self.runparams['stackid']) * self.runparams['bin']
self.Nlevels = len(
glob.glob("part" + self.params['timestamp'] + "_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part" + self.params[
'timestamp'] + "_level_%02d_.hed" % (self.Nlevels - 1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels - 1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels - 1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part" + self.params['timestamp'] +
"_level_%02d_.hed" % level, classnum + 1,
partdict[classnum], len(partdict))
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### setup Xmipp command
aligntime = time.time()
xmippopts = (" -i "+os.path.join(self.params['rundir'], self.params['localstack'])
+" --nref "+str(self.params['numrefs'])
+" --iter "+str(self.params['maxiter'])
+" --odir "+str(self.params['rundir'])
+" --oroot "+ "part"+str(self.params['timestamp'])
+" --classifyAllImages"
)
if self.params['correlation']:
xmippopts += " --distance correlation"
if self.params['classical']:
xmippopts += " --classicalMultiref"
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath(self.execFile, die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp 3", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### post-processing
# Create a stack for the class averages at each level
Nlevels=glob.glob("level_*")
for level in Nlevels:
digits = level.split("_")[1]
apParam.runCmd("xmipp_image_convert -i "+level+"/part"+self.params['timestamp']+"*xmd -o part"
+self.params['timestamp']+"_level_"+digits+"_.hed", package="Xmipp 3", verbose=True)
if self.params['align']:
apParam.runCmd("xmipp_transform_geometry -i images.xmd -o %s_aligned.stk --apply_transform" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apParam.runCmd("xmipp_image_convert -i %s_aligned.xmd -o alignedStack.hed" % self.params['timestamp'], package="Xmipp 3", verbose=True)
apFile.removeFile("%s_aligned.xmd" % self.params['timestamp'])
apFile.removeFile("%s_aligned.stk" % self.params['timestamp'])
self.parseOutput()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
self.clearIntermediateFiles()
def start(self):
self.insertMaxLikeJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackid'])
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
self.estimateIterTime()
self.dumpParameters()
### process stack to local file
self.params['localstack'] = os.path.join(self.params['rundir'], self.timestamp+".hed")
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
a.setValue('append',False)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert') is True
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
# clip not yet implemented
# self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### stack is needed by uploadMaxlike.py
#apFile.removeStack(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -nref "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.timestamp)
+" -psi_step "+str(self.params['psistep'])
)
### fast mode
if self.params['fast'] is True:
xmippopts += " -fast "
if self.params['fastmode'] == "narrow":
xmippopts += " -C 1e-10 "
elif self.params['fastmode'] == "wide":
xmippopts += " -C 1e-17 "
### convergence criteria
if self.params['converge'] == "fast":
xmippopts += " -eps 5e-3 "
elif self.params['converge'] == "slow":
xmippopts += " -eps 5e-8 "
else:
xmippopts += " -eps 5e-5 "
### mirrors
if self.params['mirror'] is True:
xmippopts += " -mirror "
### normalization
if self.params['norm'] is True:
xmippopts += " -norm "
### use student's T distribution
if self.params['student'] is True:
xmippopts += " -student "
### write cluster job file
if self.params['cluster'] is True:
self.writeClusterJobFile()
### find number of processors
if self.params['nproc'] is None:
nproc = nproc = apParam.getNumProcessors()
else:
nproc = self.params['nproc']
mpirun = self.checkMPI()
self.estimateIterTime()
if nproc > 2 and mpirun is not None:
### use multi-processor
apDisplay.printColor("Using "+str(nproc)+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_ml_align2d", die=True)
mpiruncmd = mpirun+" -np "+str(nproc)+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True)
else:
### use single processor
xmippexe = apParam.getExecPath("xmipp_ml_align2d", die=True)
xmippcmd = xmippexe+" "+xmippopts
self.writeXmippLog(xmippcmd)
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
aligntime = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
### minor post-processing
self.createReferenceStack()
self.readyUploadFlag()
self.dumpParameters()
def runAmpcor():
spidercmd = "spider bat/spi @enhance_edit"
apParam.runCmd(spidercmd, package="SPIDER", verbose=False, showcmd=True)
apFile.removeFile('pwsc.bat')
apFile.removeFile('enhance_edit.bat')
return
def start(self):
### database entry parameters
package_table = 'ApXmippRefineIterData|xmippParams'
### set projection-matching path
self.projmatchpath = os.path.abspath(
os.path.join(self.params['rundir'], "recon",
self.runparams['package_params']['WorkingDir']))
# self.projmatchpath = os.path.abspath(os.path.join(self.params['rundir'], self.runparams['package_params']['WorkingDir']))
### check for variable root directories between file systems
apXmipp.checkSelOrDocFileRootDirectoryInDirectoryTree(
self.params['rundir'], self.runparams['remoterundir'],
self.runparams['rundir'])
### determine which iterations to upload
lastiter = self.findLastCompletedIteration()
uploadIterations = self.verifyUploadIterations(lastiter)
### upload each iteration
for iteration in uploadIterations:
apDisplay.printColor("uploading iteration %d" % iteration, "cyan")
### set package parameters, as they will appear in database entries
package_database_object = self.instantiateProjMatchParamsData(
iteration)
### move FSC file to results directory
oldfscfile = os.path.join(self.projmatchpath,
"Iter_%d" % iteration,
"Iter_%d_resolution.fsc" % iteration)
newfscfile = os.path.join(
self.resultspath, "recon_%s_it%.3d_vol001.fsc" %
(self.params['timestamp'], iteration))
if os.path.exists(oldfscfile):
shutil.copyfile(oldfscfile, newfscfile)
### create a stack of class averages and reprojections (optional)
self.compute_stack_of_class_averages_and_reprojections(iteration)
### create a text file with particle information
self.createParticleDataFile(iteration)
if not self.params['euleronly']:
### create mrc file of map for iteration and reference number
oldvol = os.path.join(self.projmatchpath,
"Iter_%d" % iteration,
"Iter_%d_reconstruction.vol" % iteration)
newvol = os.path.join(
self.resultspath, "recon_%s_it%.3d_vol001.mrc" %
(self.params['timestamp'], iteration))
mrccmd = "proc3d %s %s apix=%.3f" % (oldvol, newvol,
self.runparams['apix'])
apParam.runCmd(mrccmd, "EMAN")
### make chimera snapshot of volume
self.createChimeraVolumeSnapshot(newvol, iteration)
### instantiate database objects
self.insertRefinementRunData(iteration)
self.insertRefinementIterationData(iteration, package_table,
package_database_object)
### calculate Euler jumps
if self.runparams['numiter'] > 1:
self.calculateEulerJumpsAndGoodBadParticles(uploadIterations)
### query the database for the completed refinements BEFORE deleting any files ... returns a dictionary of lists
### e.g. {1: [5, 4, 3, 2, 1]} means 5 iters completed for refine 1
complete_refinements = self.verifyNumberOfCompletedRefinements(
multiModelRefinementRun=False)
if self.params['cleanup_files'] is True:
self.cleanupFiles(complete_refinements)
def compute_stack_of_class_averages_and_reprojections(self, iteration):
''' takes Xmipp single files, doc and sel files in projection-matching, creates a stack of class averages in the results directory '''
if bool(self.runparams['package_params']['CleanUpFiles']) is False:
os.chdir(
os.path.join(self.projmatchpath, "Iter_%d" % iteration,
"ProjMatchClasses"))
### make projections, and put them back into resultspath
selfile = "proj_match_classes.sel"
refvolume = "../Iter_%d_reconstruction.vol" % iteration
docfile = "proj_match_classes.doc"
# apXmipp.compute_stack_of_class_averages_and_reprojections(d, selfile, refvolume, docfile, \
# self.runparams['boxsize'], self.resultspath, self.params['timestamp'], iteration)
### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile
f = open(selfile, "r")
lines = f.readlines()
newlines = [
re.sub("ProjMatchClasses/", "", line) for line in lines
]
f.close()
f = open(selfile[:-4] + "_new.sel", "w")
f.writelines(newlines)
f.close()
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("%s\n" % refvolume)
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" %
(self.runparams['boxsize'], self.runparams['boxsize']))
f.write("%s rot tilt psi\n" % docfile)
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else:
pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning(
"number of projections does not match number of classes")
stackarray = []
stackname = os.path.join(
self.resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" %
(self.params['timestamp'], iteration, 1))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(self.params['rundir'])
else:
apDisplay.printWarning(
"all projection-matching files were cleaned up ... NOT creating class-average / re-projection stack"
)
return
def start(self):
# self.insertCL2DJob()
self.stack = {}
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
self.stack['part'] = apStack.getOneParticleFromStackId(self.params['stackid'])
if self.params['virtualdata'] is not None:
self.stack['file'] = self.params['virtualdata']['filename']
else:
self.stack['file'] = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".hed")
if os.path.isfile(self.params['localstack']):
apFile.removeStack(self.params['localstack'])
a = proc2dLib.RunProc2d()
a.setValue('infile',self.stack['file'])
a.setValue('outfile',self.params['localstack'])
a.setValue('apix',self.stack['apix'])
a.setValue('bin',self.params['bin'])
a.setValue('last',self.params['numpart']-1)
if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
a.setValue('lowpass',self.params['lowpass'])
if self.params['highpass'] is not None and self.params['highpass'] > 1:
a.setValue('highpass',self.params['highpass'])
if self.params['invert'] is True:
a.setValue('invert',True)
# clip not yet implemented
# if self.params['clipsize'] is not None:
# clipsize = int(self.clipsize)*self.params['bin']
# if clipsize % 2 == 1:
# clipsize += 1 ### making sure that clipped boxsize is even
# a.setValue('clip',clipsize)
if self.params['virtualdata'] is not None:
vparts = self.params['virtualdata']['particles']
plist = [int(p['particleNumber'])-1 for p in vparts]
a.setValue('list',plist)
#run proc2d
a.run()
if self.params['numpart'] != apFile.numImagesInStack(self.params['localstack']):
apDisplay.printError("Missing particles in stack")
### convert stack into single spider files
self.partlistdocfile = apXmipp.breakupStackIntoSingleFiles(self.params['localstack'])
### setup Xmipp command
aligntime = time.time()
xmippopts = ( " "
+" -i "+os.path.join(self.params['rundir'], self.partlistdocfile)
+" -codes "+str(self.params['numrefs'])
+" -iter "+str(self.params['maxiter'])
+" -o "+os.path.join(self.params['rundir'], "part"+self.params['timestamp'])
)
if self.params['fast']:
xmippopts += " -fast "
if self.params['correlation']:
xmippopts += " -useCorrelation "
if self.params['classical']:
xmippopts += " -classicalMultiref "
if self.params['align']:
xmippopts += " -alignImages "
### use multi-processor command
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
xmippexe = apParam.getExecPath("xmipp_mpi_class_averages", die=True)
mpiruncmd = self.mpirun+" -np "+str(self.params['nproc'])+" "+xmippexe+" "+xmippopts
self.writeXmippLog(mpiruncmd)
apParam.runCmd(mpiruncmd, package="Xmipp", verbose=True, showcmd=True, logfile="xmipp.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(self.params['runtime']))
### minor post-processing
self.createReferenceStack()
self.parseOutput()
self.clearIntermediateFiles()
# self.readyUploadFlag()
apParam.dumpParameters(self.params, "cl2d-"+self.params['timestamp']+"-params.pickle")
### upload results ... this used to be two separate operations, I'm combining into one
self.runparams = apParam.readRunParameters("cl2d-"+self.params['timestamp']+"-params.pickle")
self.apix = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
self.Nlevels=len(glob.glob("part"+self.params['timestamp']+"_level_??_.hed"))
### create average of aligned stacks & insert aligned stack info
lastLevelStack = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
apStack.averageStack(lastLevelStack)
self.boxsize = apFile.getBoxSize(lastLevelStack)[0]
self.insertCL2DParamsIntoDatabase()
if self.runparams['align'] is True:
self.insertAlignStackRunIntoDatabase("alignedStack.hed")
self.calcResolution(self.Nlevels-1)
self.insertAlignParticlesIntoDatabase(level=self.Nlevels-1)
### loop over each class average stack & insert as clustering stacks
self.insertClusterRunIntoDatabase()
for level in range(self.Nlevels):
### NOTE: RESOLUTION CAN ONLY BE CALCULATED IF ALIGNED STACK EXISTS TO EXTRACT / READ THE PARTICLES
if self.params['align'] is True:
self.calcResolution(level)
partdict = self.getClassificationAtLevel(level)
for classnum in partdict:
self.insertClusterStackIntoDatabase(
"part"+self.params['timestamp']+"_level_%02d_.hed"%level,
classnum+1, partdict[classnum], len(partdict))
def createProjectionsEmanProp(self, pad=False, invert=False):
### first get rid of projection artifacts from insufficient padding
if self.params['threedfile'] is not None:
origfile = self.params['threedfile']
elif self.params['modelid'] is not None:
self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid'])
origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name'])
if self.params['apix'] is None:
self.params['apix'] = self.modelparams['pixelsize']
if self.params['box'] is None:
self.params['box'] = self.modelparams['boxsize']
clipped = os.path.join(self.params['rundir'], "clipped.mrc")
newsize = self.params['box'] * 1.5
emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
### project resized file
eulerfile = os.path.join(self.params['rundir'], "eulers.lst")
tempeulerfile = os.path.join(self.params['rundir'], "tmpeulers.lst")
filename = os.path.join(self.params['rundir'], "proj.img")
temp = os.path.join(self.params['rundir'], "temp.img")
emancmd = "project3d "+clipped+" out="+temp+" sym=c1 prop="+str(self.params['projinc'])+" > "+tempeulerfile
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
### update projection count & write eulerlist file
f = open(tempeulerfile, "r")
lines = f.readlines()
f.close()
apFile.removeStack(temp)
apFile.removeFile(tempeulerfile)
strip = [line.strip() for line in lines[2:]] ### first two lines are EMAN commands
iters = int(math.ceil(float(self.params['projcount']) / len(strip)))
self.params['projcount'] = iters * len(strip)
while os.path.isfile(eulerfile):
apFile.removeFile(eulerfile)
f = open(eulerfile, "a")
n = 1
for i in range(iters):
for j in range(len(strip)):
split = strip[j].split()
f.write(str(n)+"\t")
f.write(str(split[1])+"\t")
f.write(str(split[2])+"\t")
if self.params['rotang'] != 0:
randrot = random.uniform(-1*self.params['rotang'], self.params['rotang'])
f.write(str(randrot)+"\t\n")
else:
f.write(str(split[3])+"\t\n")
n += 1
f.close()
### create actual projections
apFile.removeStack(filename)
t0 = time.time()
emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile
# emancmd = "project3d "+clipped+" out="+filename+" sym=c1 prop="+str(self.params['projinc'])+" > "+tempeulerfile
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount'])))
'''
### update projection count & write eulerlist file
f = open(tempeulerfile, "r")
lines = f.readlines()
f.close()
apFile.removeStack(temp)
apFile.removeFile(tempeulerfile)
strip = [line.strip() for line in lines[2:]] ### first two lines are EMAN commands
iters = int(math.ceil(float(self.params['projcount']) / len(strip)))
self.params['projcount'] = iters * len(strip)
while os.path.isfile(eulerfile):
apFile.removeFile(eulerfile)
f = open(eulerfile, "a")
n = 1
for i in range(iters):
for j in range(len(strip)):
split = strip[j].split()
f.write(str(n)+"\t")
f.write(str(split[1])+"\t")
f.write(str(split[2])+"\t")
f.write(str(split[3])+"\t\n")
n += 1
f.close()
'''
### post-process stack
self.post_process_stack(filename, pad, invert)
return filename
def createProjectionsEmanProp(self, pad=False, invert=False):
### first get rid of projection artifacts from insufficient padding
if self.params['threedfile'] is not None:
origfile = self.params['threedfile']
elif self.params['modelid'] is not None:
self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid'])
origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name'])
if self.params['apix'] is None:
self.params['apix'] = self.modelparams['pixelsize']
if self.params['box'] is None:
self.params['box'] = self.modelparams['boxsize']
clipped = os.path.join(self.params['rundir'], "clipped.mrc")
newsize = self.params['box'] * 1.5
emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
### project resized file
eulerfile = os.path.join(self.params['rundir'], "eulers.lst")
tempeulerfile = os.path.join(self.params['rundir'], "tmpeulers.lst")
filename = os.path.join(self.params['rundir'], "proj.img")
temp = os.path.join(self.params['rundir'], "temp.img")
emancmd = "project3d "+clipped+" out="+temp+" sym=c1 prop="+str(self.params['projinc'])+" > "+tempeulerfile
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
### update projection count & write eulerlist file
f = open(tempeulerfile, "r")
lines = f.readlines()
f.close()
apFile.removeStack(temp)
apFile.removeFile(tempeulerfile)
strip = [line.strip() for line in lines[2:]] ### first two lines are EMAN commands
iters = int(math.ceil(float(self.params['projcount']) / len(strip)))
self.params['projcount'] = iters * len(strip)
while os.path.isfile(eulerfile):
apFile.removeFile(eulerfile)
f = open(eulerfile, "a")
n = 1
for i in range(iters):
for j in range(len(strip)):
split = strip[j].split()
f.write(str(n)+"\t")
f.write(str(split[1])+"\t")
f.write(str(split[2])+"\t")
if self.params['rotang'] != 0:
randrot = random.uniform(-1*self.params['rotang'], self.params['rotang'])
f.write(str(randrot)+"\t\n")
else:
f.write(str(split[3])+"\t\n")
n += 1
f.close()
### create actual projections
apFile.removeStack(filename)
t0 = time.time()
emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile
# emancmd = "project3d "+clipped+" out="+filename+" sym=c1 prop="+str(self.params['projinc'])+" > "+tempeulerfile
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount'])))
'''
### update projection count & write eulerlist file
f = open(tempeulerfile, "r")
lines = f.readlines()
f.close()
apFile.removeStack(temp)
apFile.removeFile(tempeulerfile)
strip = [line.strip() for line in lines[2:]] ### first two lines are EMAN commands
iters = int(math.ceil(float(self.params['projcount']) / len(strip)))
self.params['projcount'] = iters * len(strip)
while os.path.isfile(eulerfile):
apFile.removeFile(eulerfile)
f = open(eulerfile, "a")
n = 1
for i in range(iters):
for j in range(len(strip)):
split = strip[j].split()
f.write(str(n)+"\t")
f.write(str(split[1])+"\t")
f.write(str(split[2])+"\t")
f.write(str(split[3])+"\t\n")
n += 1
f.close()
'''
### post-process stack
self.post_process_stack(filename, pad, invert)
return filename
def runRelionPreprocess(self, newstackroot):
'''
1. Use stackIntoPicks.py to extract the particle locations from the selected stack.
2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run.
3. Run relion_preprocess with rescale and norm. Outputs .mrcs file.
Neil: most of these steps could be done more generally
'''
apDisplay.printWarning("Making a new stack from original images")
# Build the stackIntoPicks command
apDisplay.printMsg(
'Extracting the particle locations from your selected stack.')
newstackrunname = self.params['runname'] + "_particles"
newstackrundir = self.params['rundir']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
sessionid = int(self.params['expid'])
cmd = "stackIntoPicks.py "
cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s " %
(stackid, projectid, newstackrunname, newstackrundir))
cmd += (" --commit --expId=%d --jobtype=stackintopicks " % (sessionid))
# Run the command
logfilepath = os.path.join(newstackrundir, 'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Build the makestack2 command
'''
This is the command we want to make a stack from a stackIntoPicks run
makestack2.py
--single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2
--description="made from stackrun1 using stackintopicks"
--runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66
--commit --preset=en --projectid=303 --session=zz07jul25b
--no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2
--ppn=1 --nodes=1 --walltime=240 --jobid=2016
'''
apDisplay.printMsg(
'Using selected stack particle locations to make a Relion ready stack....'
)
# Get the ID of the stackIntoPicks run we just created
#apParticle.getSelectionIdFromName(runname, sessionname)
runq = appiondata.ApSelectionRunData()
runq['name'] = newstackrunname
runq['session'] = leginondata.SessionData.direct_query(
self.params['expid'])
rundatas = runq.query(results=1)
print rundatas
if rundatas:
selectionid = rundatas[0].dbid
else:
apDisplay.printError(
"Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n"
)
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(
self.params['last'], totalpart)
stackpathname = os.path.basename(self.originalStackData.path)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir, 'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText(
'lowpass',
(self.params['lowpass'], self.originalStackData.lowpass), False)
highpasstext = setArgText(
'highpass',
(self.params['highpass'], self.originalStackData.highpass), True)
partlimittext = setArgText('partlimit', (numpart, ), False)
xmipp_normtext = setArgText('xmipp-normalize',
(self.params['xmipp-norm'], ), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = "makestack2.py "
cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d " %
(os.path.basename(newstackimagicfile), selectionid, inverttext,
unbinnedboxsize, bin))
cmd += (
" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "
% (stackpathname, stackid, projectid))
cmd += (
" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d " %
(presetname, newstackrunname, newstackrundir, sessionname,
sessionid))
cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 "
# Run the command
logfilepath = os.path.join(newstackrundir, 'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Clean up
boxfiles = glob.glob("*.box")
for boxfile in boxfiles:
apFile.removeFile(boxfile)
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot + '.hed'
# Run Relion pre-process command
'''
Setup the follow relion preprocess command to normalize the new stack:
relion_preprocess \
--o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \
--operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed
'''
apDisplay.printMsg(
'Running Relion preprocessing to normalize your Relion Ready stack....'
)
bg_radius = math.floor((self.stack['boxsize'] / 2) - 1)
relioncmd = "relion_preprocess "
relioncmd += " --o particles --norm --bg_radius %d " % (bg_radius)
relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s " % (
newstackimagicfile)
apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True)
self.stack['file'] = 'particles.mrcs'
def runAmpcor():
spidercmd = "spider bat/spi @enhance_edit"
apParam.runCmd(spidercmd, package="SPIDER", verbose=False, showcmd=True)
apFile.removeFile('pwsc.bat')
apFile.removeFile('enhance_edit.bat')
return
def compute_stack_of_class_averages_and_reprojections(dir, selfile, refvolume, docfile, boxsize, resultspath, timestamp, iteration, reference_number=1, extract=False):
''' takes Xmipp single files, doc and sel files in routine, creates a stack of class averages in the results directory '''
workingdir = os.getcwd()
os.chdir(dir)
if dir.endswith("/"):
dir = dir[:-1]
head, tail = os.path.split(dir)
### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile
f = open(selfile, "r")
lines = f.readlines()
newlines = [re.sub(str(tail)+"/", "", line) for line in lines]
f.close()
f = open(selfile[:-4]+"_new.sel", "w")
f.writelines(newlines)
f.close()
if extract is True:
extractcmd = "xmipp_header_extract -i %s.sel -o %s.doc" % (selfile[:-4], docfile[:-4])
apParam.runCmd(extractcmd, "Xmipp")
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("%s\n" % refvolume)
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" % (boxsize, boxsize))
f.write("%s rot tilt psi\n" % docfile)
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else: pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning("number of projections does not match number of classes")
stackarray = []
stackname = os.path.join(resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" % (timestamp, iteration, reference_number))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(workingdir)
return
def start(self):
'''
OptiMod script
'''
############### define short-hand parameter names ############
refine = True
rundir = self.params['rundir']
box = self.params['boxsize']
rbox = self.params['refineboxsize']
apix = self.params['apix']
rapix = self.params['refineapix']
npart = self.params['numpart']
nvol = self.params['num_volumes']
# sym = self.params['sym']
sym = "c1"
psym = self.params['presumed_sym']
first = self.params['firstimage']
nws = self.params['non_weighted_sequence']
threes = self.params['threes']
lmask = self.params['linmask']
asqfilt = self.params['asqfilt']
anginc = self.params['ang_inc']
keep_ordered = self.params['keep_ordered_num']
hamwin = self.params['ham_win']
useEMAN1 = self.params['useEMAN1']
ipv = self.params['images_per_volume']
lp = self.params['3d_lpfilt']
nref = self.params['nref']
PCA = self.params['PCA']
neigens = self.params['numeigens']
recalc = self.params['recalculate']
preftype = self.params['preftype']
mrad = self.params['mask_radius']
irad = self.params['inner_radius']
orad = self.params['outer_radius']
tnproc = self.params['threadnproc']
nproc = self.params['nproc']
oldavgs = self.params['oldavgs']
avgs = os.path.join(self.params['rundir'], self.params['avgs'])
start = self.params['start_at']
############# copy to working directory ############
if not os.path.isfile(os.path.join(avgs)[:-4]+".hed"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".hed"), avgs[:-4]+".hed")
if not os.path.isfile(os.path.join(avgs)[:-4]+".img"):
shutil.copyfile(os.path.join(oldavgs[:-4]+".img"), avgs[:-4]+".img")
if self.params['ravgs'] is not None:
ravgs = os.path.join(self.params['rundir'], self.params['ravgs'])
if not os.path.isfile(self.params['ravgs'][:-4]+".hed"):
shutil.copy(self.params['oldravgs'][:-4]+".hed",
os.path.join(self.params['ravgs'][:-4]+".hed"))
if not os.path.isfile(self.params['ravgs'][:-4]+".img"):
shutil.copy(self.params['oldravgs'][:-4]+".img",
os.path.join(self.params['ravgs'][:-4]+".img"))
### Euler jumper assessment does not make sense when refine images are different
ejassess = False
else:
ravgs = avgs
ejassess = True
########### scale & prealign class averages, if specified #########
if self.params['scale'] is True:
emancmd = "proc2d %s %s_scaled.img scale=%.3f clip=%i,%i edgenorm" \
% (avgs, avgs[:-4], self.scalefactor, 64, 64)
avgs = avgs[:-4]+"_scaled.img"
if start == "none":
while os.path.isfile(avgs):
apFile.removeStack(avgs)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printColor("skipping stack scaling", "cyan")
if self.imagicroot is not None and useEMAN1 is False:
apIMAGIC.takeoverHeaders(avgs, npart, box)
if self.imagicroot is not None:
if self.params['prealign'] is True:
if start == "none":
avgs = apIMAGIC.prealignClassAverages(rundir, avgs)
apIMAGIC.checkLogFileForErrors(
os.path.join(rundir, "prealignClassAverages.log"))
### select between EMAN1 and IMAGIC for common lines
if self.imagicroot is not None and useEMAN1 is False:
commonlinesdir = os.path.join(rundir, "angular_reconstitution")
else:
commonlinesdir = os.path.join(rundir, "cross_common_lines")
### make links & calculate CCC matrix for 2D averages
apDisplay.printColor("Calculating similarity matrix", "cyan")
ccc_matrix = apCommonLines.calculate_ccc_matrix_2d(avgs)
clsavgs = os.path.split(avgs)[1][:-4]
if not os.path.isdir(commonlinesdir):
os.mkdir(commonlinesdir)
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".hed")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".hed"))
os.symlink(os.path.join(rundir, clsavgs+".hed"),
os.path.join(commonlinesdir, clsavgs+".hed"))
if os.path.islink(os.path.join(commonlinesdir, clsavgs+".img")):
os.system("rm -rf %s" % os.path.join(commonlinesdir, clsavgs+".img"))
os.symlink(os.path.join(rundir, clsavgs+".img"),
os.path.join(commonlinesdir, clsavgs+".img"))
################## create multiple 3d0s #####################
if start == "none":
if self.imagicroot is not None and useEMAN1 is False:
cmdlist = []
seqfile = open(os.path.join(rundir,
"sequences_for_angular_reconstitution.dat"), "w")
apDisplay.printColor("Running multiple raw volume calculations", "cyan")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
### create IMAGIC batch file for each model & append them to be threaded
batchfile = apCommonLines.imagic_batch_file(sequence, i+1, avgs, sym,
asqfilt, lmask, apix, box, anginc, keep_ordered, hamwin, lp,
threes=threes, do_not_remove=False)
proc = subprocess.Popen('chmod 755 '+batchfile, shell=True)
proc.wait()
cmdlist.append(batchfile)
os.chdir(rundir)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
### check for errors after execution
for i in range(nvol):
apIMAGIC.checkLogFileForErrors(
os.path.join(commonlinesdir, "3d"+str(i+1)+".log"))
else:
### use EMAN1 cross-common lines
cmdlist = []
seqfile = open(os.path.join(rundir, "sequences_for_cross_common_lines.dat"), "w")
for i in range(nvol):
sequence = apCommonLines.calculate_sequence_of_addition(avgs, npart,
ccc_matrix, first=first, normlist=False, non_weighted_sequence=nws)
apCommonLines.check_for_duplicates_in_sequence(sequence)
seqfile.write(str(sequence)+"\n")
vol_avgfile = os.path.join(commonlinesdir, "vol_%d_averages.hed" % (i+1))
vol_seqfile = os.path.join(commonlinesdir, "vol_%d_averages.txt" % (i+1))
vsf = open(vol_seqfile, "w")
for j in range(ipv):
vsf.write("%d\n" % sequence[j])
vsf.close()
tmpdir = os.path.join(commonlinesdir, "tmp%d" % (i+1))
fullcmd = "proc2d %s %s list=%s ; " % (avgs, vol_avgfile, vol_seqfile)
fullcmd+= "rm -rf %s ; mkdir %s ; cd %s ; " % (tmpdir, tmpdir, tmpdir)
fullcmd+= "startAny %s sym=c1 proc=1 rounds=2 mask=%d ; " \
% (vol_avgfile, mrad/apix)
fullcmd+= "mv threed.0a.mrc ../threed_%d.mrc ; " % (i+1)
fullcmd+= "mv CCL.hed ../CCL_%d.hed ; " % (i+1)
fullcmd+= "mv CCL.img ../CCL_%d.img ; " % (i+1)
cmdlist.append(fullcmd)
seqfile.close()
apThread.threadCommands(cmdlist, nproc=tnproc, pausetime=10)
for i in range(nvol):
shutil.rmtree(os.path.join(commonlinesdir, "tmp%d" % (i+1)))
else:
apDisplay.printColor("skipping common lines volume calculations", "cyan")
############### move 3-D models volume directory ################
### create volume directory
volumedir = os.path.join(rundir, "volumes")
if not os.path.isdir(volumedir):
os.mkdir(volumedir)
volumes = {}
cmds = []
if start == "none":
apDisplay.printColor("moving volumes for Xmipp 3-D Maximum Likelihood", "cyan")
for i in range(nvol):
if useEMAN1 is False:
volume1 = os.path.join(commonlinesdir, "3d%d_ordered%d_filt.vol" % (i+1,i+1))
else:
volume1 = os.path.join(commonlinesdir, "threed_%d.mrc" % (i+1))
volume2 = os.path.join(volumedir, "3d%d.vol" % (i+1))
if start == "none":
if useEMAN1 is False:
shutil.move(volume1, volume2)
else:
cmds.append("proc3d %s %s spidersingle" % (volume1, volume2))
volumes[(i+1)] = volume2
apThread.threadCommands(cmds, nproc=tnproc)
#################### align 3-D models ##################
if start == "3D_align" or start == "none":
# if start != "3D_assess" and start != "3D_refine":
### run Maximum Likelihood 3-D alignment & align resulting volumes
apDisplay.printColor("Running Xmipp maximum likelihood 3-D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.xmipp_max_like_3d_align(
volumes, nproc, tnproc, nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
apDisplay.printColor("Aligning volumes based on 3-D ML parameters", "cyan")
apCommonLines.align_volumes(alignparams, alignref, nvol, tnproc, apix)
else:
apDisplay.printColor("skipping 3D alignment", "cyan")
vol_doc_file, alignref = apCommonLines.findAlignmentParams(
os.path.join(rundir,"max_like_alignment"), nref)
alignparams = apCommonLines.read_vol_doc_file(vol_doc_file, nvol)
##################### Principal Component Analysis ##############
apDisplay.printColor("Calculating inter-volume similarity", "cyan")
aligned_volumes = {}
for i in range(nvol):
aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.vol" % (i+1))
# aligned_volumes[(i+1)] = os.path.join(rundir, "volumes", "3d%d.mrc" % (i+1))
if PCA is True:
simfile, sim_matrix = apCommonLines.runPrincipalComponentAnalysis(
aligned_volumes, nvol, neigens, box, apix, recalculate=recalc)
else:
simfile, sim_matrix = apCommonLines.calculate_ccc_matrix_3d(
aligned_volumes, nvol)
################ 3-D affinity propagation #############
### 3-D Affinity Propagation
apDisplay.printColor("Averaging volumes with Affinity Propagation", "cyan")
preffile = apCommonLines.set_preferences(sim_matrix, preftype, nvol)
classes = apCommonLines.run_affinity_propagation(aligned_volumes, simfile,
preffile, box, apix)
######### refine volumes using Xmipp projection matching #########
if start == "3D_align" or start == "3D_refine" or start == "none":
# if start != "3D_assess":
if not os.path.isdir("refinement"):
os.mkdir("refinement")
os.chdir("refinement")
apXmipp.breakupStackIntoSingleFiles(ravgs)
xmippcmd = "xmipp_normalize -i partlist.sel -method OldXmipp"
apParam.runCmd(xmippcmd, "Xmipp")
for i in classes.keys():
emancmd = "proc3d %s %s scale=%.3f clip=%d,%d,%d mask=%s spidersingle" \
% (os.path.join(rundir, "%d.mrc" % i), "%d.vol" % i, \
(1/self.scalefactor), rbox, rbox, rbox, (mrad / rapix))
apParam.runCmd(emancmd, "EMAN")
apCommonLines.refine_volume(rundir, ravgs, i, mrad, irad, orad, rapix, nproc)
emancmd = "proc3d %s %s apix=%.3f" \
% (os.path.join("refine_%d" % i, "3d%d_refined.vol" % i), \
os.path.join(rundir, "%d_r.mrc" % i), rapix)
apParam.runCmd(emancmd, "EMAN")
os.chdir(rundir)
else:
apDisplay.printColor("skipping 3D refinement", "cyan")
################# model evaluation ################
if start == "3D_align" or start == "3D_refine" or start == "3D_assess" or start == "none":
# if start == "none" or start=="3D_assess":
### final model assessment
try:
euler_array = apCommonLines.getEulerValuesForModels(alignparams,
rundir, nvol, npart, threes=threes)
except:
euler_array = None
ejassess = False
if refine is True:
apCommonLines.assess_3Dclass_quality2(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, rbox, rapix, ravgs, psym, npart, ejassess=ejassess)
else:
apCommonLines.assess_3Dclass_quality(rundir, aligned_volumes, sim_matrix,
classes, euler_array, box, apix, avgs, psym, npart, ejassess=ejassess)
apCommonLines.combineMetrics("final_model_stats.dat",
"final_model_stats_sorted_by_Rcrit.dat", **{"CCPR":(1,1)})
################## upload & cleanup ##################
### upload to database, if specified
self.upload()
### make chimera snapshots
if refine is True:
for i in classes.keys():
if self.params['mass'] is not None:
apChimera.filterAndChimera(
os.path.join(self.params['rundir'], "%d_r.mrc" % i),
res=self.params['3d_lpfilt'],
apix=self.params['refineapix'],
box=self.params['refineboxsize'],
chimtype="snapshot",
contour=2,
zoom=1,
sym="c1",
color="gold",
mass=self.params['mass']
)
### cleanup
snapshots = glob.glob("*.png")
mtlfiles = glob.glob("*.mtl")
objfiles = glob.glob("*.obj")
pyfiles = glob.glob("*mrc.py")
for file in mtlfiles:
os.remove(file)
for file in objfiles:
os.remove(file)
for file in pyfiles:
os.remove(file)
if not os.path.isdir("snapshots"):
os.mkdir("snapshots")
for s in snapshots:
shutil.move(s, os.path.join("snapshots", s))
def start(self):
### get stack parameteres
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackId'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(
self.params['stackId'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'],
self.stack['data']['name'])
### copy stack into working directory
if os.path.isfile(self.stack['file']):
apDisplay.printColor("copying stack into running directoroy",
"cyan")
if self.stack['file'][-4:] == ".img" or self.stack['file'][
-4:] == ".hed":
strippedfile = self.stack['file'][:-4]
else:
strippedfile = self.stack['file']
while os.path.isfile(
os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(
os.path.join(self.params['rundir'], "start.img"))
emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\
"first=0 last="+str(self.params['numpart']-1)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("stack not found in database")
### get template stack parameters
self.templatestack = {}
self.templatestack[
'data'] = appiondata.ApTemplateStackData.direct_query(
self.params['templateStackId'])
self.templatestack['apix'] = self.templatestack['data']['apix']
self.templatestack['boxsize'] = self.templatestack['data']['boxsize']
self.templatestack['file'] = os.path.join(
self.templatestack['data']['path']['path'],
self.templatestack['data']['templatename'])
self.templatestack['numimages'] = self.templatestack['data'][
'numimages']
### copy templates into working directory
if os.path.isfile(self.templatestack['file']):
apDisplay.printColor("copying templates into running directoroy",
"cyan")
ts = os.path.join(self.params['rundir'], "references.img")
while os.path.isfile(ts):
apFile.removeStack(ts)
if self.templatestack['file'][-4:] == ".img" or self.templatestack[
'file'][-4:] == ".hed":
strippedfile = self.templatestack['file'][:-4]
else:
strippedfile = self.templatestack['file']
emancmd = "proc2d " + strippedfile + ".img " + ts
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("template stack not found in database")
### set new pixelsize
if self.params['bin'] is not None and self.params['bin'] != 0:
self.params['apix'] = float(self.stack['apix']) * int(
self.params['bin'])
else:
self.params['apix'] = self.stack['apix']
### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes
scalingbatchfile = self.createImagicBatchFileScaling()
preptime = time.time()
proc = subprocess.Popen("chmod 775 " + str(scalingbatchfile),
shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(scalingbatchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(
os.path.join(self.params['rundir'], "prepareStack.log"))
apDisplay.printColor(
"finished IMAGIC in " +
apDisplay.timeString(time.time() - preptime), "cyan")
### set new boxsize, done only after scaling is complete
if self.params['bin'] is not None:
self.params['boxsize'] = apFile.getBoxSize(
os.path.join(self.params['rundir'], "start.hed"))[0]
else:
self.params['boxsize'] = self.stack['boxsize']
### make sure template stack boxsize matches that of the input stack
if self.params['apix'] != self.templatestack['apix'] or self.params[
'boxsize'] != self.templatestack['boxsize']:
self.scaleTemplates()
starttime = time.time()
print self.params
print "... stack pixel size: " + str(self.params['apix'])
print "... stack box size: " + str(self.params['boxsize'])
apDisplay.printColor(
"Running IMAGIC .batch file: See multiReferenceAlignment.log file for details",
"cyan")
### create IMAGIC batch file
batchfile = self.createImagicBatchFileMRA()
### execute IMAGIC batch file
aligntime0 = time.time()
proc = subprocess.Popen("chmod 775 " + str(batchfile), shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(batchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(
os.path.join(self.params['rundir'], "multiReferenceAlignment.log"))
apDisplay.printColor(
"finished IMAGIC in " +
apDisplay.timeString(time.time() - aligntime0), "cyan")
### get particle parameters (shift, rotate, refnum, mirror, ccc)
partparams = self.getParticleParams()
### average stack
alignstack = os.path.join(self.params['rundir'], "alignstack.hed")
apStack.averageStack(alignstack)
### normalize particles (otherwise found problems in viewing with stackviewer)
emancmd = "proc2d " + alignstack + " " + alignstack + ".norm.hed norm"
while os.path.isfile(alignstack + ".norm.img"):
apFile.removeStack(alignstack + ".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(alignstack + ".norm.hed", alignstack)
os.rename(alignstack + ".norm.img", alignstack[:-4] + ".img")
### normalize references
emancmd = "proc2d " + ts + " " + ts + ".norm.hed norm"
while os.path.isfile(ts + ".norm.img"):
apFile.removeStack(ts + ".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(ts + ".norm.hed", ts)
os.rename(ts + ".norm.img", ts[:-4] + ".img")
### remove copied stack
while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(os.path.join(self.params['rundir'],
"start.img"))
### insert run into database
self.insertAlignmentRun(insert=True)
self.insertParticlesIntoDatabase(partparams, insert=True)
def start(self):
### get stack parameteres
self.stack = {}
self.stack['data'] = apStack.getOnlyStackData(self.params['stackId'])
self.stack['apix'] = apStack.getStackPixelSizeFromStackId(self.params['stackId'])
self.stack['boxsize'] = apStack.getStackBoxsize(self.params['stackId'])
self.stack['file'] = os.path.join(self.stack['data']['path']['path'], self.stack['data']['name'])
### copy stack into working directory
if os.path.isfile(self.stack['file']):
apDisplay.printColor("copying stack into running directoroy", "cyan")
if self.stack['file'][-4:] == ".img" or self.stack['file'][-4:] == ".hed":
strippedfile = self.stack['file'][:-4]
else:
strippedfile = self.stack['file']
while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(os.path.join(self.params['rundir'], "start.img"))
emancmd = "proc2d "+strippedfile+".hed "+os.path.join(self.params['rundir'], "start.hed ")+\
"first=0 last="+str(self.params['numpart']-1)
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("stack not found in database")
### get template stack parameters
self.templatestack = {}
self.templatestack['data'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId'])
self.templatestack['apix'] = self.templatestack['data']['apix']
self.templatestack['boxsize'] = self.templatestack['data']['boxsize']
self.templatestack['file'] = os.path.join(self.templatestack['data']['path']['path'], self.templatestack['data']['templatename'])
self.templatestack['numimages'] = self.templatestack['data']['numimages']
### copy templates into working directory
if os.path.isfile(self.templatestack['file']):
apDisplay.printColor("copying templates into running directoroy", "cyan")
ts = os.path.join(self.params['rundir'], "references.img")
while os.path.isfile(ts):
apFile.removeStack(ts)
if self.templatestack['file'][-4:] == ".img" or self.templatestack['file'][-4:] == ".hed":
strippedfile = self.templatestack['file'][:-4]
else:
strippedfile = self.templatestack['file']
emancmd = "proc2d "+strippedfile+".img "+ts
apParam.runCmd(emancmd, "EMAN")
else:
apDisplay.printError("template stack not found in database")
### set new pixelsize
if self.params['bin'] is not None and self.params['bin'] != 0:
self.params['apix'] = float(self.stack['apix']) * int(self.params['bin'])
else:
self.params['apix'] = self.stack['apix']
### scale, low-pass, and high-pass filter stack ... do this with imagic, because it determines the appropriate boxsizes
scalingbatchfile = self.createImagicBatchFileScaling()
preptime = time.time()
proc = subprocess.Popen("chmod 775 "+str(scalingbatchfile), shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(scalingbatchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(os.path.join(self.params['rundir'], "prepareStack.log"))
apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-preptime), "cyan")
### set new boxsize, done only after scaling is complete
if self.params['bin'] is not None:
self.params['boxsize'] = apFile.getBoxSize(os.path.join(self.params['rundir'], "start.hed"))[0]
else:
self.params['boxsize'] = self.stack['boxsize']
### make sure template stack boxsize matches that of the input stack
if self.params['apix'] != self.templatestack['apix'] or self.params['boxsize'] != self.templatestack['boxsize']:
self.scaleTemplates()
starttime=time.time()
print self.params
print "... stack pixel size: "+str(self.params['apix'])
print "... stack box size: "+str(self.params['boxsize'])
apDisplay.printColor("Running IMAGIC .batch file: See multiReferenceAlignment.log file for details", "cyan")
### create IMAGIC batch file
batchfile = self.createImagicBatchFileMRA()
### execute IMAGIC batch file
aligntime0 = time.time()
proc = subprocess.Popen("chmod 775 "+str(batchfile), shell=True)
proc.wait()
os.chdir(self.params['rundir'])
apParam.runCmd(batchfile, "IMAGIC")
apIMAGIC.checkLogFileForErrors(os.path.join(self.params['rundir'], "multiReferenceAlignment.log"))
apDisplay.printColor("finished IMAGIC in "+apDisplay.timeString(time.time()-aligntime0), "cyan")
### get particle parameters (shift, rotate, refnum, mirror, ccc)
partparams = self.getParticleParams()
### average stack
alignstack = os.path.join(self.params['rundir'], "alignstack.hed")
apStack.averageStack(alignstack)
### normalize particles (otherwise found problems in viewing with stackviewer)
emancmd = "proc2d "+alignstack+" "+alignstack+".norm.hed norm"
while os.path.isfile(alignstack+".norm.img"):
apFile.removeStack(alignstack+".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(alignstack+".norm.hed", alignstack)
os.rename(alignstack+".norm.img", alignstack[:-4]+".img")
### normalize references
emancmd = "proc2d "+ts+" "+ts+".norm.hed norm"
while os.path.isfile(ts+".norm.img"):
apFile.removeStack(ts+".norm.img")
apParam.runCmd(emancmd, "EMAN")
os.rename(ts+".norm.hed", ts)
os.rename(ts+".norm.img", ts[:-4]+".img")
### remove copied stack
while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
apFile.removeStack(os.path.join(self.params['rundir'], "start.img"))
### insert run into database
self.insertAlignmentRun(insert=True)
self.insertParticlesIntoDatabase(partparams, insert=True)
def compute_stack_of_class_averages_and_reprojections(dir,
selfile,
refvolume,
docfile,
boxsize,
resultspath,
timestamp,
iteration,
reference_number=1,
extract=False):
''' takes Xmipp single files, doc and sel files in routine, creates a stack of class averages in the results directory '''
workingdir = os.getcwd()
os.chdir(dir)
if dir.endswith("/"):
dir = dir[:-1]
head, tail = os.path.split(dir)
### remove "lastdir" component from selfile (created by Xmipp program), then extract header information to docfile
f = open(selfile, "r")
lines = f.readlines()
newlines = [re.sub(str(tail) + "/", "", line) for line in lines]
f.close()
f = open(selfile[:-4] + "_new.sel", "w")
f.writelines(newlines)
f.close()
if extract is True:
extractcmd = "xmipp_header_extract -i %s.sel -o %s.doc" % (
selfile[:-4], docfile[:-4])
apParam.runCmd(extractcmd, "Xmipp")
### create a projection params file and project the volume along identical Euler angles
f = open("paramfile.descr", "w")
f.write("%s\n" % refvolume)
f.write("tmpproj 1 xmp\n")
f.write("%d %d\n" % (boxsize, boxsize))
f.write("%s rot tilt psi\n" % docfile)
f.write("NULL\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.write("0 0\n")
f.close()
projectcmd = "xmipp_project -i paramfile.descr"
apParam.runCmd(projectcmd, "Xmipp")
### get order of projections in docfile
d = open(docfile, "r")
lines = d.readlines()[1:]
d.close()
projfile_sequence = []
for i, l in enumerate(lines):
if i % 2 == 0:
filename = os.path.basename(l.split()[1])
projfile_sequence.append(filename)
else:
pass
### create stack of projections and class averages
projections = glob.glob("tmpproj**xmp")
projections.sort()
if len(projections) != len(projfile_sequence):
apDisplay.printWarning(
"number of projections does not match number of classes")
stackarray = []
stackname = os.path.join(
resultspath, "proj-avgs_%s_it%.3d_vol%.3d.hed" %
(timestamp, iteration, reference_number))
for i in range(len(projections)):
stackarray.append(spider.read(projections[i]))
stackarray.append(spider.read(projfile_sequence[i]))
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### remove unnecessary files
for file in glob.glob("tmpproj*"):
apFile.removeFile(file)
os.chdir(workingdir)
return
def start(self):
t0 = time.time()
self.checkAnalysisRun()
# get stack parameters
if self.params['alignid'] is not None:
self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignid'])
stackpixelsize = self.alignstackdata['pixelsize']
stack_box_size = self.alignstackdata['boxsize']
self.params['boxsize'] = stack_box_size / int(self.params['bin'])
self.params['apix'] = stackpixelsize * int(self.params['bin'])
orig_path = self.alignstackdata['path']['path']
orig_file = self.alignstackdata['imagicfile']
linkingfile = orig_path+"/"+orig_file
linkingfile = linkingfile.replace(".hed", "")
else:
apDisplay.printError("stack not in the database")
# link stack file to working directory
if not os.path.isfile(linkingfile+".hed"):
apDisplay.printError("stackfile does not exist: "+linkingfile+".img")
else:
if not os.path.isfile(os.path.join(str(self.params['rundir']), "start.img")):
apDisplay.printMsg("copying aligned stack into working directory for operations with IMAGIC")
# shutil.copyfile(linkingfile+".img", str(self.params['rundir'])+"/start.img")
# shutil.copyfile(linkingfile+".hed", str(self.params['rundir'])+"/start.hed")
lnkcmd1 = "ln -s "+linkingfile+".img "+os.path.join(self.params['rundir'], "start.img")
lnkcmd2 = "ln -s "+linkingfile+".hed "+os.path.join(self.params['rundir'], "start.hed")
proc = subprocess.Popen(lnkcmd1, shell=True)
proc.wait()
proc = subprocess.Popen(lnkcmd2, shell=True)
proc.wait()
### header bug
apImagicFile.setImagic4DHeader(os.path.join(self.params['rundir'], "start.hed"))
else:
apDisplay.printColor("aligned stack already exists in working directory", "green")
### NEED TO CONVERT FILTERING PARAMETERS TO IMAGIC FORMAT BETWEEN 0-1
if self.params['lpfilt'] is not None:
self.params['lpfilt_imagic'] = 2 * float(self.params['apix']) / int(self.params['lpfilt'])
else:
self.params['lpfilt_imagic'] = False
if float(self.params['lpfilt_imagic']) > 1:
self.params['lpfilt_imagic'] = 1 # imagic cannot perform job when lowpass > 1
if self.params['hpfilt'] is not None:
self.params['hpfilt_imagic'] = 2 * float(self.params['apix']) / int(self.params['hpfilt'])
else:
self.params['hpfilt_imagic'] = False
print self.params
print "... aligned stack pixel size: "+str(self.params['apix'])
print "... aligned stack box size: "+str(self.params['boxsize'])
apDisplay.printColor("Running IMAGIC .batch file: See imagicMultivariateStatisticalAnalysis.log for details", "cyan")
### create imagic batch file
filename = self.createImagicBatchFile()
### execute batch file that was created
aligntime = time.time()
proc = subprocess.Popen('chmod 775 '+filename, shell=True)
proc.wait()
apParam.runCmd(filename, package="IMAGIC")
logfile = open(os.path.join(self.params['rundir'], "imagicMultivariateStatisticalAnalysis.log"))
apIMAGIC.checkLogFileForErrors(os.path.join(self.params['rundir'], "imagicMultivariateStatisticalAnalysis.log"))
if not os.path.isfile(os.path.join(self.params['rundir'], "eigenimages.hed")):
apDisplay.printError("IMAGIC did not run and did not create eigenimages")
aligntime = time.time() - aligntime
### remove copied stack
# while os.path.isfile(os.path.join(self.params['rundir'], "start.img")):
# apFile.removeStack(os.path.join(self.params['rundir'], "start.img"))
### normalize eigenimages
eigenimages = os.path.join(self.params['rundir'], "eigenimages.img")
emancmd = "proc2d "+str(eigenimages)+" "+str(eigenimages)+" inplace"
apParam.runCmd(emancmd, package="EMAN")
### upload alignment
imagicstack = os.path.join(self.params['rundir'], "start.hed")
inserttime = time.time()
if self.params['commit'] is True:
self.insertAnalysis(imagicstack, runtime=aligntime, insert=True)
else:
apDisplay.printWarning("not committing results to DB")
inserttime = time.time() - inserttime
apDisplay.printMsg("Alignment time: "+apDisplay.timeString(aligntime))
apDisplay.printMsg("Database Insertion time: "+apDisplay.timeString(inserttime))
def start(self):
### simple is written in Fortran, which cannot take inputs of certain length, therefore one needs
### to change to the directory to minimize the filename length, in particular for the stack
os.chdir(self.params['rundir'])
### stack needs to be centered
if self.params['no_center'] is False:
if os.path.isfile(os.path.join(self.params['rundir'], "ali.hed")):
apFile.removeStack(os.path.join(self.params['rundir'], "ali.hed"))
centstack = os.path.join(self.params['rundir'], "ali.hed")
centcmd = "cenalignint %s > cenalignint.log" % (self.stack['file'])
apParam.runCmd(centcmd, "EMAN")
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(self.params['rundir'], self.params['timestamp']+".spi")
if os.path.isfile(self.params['localstack']):
apFile.removeFile(self.params['localstack'])
if self.params['no_center'] is False:
proccmd = "proc2d "+centstack+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
else:
proccmd = "proc2d "+self.stack['file']+" "+self.params['localstack']+" apix="+str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'], self.boxsize, self.boxsize)
proccmd += " last="+str(self.params['numpart']-1)
proccmd += " spiderswap"
# if self.params['highpass'] is not None and self.params['highpass'] > 1:
# proccmd += " hp="+str(self.params['highpass'])
# if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
# proccmd += " lp="+str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
# if self.params['numpart'] != int(spider.getSpiderHeader(self.params['localstack'])[-2]):
# apDisplay.printError("Missing particles in stack")
### setup Simple command
aligntime = time.time()
simpleopts = (""
+" stk=%s" % os.path.basename(self.params['localstack'])
+" box=%d" % self.boxsize
+" nptcls=%d" % self.params['numpart']
+" smpd=%.3f" % self.apix
+" ring2=%d" % self.params['ring2']
+" ncls=%d" % self.params['ncls']
+" minp=%d" % self.params['minp']
+" nvars=%d" % self.params['nvars']
+" nthr=%d" % self.params['nproc']
)
if self.params['no_kmeans'] is True:
simpleopts += " kmeans=off"
if self.params['nran'] is not None:
simpleopts += "nran=%d" % self.params['nran']
### SIMPLE 2D clustering
apDisplay.printColor("Using "+str(self.params['nproc'])+" processors!", "green")
simpleexe = apParam.getExecPath("cluster", die=True)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="cluster.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment & Classification time: "+apDisplay.timeString(self.params['runtime']))
### SIMPLE spider to Fourier format
clsavgs = "cavgstk.spi"
if not os.path.isfile(os.path.join(self.params['rundir'], clsavgs)):
apDisplay.printError("class averages were not created! try rerunning with centering, more particles, or less ppc")
try:
nptcls = spider.getSpiderHeader(clsavgs)[-2]
except:
nptcls = self.params['ncls']
apDisplay.printWarning("class average file may not have been created! Please check existence of file cavgstk.spi")
projfile = "projs"
projext = ".fim"
simpleexe = apParam.getExecPath("spi_to_fim", die=True)
simpleopts = (""
+" stk=%s" % clsavgs
+" box=%d" % self.boxsize
+" nptcls=%d" % nptcls
+" smpd=%.3f" % self.apix
+" outbdy=%s" % projfile
+" msk=%d" % self.params['mask']
)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="spi_to_fim.std")
### SIMPLE origami, ab initio 3D reconstruction
refinetime = time.time()
simpleexe = apParam.getExecPath("origami", die=True)
simpleopts = (""
+" fstk=%s" % projfile+projext
+" froms=%d" % self.params['froms']
+" tos=%d" % self.params['tos']
+" lp=%d" % self.params['lp']
+" hp=%d" % self.params['hp']
+" maxits=%d" % self.params['maxits']
+" msk=%d" % self.params['mask']
+" mw=%d" % self.params['mw']
+" frac=%.3f" % self.params['frac']
+" amsklp=%d" % self.params['amsklp']
+" edge=%d" % self.params['edge']
+" trs=%d" % self.params['trs']
+" nthr=%d" % self.params['nproc']
)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd, package="SIMPLE", verbose=True, showcmd=True, logfile="origami.std")
refinetime = time.time() - refinetime
apDisplay.printMsg("Origami reconstruction time: "+apDisplay.timeString(refinetime))
# '''
### minor post-processing
self.clearIntermediateFiles()
apParam.dumpParameters(self.params, "simple-"+self.params['timestamp']+"-params.pickle")
### upload results
self.runparams = apParam.readRunParameters("simple-"+self.params['timestamp']+"-params.pickle")
### create average of aligned and clustered stacks, convert to IMAGIC
alignedStackSpi = "inplalgnstk.spi"
alignedStack = "inplalgnstk.hed"
if os.path.isfile(alignedStack):
apFile.removeStack(alignedStack)
emancmd = "proc2d %s %s flip" % (alignedStackSpi, alignedStack)
apParam.runCmd(emancmd, "EMAN")
clusterStackSpi = "cavgstk.spi"
clusterStack = "cavgstk.hed"
if os.path.isfile(clusterStack):
apFile.removeStack(clusterStack)
emancmd = "proc2d %s %s flip" % (clusterStackSpi, clusterStack)
apParam.runCmd(emancmd, "EMAN")
# apStack.averageStack(alignedStack)
### parse alignment and classification results
if self.params['no_center'] is False:
self.alignD = self.getAlignParameters(centparams="cenalignint.log")
else:
self.alignD = self.getAlignParameters()
if self.params['no_kmeans'] is False:
self.classD = self.getClassification("kmeans.spi", clusterStack)
else:
self.classD = self.getClassification("hcl.spi", clusterStack)
### upload to database
self.insertSIMPLEAlignParamsIntoDatabase()
self.insertAlignStackRunIntoDatabase(alignedStack, clusterStack)
self.calcResolution(alignedStack)
self.insertAlignParticlesIntoDatabase()
self.insertClusterRunIntoDatabase()
self.insertClusterStackIntoDatabase(clusterStack, len(self.classD))
self.insertSIMPLEOrigamiParamsIntoDatabase()
def calculateFSCforIteration(self, iteration, reference_number):
''' requires manipulation of data using Xmipp programs, benefits from MPI parallelization '''
os.chdir(self.ml3dpath)
### split selfile
selfile = "ml3d_it%.6d_class_vol%.6d.sel" % (iteration,reference_number)
xmipp_split_selfile_cmd = "xmipp_selfile_split -i %s" % selfile
apParam.runCmd(xmipp_split_selfile_cmd, "Xmipp")
### make even and odd docfiles from split selfile
xmipp_mk_docfile_cmd1 = "xmipp_docfile_select_subset -i ml3d_it%.6d.doc -sel ml3d_it%.6d_class_vol%.6d_1.sel -o tmpdocfile_1.doc" \
% (iteration,iteration,reference_number)
apParam.runCmd(xmipp_mk_docfile_cmd1, "Xmipp")
xmipp_mk_docfile_cmd2 = "xmipp_docfile_select_subset -i ml3d_it%.6d.doc -sel ml3d_it%.6d_class_vol%.6d_2.sel -o tmpdocfile_2.doc" \
% (iteration,iteration,reference_number)
apParam.runCmd(xmipp_mk_docfile_cmd2, "Xmipp")
### create angular class averages from split docfiles and selfiles
if self.params['nproc'] > 1:
xmipp_average_cmd1 = "mpirun -np %d xmipp_mpi_angular_class_average -i tmpdocfile_1.doc -lib ml3d_lib_new.doc -o tmpclass_1" \
% (self.params['nproc'])
xmipp_average_cmd2 = "mpirun -np %d xmipp_mpi_angular_class_average -i tmpdocfile_2.doc -lib ml3d_lib_new.doc -o tmpclass_2" \
% (self.params['nproc'])
else:
xmipp_average_cmd1 = "xmipp_angular_class_average -i tmpdocfile_1.doc -lib ml3d_lib_new.doc -o tmpclass_1"
xmipp_average_cmd2 = "xmipp_angular_class_average -i tmpdocfile_2.doc -lib ml3d_lib_new.doc -o tmpclass_2"
apParam.runCmd(xmipp_average_cmd1, "Xmipp")
apParam.runCmd(xmipp_average_cmd2, "Xmipp")
### reconstruct even volume
if self.params['nproc'] > 1:
xmipp_reconstruct_cmd = "mpirun -np %d xmipp_mpi_reconstruct_fourier -i tmpclass_1_classes.sel -o %s -weight" \
% (self.params['nproc'], "even.vol")
else:
xmipp_reconstruct_cmd = "xmipp_reconstruct_fourier -i tmpclass_1_classes.sel -o %s -weight" \
% ("even.vol")
apParam.runCmd(xmipp_reconstruct_cmd, "Xmipp")
### reconstruct odd volume
if self.params['nproc'] > 1:
xmipp_reconstruct_cmd = "mpirun -np %d xmipp_mpi_reconstruct_fourier -i tmpclass_2_classes.sel -o %s -weight" \
% (self.params['nproc'], "odd.vol")
else:
xmipp_reconstruct_cmd = "xmipp_reconstruct_fourier -i tmpclass_2_classes.sel -o %s -weight" \
% ("odd.vol")
apParam.runCmd(xmipp_reconstruct_cmd, "Xmipp")
### calculate FSC
apDisplay.printMsg("calculating FSC resolution for iteration %d, volume %d using split selfile" % (iteration,reference_number))
xmipp_resolution_cmd = "xmipp_resolution_fsc -ref even.vol -i odd.vol -sam %.3f" % (self.runparams['apix'])
apParam.runCmd(xmipp_resolution_cmd, "Xmipp")
### rename FSC file, get resolution value, and remove unwanted files
oldfscfile = os.path.join(self.ml3dpath, "odd.vol.frc")
newfscfile = os.path.join(self.resultspath, "recon_%s_it%.3d_vol%.3d.fsc" % (self.params['timestamp'],iteration,reference_number))
if os.path.exists(oldfscfile):
shutil.move(oldfscfile, newfscfile)
apFile.removeFile("ml3d_it%.6d_class_vol%.6d_1.sel" % (iteration,reference_number))
apFile.removeFile("ml3d_it%.6d_class_vol%.6d_2.sel" % (iteration,reference_number))
list = []
list += [f for f in glob.glob("odd*")]
list += [f for f in glob.glob("even*")]
list += [f for f in glob.glob("tmp*")]
for file in list:
apFile.removeFile(file)
os.chdir(self.params['rundir'])
return
def createParticleDataFile(self, iteration, reference_number, total_num_2d_classes):
''' puts all relevant particle information into a single text file that can be read by the uploader '''
os.chdir(self.ml3dpath)
### create docfile for the iteration and reference number
docfile = os.path.join(self.ml3dpath, "ml3d_it%.6d.doc" % (iteration))
selfile = "ml3d_it%.6d_class_vol%.6d.sel" % (iteration,reference_number)
makedocfilecmd = "xmipp_docfile_select_subset -i %s -sel %s -o ml3d_it%.6d_class_vol%.6d.doc" \
% (docfile, selfile, iteration,reference_number)
apParam.runCmd(makedocfilecmd, "Xmipp")
classes_per_volume = total_num_2d_classes / len(self.modeldata)
### read output from ml3d
ml3ddocfile = os.path.join(self.ml3dpath, "ml3d_it%.6d_class_vol%.6d.doc" % (iteration, reference_number))
ml3df = open(ml3ddocfile, "r")
ml3dlines = ml3df.readlines()[1:]
ml3dsplitlines = [l.strip().split() for l in ml3dlines]
ml3df.close()
### write data in appion format to input file for uploading to the database
particledataf = open(os.path.join(self.resultspath, "particle_data_%s_it%.3d_vol%.3d.txt" % (self.params['timestamp'], iteration, reference_number)), "w")
# particledataf.write("### column info: ")
particledataf.write("#%8s" % "partnum")
particledataf.write("%10s" % "phi")
particledataf.write("%10s" % "theta")
particledataf.write("%10s" % "omega")
particledataf.write("%10s" % "shiftx")
particledataf.write("%10s" % "shifty")
# particledataf.write("(7) mirror")
particledataf.write("%8s" % "3D_ref#")
particledataf.write("%8s" % "2D_cls#")
particledataf.write("%10s" % "qfact")
particledataf.write("%8s" % "keptp")
particledataf.write("%8s\n" % "p_keptp")
for i in range(len(ml3dsplitlines)/2):
if int(float(ml3dsplitlines[i*2+1][7])) % classes_per_volume == 0:
n = classes_per_volume
else:
n = int(float(ml3dsplitlines[i*2+1][7])) % classes_per_volume
phi = float(ml3dsplitlines[i*2+1][2]))
theta = float(ml3dsplitlines[i*2+1][3]))
psi = float(ml3dsplitlines[i*2+1][4]))
mirror = bool(float(ml3dsplitlines[i*2+1][8])))
if mirror is True:
phi, theta, psi = apXmipp.calculate_equivalent_Eulers_without_flip(phi, theta, psi)
particledataf.write("%9d" % (int(ml3dsplitlines[i*2][1][-10:-4])+1)) ### NOTE: IT IS IMPORTANT TO START WITH 1, OTHERWISE STACKMAPPING IS WRONG!!!
particledataf.write("%10.4f" % phi)
particledataf.write("%10.4f" % theta)
particledataf.write("%10.4f" % psi)
particledataf.write("%10.4f" % float(ml3dsplitlines[i*2+1][5]))
particledataf.write("%10.4f" % float(ml3dsplitlines[i*2+1][6]))
# particledataf.write("%8d" % int(float(ml3dsplitlines[i*2+1][8]))) # deprecated: mirror is flipped already. Set to False
particledataf.write("%8d" % int(reference_number))
particledataf.write("%8d" % n)
particledataf.write("%10.4f" % 0)
particledataf.write("%8d" % 1)
particledataf.write("%8d\n" % 1)
particledataf.close()
os.chdir(self.params['rundir'])
return
def start(self): self.ace2correct = self.getACE2Path() ### determine amount of memory needed for entire stack memorylimit = 0.3 bytelimit = memorylimit*(1024**3) writeiters = 1 partbytes = 4*self.params['box']*self.params['box'] if partbytes*self.params['projcount'] > bytelimit: writeiters = int(math.ceil(float(partbytes)*self.params['projcount'] / bytelimit)) partsperiter = int(float(self.params['projcount']) / writeiters) ### number of particles read each time ### some defaults, and workarounds for now self.params['projpergraph'] = 100 self.params['filesperdir'] = partsperiter if self.params['filesperdir'] > 2048: self.params['filesperdir'] = 2048 ### first create projections if self.params['preforient'] is True: filename = self.createProjections(pad=self.params['pad'], invert=self.params['invert']) else: filename = self.createProjectionsEmanProp(pad=self.params['pad'], invert=self.params['invert']) ### shift & rotate randomly if self.params['rotang']!=0 or self.params['shiftrad']!=0: shiftstackname = self.shift_images(filename) else: shiftstackname = filename ### read MRC stats to figure out noise level addition mean1, stdev1 = self.readFileStats(shiftstackname) ### determine noise multiplication factor to ensure that appropriate amount of noise gets added to particles inside circular mask multfactor = 1.0/((float(self.params['radius'])/self.params['box'])*(float(self.params['radius'])/self.params['box'])*math.pi) ### calculate noiselevel additions and add noise to an initial ratio of 1.8, simulating beam and structural damage ### NOTE: THERE ARE DIFFERENT DEFINITIONS OF SNR, see below noiselevel1 = math.sqrt((float(stdev1)*float(stdev1)) / float(self.params['snr1'])) # noiselevel1 = float(stdev1) / float(self.params['snr1']) noiselevel1 = noiselevel1 * multfactor noisystack = self.addNoise(shiftstackname, noiselevel1, SNR=self.params['snr1']) ### get list of defocus values self.getListOfDefoci(self.params['projcount']) ### apply envelope and ctf to each .mrc file, then correct based on how well ace2 works on raw micrographs ctfstack, ctfpartlist = self.applyEnvelopeAndCTF(noisystack) #recoverlists = self.recoverLists() ### read IMAGIC stats to figure out noise level addition mean2, stdev2 = self.readFileStats(ctfstack) ### cascading of noise processes according to Frank and Al-Ali (1975) & Baxter (2009) # snr2 = 1 / ((1+1/float(self.params['snrtot'])) / (1/float(self.params['snr1']) + 1) - 1) snr2 = (1+1/self.params['snr1'])/(1/self.params['snrtot']-1/self.params['snr1']) ### NOTE: THERE ARE DIFFERENT DEFINITIONS OF SNR, see below noiselevel2 = math.sqrt((float(stdev2)*float(stdev2)) / float(snr2)) # noiselevel2 = float(stdev2) / float(snr2) noiselevel2 = noiselevel2 * multfactor ### add a last layer of noise noisystack2 = self.addNoise(ctfstack, noiselevel2, SNR=self.params['snrtot']) ### low-pass / high-pass filter resulting stack, if specified if self.params['hpfilt'] is not None or self.params['lpfilt'] is not None or self.params['norm'] is True: filtstack = noisystack2[:-4] if self.params['norm'] is True: filtstack = filtstack+"_norm.hed" else: filtstack = filtstack+"_filt.hed" apFile.removeStack(filtstack) emancmd = "proc2d "+noisystack2+" "+filtstack+" apix="+str(self.params['apix'])+" " if self.params['hpfilt'] is not None: emancmd = emancmd+"hp="+str(self.params['hpfilt'])+" " if self.params['lpfilt'] is not None: emancmd = emancmd+"lp="+str(self.params['lpfilt'])+" " if self.params['norm'] is True: emancmd = emancmd+"norm="+str(self.params['norm'])+" " apParam.runCmd(emancmd, "EMAN") self.params['finalstack'] = os.path.basename(filtstack) finalstack = filtstack else: self.params['finalstack'] = os.path.basename(noisystack2) finalstack = noisystack2 ### post-processing: create average file for viewing on webpages apStack.averageStack(finalstack) ### upload if commit is checked self.uploadData(ctfpartlist) ### post-processing: Create Stack Mean Plot if self.params['commit'] is True: stackid = apStack.getStackIdFromPath(finalstack) if stackid is not None: apStackMeanPlot.makeStackMeanPlot(stackid, gridpoints=8)
def createProjections(self, pad=False, invert=False):
timestamp = apParam.makeTimestamp()
eulerlist = self.setEulers()
eulerfile = os.path.join(self.params['rundir'], "eulers.lst")
f = open(eulerfile, "w")
projcount = numpy.zeros((len(eulerlist)), dtype=numpy.uint16)
angsum = numpy.zeros((len(eulerlist),3), dtype=numpy.float32)
t0 = time.time()
for i in range(self.params['projcount']):
projnum = int(random.random()*len(eulerlist))
alt = random.gauss(eulerlist[projnum][0], self.params['projstdev'])
az = random.gauss(eulerlist[projnum][1], self.params['projstdev'])
#phi = random.random()*360.0-180.0
#phi = random.random()*360.0
if self.params['rotang'] != 0:
phi = random.uniform(-1*self.params['rotang'], self.params['rotang'])
else:
phi = 0.0
f.write("%.8f\t%.8f\t%.8f\n"%(alt,az,phi))
### stats
projcount[projnum] += 1
angsum[projnum,0] += alt
angsum[projnum,1] += az
angsum[projnum,2] += phi
apDisplay.printMsg("Finished random in %s, %.3f ns per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e6 * (time.time()-t0)/float(self.params['projcount'])))
f.close()
print "projection count", projcount
for i in range(len(eulerlist)):
angavg = angsum[i,:]/projcount[i]
print "angle average %d: %03.3f, %03.3f, %03.3f"%(i, angavg[0], angavg[1], angavg[2])
### first get rid of projection artifacts from insufficient padding
if self.params['threedfile'] is not None:
origfile = self.params['threedfile']
elif self.params['modelid'] is not None:
self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid'])
origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name'])
if self.params['apix'] is None:
self.params['apix'] = self.modelparams['pixelsize']
if self.params['box'] is None:
self.params['box'] = self.modelparams['boxsize']
clipped = os.path.join(self.params['rundir'], "clipped.mrc")
newsize = self.params['box'] * 1.5
emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
### project resized file
filename = os.path.join(self.params['rundir'], 'proj.img')
apFile.removeStack(filename)
emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile
t0 = time.time()
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount'])))
### post-process stack
self.post_process_stack(filename, pad, invert)
return filename
def start(self):
### simple is written in Fortran, which cannot take inputs of certain length, therefore one needs
### to change to the directory to minimize the filename length, in particular for the stack
os.chdir(self.params['rundir'])
### stack needs to be centered
if self.params['no_center'] is False:
if os.path.isfile(os.path.join(self.params['rundir'], "ali.hed")):
apFile.removeStack(
os.path.join(self.params['rundir'], "ali.hed"))
centstack = os.path.join(self.params['rundir'], "ali.hed")
centcmd = "cenalignint %s > cenalignint.log" % (self.stack['file'])
apParam.runCmd(centcmd, "EMAN")
### process stack to local file
if self.params['timestamp'] is None:
apDisplay.printMsg("creating timestamp")
self.params['timestamp'] = self.timestamp
self.params['localstack'] = os.path.join(
self.params['rundir'], self.params['timestamp'] + ".spi")
if os.path.isfile(self.params['localstack']):
apFile.removeFile(self.params['localstack'])
if self.params['no_center'] is False:
proccmd = "proc2d " + centstack + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
else:
proccmd = "proc2d " + self.stack['file'] + " " + self.params[
'localstack'] + " apix=" + str(self.stack['apix'])
if self.params['bin'] > 1 or self.params['clipsize'] is not None:
proccmd += " shrink=%d clip=%d,%d " % (self.params['bin'],
self.boxsize, self.boxsize)
proccmd += " last=" + str(self.params['numpart'] - 1)
proccmd += " spiderswap"
# if self.params['highpass'] is not None and self.params['highpass'] > 1:
# proccmd += " hp="+str(self.params['highpass'])
# if self.params['lowpass'] is not None and self.params['lowpass'] > 1:
# proccmd += " lp="+str(self.params['lowpass'])
apParam.runCmd(proccmd, "EMAN", verbose=True)
# if self.params['numpart'] != int(spider.getSpiderHeader(self.params['localstack'])[-2]):
# apDisplay.printError("Missing particles in stack")
### setup Simple command
aligntime = time.time()
simpleopts = (
"" + " stk=%s" % os.path.basename(self.params['localstack']) +
" box=%d" % self.boxsize + " nptcls=%d" % self.params['numpart'] +
" smpd=%.3f" % self.apix + " ring2=%d" % self.params['ring2'] +
" ncls=%d" % self.params['ncls'] +
" minp=%d" % self.params['minp'] +
" nvars=%d" % self.params['nvars'] +
" nthr=%d" % self.params['nproc'])
if self.params['no_kmeans'] is True:
simpleopts += " kmeans=off"
if self.params['nran'] is not None:
simpleopts += "nran=%d" % self.params['nran']
### SIMPLE 2D clustering
apDisplay.printColor(
"Using " + str(self.params['nproc']) + " processors!", "green")
simpleexe = apParam.getExecPath("cluster", die=True)
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="cluster.std")
self.params['runtime'] = time.time() - aligntime
apDisplay.printMsg("Alignment & Classification time: " +
apDisplay.timeString(self.params['runtime']))
### SIMPLE spider to Fourier format
clsavgs = "cavgstk.spi"
if not os.path.isfile(os.path.join(self.params['rundir'], clsavgs)):
apDisplay.printError(
"class averages were not created! try rerunning with centering, more particles, or less ppc"
)
try:
nptcls = spider.getSpiderHeader(clsavgs)[-2]
except:
nptcls = self.params['ncls']
apDisplay.printWarning(
"class average file may not have been created! Please check existence of file cavgstk.spi"
)
projfile = "projs"
projext = ".fim"
simpleexe = apParam.getExecPath("spi_to_fim", die=True)
simpleopts = ("" + " stk=%s" % clsavgs + " box=%d" % self.boxsize +
" nptcls=%d" % nptcls + " smpd=%.3f" % self.apix +
" outbdy=%s" % projfile +
" msk=%d" % self.params['mask'])
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="spi_to_fim.std")
### SIMPLE origami, ab initio 3D reconstruction
refinetime = time.time()
simpleexe = apParam.getExecPath("origami", die=True)
simpleopts = (
"" + " fstk=%s" % projfile + projext +
" froms=%d" % self.params['froms'] +
" tos=%d" % self.params['tos'] + " lp=%d" % self.params['lp'] +
" hp=%d" % self.params['hp'] +
" maxits=%d" % self.params['maxits'] +
" msk=%d" % self.params['mask'] + " mw=%d" % self.params['mw'] +
" frac=%.3f" % self.params['frac'] +
" amsklp=%d" % self.params['amsklp'] +
" edge=%d" % self.params['edge'] + " trs=%d" % self.params['trs'] +
" nthr=%d" % self.params['nproc'])
simplecmd = "%s %s" % (simpleexe, simpleopts)
self.writeSimpleLog(simplecmd)
apParam.runCmd(simplecmd,
package="SIMPLE",
verbose=True,
showcmd=True,
logfile="origami.std")
refinetime = time.time() - refinetime
apDisplay.printMsg("Origami reconstruction time: " +
apDisplay.timeString(refinetime))
# '''
### minor post-processing
self.clearIntermediateFiles()
apParam.dumpParameters(
self.params,
"simple-" + self.params['timestamp'] + "-params.pickle")
### upload results
self.runparams = apParam.readRunParameters("simple-" +
self.params['timestamp'] +
"-params.pickle")
### create average of aligned and clustered stacks, convert to IMAGIC
alignedStackSpi = "inplalgnstk.spi"
alignedStack = "inplalgnstk.hed"
if os.path.isfile(alignedStack):
apFile.removeStack(alignedStack)
emancmd = "proc2d %s %s flip" % (alignedStackSpi, alignedStack)
apParam.runCmd(emancmd, "EMAN")
clusterStackSpi = "cavgstk.spi"
clusterStack = "cavgstk.hed"
if os.path.isfile(clusterStack):
apFile.removeStack(clusterStack)
emancmd = "proc2d %s %s flip" % (clusterStackSpi, clusterStack)
apParam.runCmd(emancmd, "EMAN")
# apStack.averageStack(alignedStack)
### parse alignment and classification results
if self.params['no_center'] is False:
self.alignD = self.getAlignParameters(centparams="cenalignint.log")
else:
self.alignD = self.getAlignParameters()
if self.params['no_kmeans'] is False:
self.classD = self.getClassification("kmeans.spi", clusterStack)
else:
self.classD = self.getClassification("hcl.spi", clusterStack)
### upload to database
self.insertSIMPLEAlignParamsIntoDatabase()
self.insertAlignStackRunIntoDatabase(alignedStack, clusterStack)
self.calcResolution(alignedStack)
self.insertAlignParticlesIntoDatabase()
self.insertClusterRunIntoDatabase()
self.insertClusterStackIntoDatabase(clusterStack, len(self.classD))
self.insertSIMPLEOrigamiParamsIntoDatabase()
def createProjections(self, pad=False, invert=False):
timestamp = apParam.makeTimestamp()
eulerlist = self.setEulers()
eulerfile = os.path.join(self.params['rundir'], "eulers.lst")
f = open(eulerfile, "w")
projcount = numpy.zeros((len(eulerlist)), dtype=numpy.uint16)
angsum = numpy.zeros((len(eulerlist),3), dtype=numpy.float32)
t0 = time.time()
for i in range(self.params['projcount']):
projnum = int(random.random()*len(eulerlist))
alt = random.gauss(eulerlist[projnum][0], self.params['projstdev'])
az = random.gauss(eulerlist[projnum][1], self.params['projstdev'])
#phi = random.random()*360.0-180.0
#phi = random.random()*360.0
if self.params['rotang'] != 0:
phi = random.uniform(-1*self.params['rotang'], self.params['rotang'])
else:
phi = 0.0
f.write("%.8f\t%.8f\t%.8f\n"%(alt,az,phi))
### stats
projcount[projnum] += 1
angsum[projnum,0] += alt
angsum[projnum,1] += az
angsum[projnum,2] += phi
apDisplay.printMsg("Finished random in %s, %.3f ns per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e6 * (time.time()-t0)/float(self.params['projcount'])))
f.close()
print "projection count", projcount
for i in range(len(eulerlist)):
angavg = angsum[i,:]/projcount[i]
print "angle average %d: %03.3f, %03.3f, %03.3f"%(i, angavg[0], angavg[1], angavg[2])
### first get rid of projection artifacts from insufficient padding
if self.params['threedfile'] is not None:
origfile = self.params['threedfile']
elif self.params['modelid'] is not None:
self.modelparams = appiondata.ApInitialModelData.direct_query(self.params['modelid'])
origfile = os.path.join(self.modelparams['path']['path'], self.modelparams['name'])
if self.params['apix'] is None:
self.params['apix'] = self.modelparams['pixelsize']
if self.params['box'] is None:
self.params['box'] = self.modelparams['boxsize']
clipped = os.path.join(self.params['rundir'], "clipped.mrc")
newsize = self.params['box'] * 1.5
emancmd = "proc3d "+origfile+" "+clipped+" clip="+str(int(newsize))+","+str(int(newsize))+","+str(int(newsize))+" edgenorm"
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
### project resized file
filename = os.path.join(self.params['rundir'], 'proj.img')
apFile.removeStack(filename)
emancmd = "project3d "+clipped+" out="+filename+" list="+eulerfile
t0 = time.time()
apParam.runCmd(emancmd, "EMAN", showcmd=True, verbose=True)
apDisplay.printMsg("Finished project3d in %s, %.3f ms per iteration"
%(apDisplay.timeString(time.time()-t0), 1.0e3 * (time.time()-t0)/float(self.params['projcount'])))
### post-process stack
self.post_process_stack(filename, pad, invert)
return filename
