def xmippNormStack(self, inStackPath, outStackPath):
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)
### setup Xmipp command
xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
apDisplay.printMsg("Using Xmipp to normalize particle stack")
normtime = time.time()
xmippopts = ( " "
+" -i %s"%os.path.join(self.params['rundir'],selfile)
+" -method Ramp "
+" -background circle %i"%(self.stack['boxsize']/self.params['bin']*0.4)
+" -remove_black_dust"
+" -remove_white_dust"
+" -thr_black_dust -%.2f"%(self.params['xmipp-norm'])
+" -thr_white_dust %.2f"%(self.params['xmipp-norm'])
)
xmippcmd = xmippexe+" "+xmippopts
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
normtime = time.time() - normtime
apDisplay.printMsg("Xmipp normalization time: "+apDisplay.timeString(normtime))
### recombine particles to a single imagic stack
tmpstack = "tmp.xmippStack.hed"
apXmipp.gatherSingleFilesIntoStack(selfile,tmpstack)
apFile.moveStack(tmpstack,outStackPath)
### clean up directory
apFile.removeFile(selfile)
apFile.removeDir("partfiles")
def xmippNormStack(self, inStackPath, outStackPath):
### convert stack into single spider files
selfile = apXmipp.breakupStackIntoSingleFiles(inStackPath)
### setup Xmipp command
xmippexe = apParam.getExecPath("xmipp_normalize", die=True)
apDisplay.printMsg("Using Xmipp to normalize particle stack")
normtime = time.time()
xmippopts = (" " +
" -i %s" % os.path.join(self.params['rundir'], selfile) +
" -method Ramp " + " -background circle %i" %
(self.stack['boxsize'] / self.params['bin'] * 0.4) +
" -remove_black_dust" + " -remove_white_dust" +
" -thr_black_dust -%.2f" % (self.params['xmipp-norm']) +
" -thr_white_dust %.2f" % (self.params['xmipp-norm']))
xmippcmd = xmippexe + " " + xmippopts
apParam.runCmd(xmippcmd, package="Xmipp", verbose=True, showcmd=True)
normtime = time.time() - normtime
apDisplay.printMsg("Xmipp normalization time: " +
apDisplay.timeString(normtime))
### recombine particles to a single imagic stack
tmpstack = "tmp.xmippStack.hed"
apXmipp.gatherSingleFilesIntoStack(selfile, tmpstack)
apFile.moveStack(tmpstack, outStackPath)
### clean up directory
apFile.removeFile(selfile)
apFile.removeDir("partfiles")
def cleanupFiles(self, complete_refinements):
''' deletes all intermediate files for which database entries exitst '''
### cleanup directories (projection-matching and reference libraries)
os.chdir(self.runparams['package_params']['WorkingDir'])
if os.path.isdir("ProjMatchClasses"):
apFile.removeDir("ProjMatchClasses")
if os.path.isdir("ReferenceLibrary"):
apFile.removeDir("ReferenceLibrary")
### cleanup temp files
for file in glob.glob(os.path.join(self.resultspath, "*tmp.mrc")):
apFile.removeFile(file)
### cleanup files (.vol only now ... I'm leaving the .fsc, .sel, and .doc files *** Dmitry)
for reference_number, iters in complete_refinements.iteritems():
for iteration in iters:
for file in glob.glob(os.path.join("Iter_%d" % iteration, "*.vol")):
apFile.removeFile(file)
os.chdir(self.params['rundir'])
return
def cleanupFiles(self, complete_refinements):
''' deletes all intermediate files for which database entries exitst '''
### cleanup directories (projection-matching and reference libraries)
os.chdir(self.runparams['package_params']['WorkingDir'])
if os.path.isdir("ProjMatchClasses"):
apFile.removeDir("ProjMatchClasses")
if os.path.isdir("ReferenceLibrary"):
apFile.removeDir("ReferenceLibrary")
### cleanup temp files
for file in glob.glob(os.path.join(self.resultspath, "*tmp.mrc")):
apFile.removeFile(file)
### cleanup files (.vol only now ... I'm leaving the .fsc, .sel, and .doc files *** Dmitry)
for reference_number, iters in complete_refinements.iteritems():
for iteration in iters:
for file in glob.glob(
os.path.join("Iter_%d" % iteration, "*.vol")):
apFile.removeFile(file)
os.chdir(self.params['rundir'])
return
def cleanupFiles(self, complete_refinements):
''' deletes all intermediate files for which database entries exitst '''
### cleanup directories (grey-scale correction and initial reference generation)
os.chdir(self.runparams['package_params']['WorkingDir'])
for i in range(self.runparams['package_params']['NumberOfReferences']):
if os.path.isdir("GenerateSeed_%d" % (i+1)):
apFile.removeDir("GenerateSeed_%d" % (i+1))
if os.path.isdir("CorrectGreyscale"):
apFile.removeDir("CorrectGreyscale")
### cleanup temp files
for file in glob.glob(os.path.join(self.resultspath, "*tmp.mrc")):
apFile.removeFile(file)
### cleanup ML3D files (.vol, .xmp, and .proj files for now ... I'm leaving the .basis, .sel, .doc, .hist, and .log files *** Dmitry)
apFile.removeFile("corrected_reference.vol")
apFile.removeFile("initial_reference.vol")
os.chdir(self.ml3dpath)
delete_projections = True
for reference_number, iters in complete_refinements.iteritems():
apFile.removeFile("ml3d_it000000_vol%.6d.vol" % reference_number)
for iteration in iters:
for file in glob.glob("ml3d_it%.6d*.xmp" % iteration):
apFile.removeFile(os.path.join(self.ml3dpath, file))
for file in glob.glob("ml3d_it%.6d_vol%.6d.vol" % (iteration, reference_number)):
apFile.removeFile(os.path.join(self.ml3dpath, file))
if self.runparams['numiter'] != iters[-1]: ### if one of the iterations did not completely upload
delete_projections = False
if delete_projections is True:
for file in glob.glob("ml3d_ref*.xmp"):
apFile.removeFile(file)
for file in glob.glob("ml3d_lib*.proj"):
apFile.removeFile(file)
os.chdir(self.params['rundir'])
return
class uploadEmanProjectionMatchingRefinementScript(
reconUploader.generalReconUploader):
def __init__(self):
### DEFINE THE NAME OF THE PACKAGE
self.package = "EMAN"
self.multiModelRefinementRun = False
super(uploadEmanProjectionMatchingRefinementScript, self).__init__()
#=====================
def findLastCompletedIteration(self):
''' find the last iteration that finished in EMAN job, trying to make this more clever and look at actual volumes '''
lastiter = 0
if os.path.isdir(self.projmatchpath) is False:
apDisplay.printError(
"projection matching did not run. Please double check and restart the job"
)
files = glob.glob(
os.path.join(self.projmatchpath, "threed.[0-9]*a.mrc"))
if isinstance(files, list) and len(files) > 0:
for file in files:
m = re.search("[0-9]+", os.path.basename(file))
iternum = int(m.group(0))
if iternum > lastiter:
lastiter = iternum
### now open up in numpy and make sure that it's a valid file
vol = mrc.read(
os.path.join(self.projmatchpath, "threed.%da.mrc" % lastiter))
if vol.mean() == 0 and vol.max() == 0 and vol.min() == 0:
apDisplay.printError(
"there is something wrong with your volumes, make sure that the refinement ran correctly"
)
else:
apDisplay.printMsg("EMAN ran " + str(lastiter) + " iterations")
return lastiter
#======================
def findEmanCommandFile(self):
''' tries to find EMAN log file either through the jobinfo (retrieved through jobid) or runname '''
### find the job file
if 'jobinfo' in self.params and self.params['jobinfo'] is not None:
jobfile = os.path.join(self.params['rundir'],
self.params['jobinfo']['name'])
elif 'runname' in self.params:
jobfile = os.path.join(self.params['rundir'],
self.params['runname'] + ".appionsub.job")
else:
self.params['jobinfo'] = None
apDisplay.printError(
"no command file or jobfile found ... try uploading refinement as an external package"
)
basename, extension = os.path.splitext(jobfile)
commandfile = basename + ".commands"
if not os.path.isfile(commandfile):
apDisplay.printError(
"no command file found ... try uploading refinement as an external package"
)
return commandfile
#======================
def findEmanJobFile(self):
''' tries to find EMAN log file either through the jobinfo (retrieved through jobid) or runname '''
### find the job file
if 'jobinfo' in self.params and self.params['jobinfo'] is not None:
jobfile = os.path.join(self.params['rundir'],
self.params['jobinfo']['name'])
elif 'runname' in self.params:
jobfile = os.path.join(self.params['rundir'],
self.params['runname'] + ".appionsub.job")
else:
self.params['jobinfo'] = None
apDisplay.printError(
"no pickle file or jobfile found ... try uploading refinement as an external package"
)
if not os.path.isfile(jobfile):
apDisplay.printError(
"no pickle file or jobfile found ... try uploading refinement as an external package"
)
return jobfile
# logfile = os.path.join(self.params['rundir'], 'eman.log')
# if os.path.isfile(logfile):
# return logfile
# logfile = os.path.join(self.params['rundir'], '.emanlog')
# if os.path.isfile(logfile):
# return logfile
# apDisplay.printError("Could not find eman job or log file")
#=====================
def readParticleLog(self, iteration):
plogf = os.path.join(self.projmatchpath, "particle.log")
if not os.path.isfile(plogf):
apDisplay.printError("no particle.log file found")
f = open(plogf, 'r')
badprtls = []
n = str(int(iteration) + 1)
for line in f:
rline = line.rstrip()
if re.search("X\t\d+\t" + str(iteration) + "$", rline):
bits = rline.split()
badprtls.append(int(bits[1]))
# break out of into the next iteration
elif re.search("X\t\d+\t" + n + "$", rline):
break
f.close()
return badprtls
#=====================
def getEulersFromProj(self, iteration):
''' EMAN Eulers saved in proj.#.txt file for alt and az eulers, phi is in cls####.lst file '''
### get Eulers from the projection file
eulers = []
projfile = os.path.join(self.projmatchpath, "proj.%d.txt" % iteration)
if not os.path.exists(projfile):
apDisplay.printError("no projection file found for iteration %d" %
iteration)
f = open(projfile, 'r')
for line in f:
line = line[:-1] # remove newline at end
i = line.split()
angles = [i[1], i[2], i[3]]
eulers.append(angles)
f.close()
return eulers
#=====================
def setDefaultPackageParameters(self):
packageparams = {
'package': '',
'num': '',
'ang': '',
'mask': '',
'imask': '',
'pad': '',
'sym': '',
'maxshift': '',
'hard': '',
'classkeep': '',
'classiter': '',
'filt3d': '',
'shrink': '',
'euler2': '',
'xfiles': '',
'amask1': '',
'amask2': '',
'amask3': '',
'median': '',
'phasecls': '',
'fscls': '',
'refine': '',
'goodbad': '',
'perturb': '',
'hpfilter': '',
'lpfilter': '',
'msgpasskeep': '',
'msgpassminp': '',
}
return packageparams
#=====================
def parseFileForRunParameters(self):
''' PACKAGE-SPECIFIC FILE PARSER: if the parameters were not pickled, parse protocols script to determine projection-matching params '''
### parse out the refine command from the .emanlog to get the parameters for each iteration
#jobfile = self.findEmanJobFile()
jobfile = self.findEmanCommandFile()
apDisplay.printMsg("parsing eman log file: " + jobfile)
jobf = open(jobfile, 'r')
lines = jobf.readlines()
jobf.close()
itercount = 0
packageparams = self.setDefaultPackageParameters()
for i in range(len(lines)):
### if read a refine line, get the parameters
line = lines[i].rstrip()
if re.search("refine \d+ ", line):
itercount += 1
emanparams = line.split(' ')
if emanparams[0] is "#":
emanparams.pop(0)
### get rid of first "refine"
emanparams.pop(0)
packageparams['num'] += emanparams[0]
packageparams['sym'] = ''
for p in emanparams:
elements = p.strip().split('=')
if elements[0] == 'ang':
packageparams['ang'] += str(elements[1]) + " "
elif elements[0] == 'mask':
packageparams['mask'] += str(elements[1]) + " "
elif elements[0] == 'imask':
packageparams['imask'] += str(elements[1]) + " "
elif elements[0] == 'pad':
packageparams['pad'] += str(elements[1]) + " "
elif elements[0] == 'sym':
packageparams['sym'] += str(elements[1]) + " "
elif elements[0] == 'maxshift':
packageparams['maxshift'] += str(elements[1]) + " "
elif elements[0] == 'hard':
packageparams['hard'] += str(elements[1]) + " "
elif elements[0] == 'classkeep':
packageparams['classkeep'] += str(
elements[1].strip()) + " "
elif elements[0] == 'classiter':
packageparams['classiter'] += str(elements[1]) + " "
elif elements[0] == 'filt3d':
packageparams['filt3d'] += str(elements[1]) + " "
elif elements[0] == 'shrink':
packageparams['shrink'] += str(elements[1]) + " "
elif elements[0] == 'euler2':
packageparams['euler2'] += str(elements[1]) + " "
elif elements[0] == 'xfiles':
### trying to extract "xfiles" as entered into emanJobGen.php
values = elements[1]
apix, mass, alito = values.split(',')
packageparams['xfiles'] += str(mass) + " "
elif elements[0] == 'amask1':
packageparams['amask1'] += str(elements[1]) + " "
elif elements[0] == 'amask2':
packageparams['amask2'] += str(elements[1]) + " "
elif elements[0] == 'amask3':
packageparams['amask3'] += str(elements[1]) + " "
elif elements[0] == 'median':
packageparams['median'] += str(True) + " "
elif elements[0] == 'phasecls':
packageparams['phasecls'] += str(True) + " "
elif elements[0] == 'fscls':
packageparams['fscls'] += str(True) + " "
elif elements[0] == 'refine':
packageparams['refine'] += str(True) + " "
elif elements[0] == 'goodbad':
packageparams['goodbad'] += str(True) + " "
elif elements[0] == 'perturb':
packageparams['perturb'] += str(True) + " "
### Coran / MsgP performed 6 lines after standard refinement
if re.search("coran_for_cls.py", lines[i + 6]):
packageparams['package'] += 'EMAN/SpiCoran '
apDisplay.printMsg(
"correspondence analysis was performed on iteration %d to refine averages"
% itercount)
elif re.search("msgPassing_subClassification.py",
lines[i + 6]):
packageparams['package'] += 'EMAN/MsgP '
apDisplay.printMsg(
"Message-passing was performed on iteration %d to refine averages"
% itercount)
msgpassparams = lines[i + 6].split()
for p in msgpassparams:
elements = p.split('=')
if elements[0] == 'corCutOff':
packageparams['msgpasskeep'] += str(
elements[1]) + " "
elif elements[0] == 'minNumOfPtcls':
packageparams['msgpassminp'] += str(
elements[1]) + " "
else:
packageparams['package'] += 'EMAN '
apDisplay.printColor("Found %d iterations" % itercount, "green")
### set global parameters
runparams = {}
runparams['numiter'] = itercount
runparams['mask'] = packageparams['mask']
runparams['imask'] = packageparams['imask']
runparams['symmetry'] = apSymmetry.findSymmetry(packageparams['sym'])
runparams['angularSamplingRate'] = packageparams['ang']
runparams['package_params'] = packageparams
return runparams
#=====================
def createParticleDataFile(self, iteration, badparticles):
''' puts all relevant particle information into a single text file that can be read by the uploader '''
os.chdir(self.projmatchpath)
### extract tarfile
tmpdir = "tmp"
apParam.createDirectory(tmpdir)
clsf = "cls.%d.tar" % iteration
try:
clstar = tarfile.open(clsf)
clsnames = clstar.getnames()
clslist = clstar.getmembers()
except tarfile.ReadError, e:
apDisplay.printError("cannot open tarfile %s" % clsf)
for clsfile in clslist:
clstar.extract(clsfile, tmpdir)
clstar.close()
### write data in appion format to input file for uploading to the database
particledataf = open(
os.path.join(
self.resultspath, "particle_data_%s_it%.3d_vol001.txt" %
(self.params['timestamp'], iteration)), "w")
# particledataf.write("### column info: ")
particledataf.write("#%8s" % "partnum")
particledataf.write("%10s" % "phi")
particledataf.write("%10s" % "theta")
particledataf.write("%10s" % "omega")
particledataf.write("%10s" % "shiftx")
particledataf.write("%10s" % "shifty")
# particledataf.write("(7) mirror")
particledataf.write("%8s" % "3D_ref#")
particledataf.write("%8s" % "2D_cls#")
particledataf.write("%10s" % "qfact")
particledataf.write("%8s" % "keptp")
particledataf.write("%8s\n" % "p_keptp")
### read parameters from extracted cls files
eulers = self.getEulersFromProj(iteration)
for i, cls in enumerate(clsnames):
f = open(os.path.join(tmpdir, cls))
coranfail = False
for line in f:
if re.search("start", line):
ali = line.split()
prtlnum = int(ali[0])
# check if bad particle
if prtlnum in badparticles:
keptp = 0
else:
keptp = 1
other = ali[3].split(',')
### SPIDER coran kept particle
refine_keptp = 0
if apRecon.getComponentFromVector(
self.runparams['package_params']['package'],
iteration - 1) == 'EMAN/SpiCoran':
if len(other) > 4:
refine_keptp = bool(int(other[4]))
else:
if coranfail is False:
apDisplay.printWarning(
"Coran failed on this iteration")
coranfail = True
### message passing kept particle
if apRecon.getComponentFromVector(
self.runparams['package_params']['package'],
iteration - 1) == 'EMAN/MsgP' and len(ali) > 4:
refine_keptp = bool(int(ali[4]))
### write the results to Appion format
alt = float(eulers[i][0])
az = float(eulers[i][1])
Ephi = float(other[0]) * 180. / math.pi
mirror = int(float(other[3]))
if mirror:
alt, az, Ephi = apEulerCalc.calculate_equivalent_EMANEulers_without_flip(
alt, az, Ephi)
phi, theta, omega = apEulerCalc.convertEmanEulersToXmipp(
alt, az, Ephi)
full_sym_name = self.runparams['symmetry']['symmetry']
if 'Icos' in full_sym_name:
# EMAN Icos is always 532 orientation, while particlefile is set to 235
phi, theta, omega = apEulerCalc.convert3DEMIcosEulerFrom532(
'Icos (2 3 5) Viper/3DEM', phi, theta, omega)
particledataf.write(
"%9d" % (int(prtlnum) + 1)
) ### NOTE: IT IS IMPORTANT TO START WITH 1, OTHERWISE STACKMAPPING IS WRONG!!!
particledataf.write("%10.4f" % phi)
particledataf.write("%10.4f" % theta)
particledataf.write("%10.4f" % omega)
particledataf.write("%10.4f" % float(other[1]))
particledataf.write("%10.4f" % float(other[2]))
# particledataf.write("%6d" % int(float(other[3]))) ### mirror is no longer accounted for
particledataf.write("%8d" % 1)
particledataf.write("%8d" % int(i))
particledataf.write("%10.2f" % float(ali[2].strip(',')))
particledataf.write("%8d" % keptp)
particledataf.write("%8d\n" % refine_keptp)
particledataf.close()
### remove tmp directory
apFile.removeDir(tmpdir)
os.chdir(self.basepath)
return
