def writeToStack(self,partarray):
if self.partperiter == 0:
arrayshape = partarray.shape
partperiter = int(1e9/(arrayshape[0]*arrayshape[1])/16.)
if partperiter > 4096:
partperiter = 4096
self.partperiter = partperiter
apDisplay.printMsg("Using %d particle per iteration"%(partperiter))
stackroot = self.outstackfile[:-4]
imgnum = self.imgnum
index = imgnum % self.partperiter
### Process images
startmem = mem.active()
index = imgnum % self.partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active()-startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing single particles to file "+stackname)
self.stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(self.stackarray, stackname, msg=False)
perpart = (time.time()-self.starttime)/imgnum
apDisplay.printColor("%d :: %.1fM mem :: %s/part "%
(imgnum+1, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart)),
"blue")
self.stackarray = []
### merge particles
self.stackarray.append(partarray)
def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
### stack data
stackdata = apStack.getOnlyStackData(self.params['stackid'])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]['particle']['selectionrun']
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq['oldselectionrun'] = oldselectrun
manualparamsq['trace'] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['hidden'] = False
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrunq['session'] = apStack.getSessionDataFromStackId(self.params['stackid'])
selectrunq['manparams'] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count%100 == 0:
perpart = (time.time()-t0)/float(count+1)
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(count, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-count))), "blue")
oldpartdata = stackpart['particle']
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq['selectionrun'] = selectrunq
if self.params['commit'] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s"%(apDisplay.timeString(time.time()-t0)))
def makeEulerDoc(self, tiltParticlesData, cnum):
count = 0
eulerfile = os.path.join(self.params['rundir'], str(cnum),
"eulersdoc" + self.timestamp + ".spi")
eulerf = open(eulerfile, "w")
apDisplay.printMsg("Creating Euler angles doc file")
starttime = time.time()
tiltParticlesData.sort(self.sortTiltParticlesData)
startmem = mem.active()
for stackpartdata in tiltParticlesData:
count += 1
if count % 50 == 0:
sys.stderr.write(".")
eulerf.flush()
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 3:
apDisplay.printColor(
"Memory increase: %d MB/part" % (memdiff), "red")
tiltrot, theta, notrot, tiltangle = apTiltPair.getParticleTiltRotationAnglesOTR(
stackpartdata)
### Hack for OTR to work ( bad tilt axis angle from tilt picker )
tiltrot = -7.0
notrot = -7.0
inplane, mirror = self.getParticleInPlaneRotation(stackpartdata)
totrot = -1.0 * (notrot + inplane)
if mirror is True:
#theta flips to the back
tiltangle = -1.0 * tiltangle + 180 #tiltangle = tiltangle + 180.0 #theta
totrot = -1.0 * totrot - 180.0 #phi
tiltrot = tiltrot + 180 #tiltrot = -1.0 * tiltrot + 180.0 #psi
while totrot < 0:
totrot += 360.0
### this is the original eman part num; count is new part num
partnum = stackpartdata['particleNumber'] - 1
line = operations.spiderOutLine(count,
[tiltrot, tiltangle, totrot])
eulerf.write(line)
eulerf.close()
apDisplay.printColor(
"\nFinished Euler angle doc file in " +
apDisplay.timeString(time.time() - starttime), "cyan")
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 0.1:
apDisplay.printColor("Memory increase: %.2f MB/part" % (memdiff),
"red")
return eulerfile
def __init__(self,optargs=sys.argv[1:],quiet=False):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.quiet = quiet
self.timestamp = apParam.makeTimestamp()
if not self.quiet:
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
if not self.quiet:
apDisplay.printMsg("Function name: "+self.functionname)
apParam.setUmask()
self.parsePythonPath()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is %.2f, wait for %.1f second " % (round(loadavg,2),loadavg**2))
time.sleep(loadavg**2)
apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
### setup default parser: run directory, etc.
self.setParams(optargs)
self.checkConflicts()
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def __init__(self,optargs=sys.argv[1:],quiet=False):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.quiet = quiet
# self.timestamp = apParam.makeTimestamp()
# if not self.quiet:
# apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
if not self.quiet:
apDisplay.printMsg("Function name: "+self.functionname)
apParam.setUmask()
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is %.2f, wait for %.1f second " % (round(loadavg,2),loadavg**2))
# time.sleep(loadavg**2)
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
### setup default parser: run directory, etc.
self.setParams(optargs)
self.checkConflicts()
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def __init__(self,
optargs=sys.argv[1:],
quiet=False,
useglobalparams=True,
maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: " + self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: " + self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: " + self.appiondir)
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# loadsquared = loadavg*loadavg
# time.sleep(loadavg)
# apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs, useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(
2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def createDefaultStats(self):
self.stats = {}
self.stats['starttime'] = time.time()
self.stats['count'] = 1
self.stats['lastcount'] = 0
self.stats['startmem'] = mem.active()
self.stats['memleak'] = 0
self.stats['peaksum'] = 0
self.stats['lastpeaks'] = None
self.stats['imagesleft'] = 1
self.stats['peaksumsq'] = 0
self.stats['timesum'] = 0
self.stats['timesumsq'] = 0
self.stats['skipcount'] = 0
self.stats['waittime'] = 0
self.stats['lastimageskipped'] = False
self.stats['notpair'] = 0
self.stats['memlist'] = [mem.active()]
def createDefaultStats(self):
self.stats = {}
self.stats['starttime'] = time.time()
self.stats['count'] = 1
self.stats['lastcount'] = 0
self.stats['startmem'] = mem.active()
self.stats['memleak'] = 0
self.stats['peaksum'] = 0
self.stats['lastpeaks'] = None
self.stats['imagesleft'] = 1
self.stats['peaksumsq'] = 0
self.stats['timesum'] = 0
self.stats['timesumsq'] = 0
self.stats['skipcount'] = 0
self.stats['waittime'] = 0
self.stats['lastimageskipped'] = False
self.stats['notpair'] = 0
self.stats['memlist'] = [mem.active()]
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: "+self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: "+self.appiondir)
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# loadsquared = loadavg*loadavg
# time.sleep(loadavg)
# apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs,useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### any custom init functions go here
self.onInit()
def makeEulerDoc(self, tiltParticlesData, cnum):
count = 0
eulerfile = os.path.join(self.params["rundir"], str(cnum), "eulersdoc" + self.timestamp + ".spi")
eulerf = open(eulerfile, "w")
apDisplay.printMsg("Creating Euler angles doc file")
starttime = time.time()
tiltParticlesData.sort(self.sortTiltParticlesData)
startmem = mem.active()
for stackpartdata in tiltParticlesData:
count += 1
if count % 50 == 0:
sys.stderr.write(".")
eulerf.flush()
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 3:
apDisplay.printColor("Memory increase: %d MB/part" % (memdiff), "red")
tiltrot, theta, notrot, tiltangle = apTiltPair.getParticleTiltRotationAnglesOTR(stackpartdata)
### Hack for OTR to work ( bad tilt axis angle from tilt picker )
tiltrot = -7.0
notrot = -7.0
inplane, mirror = self.getParticleInPlaneRotation(stackpartdata)
totrot = -1.0 * (notrot + inplane)
if mirror is True:
# theta flips to the back
tiltangle = -1.0 * tiltangle + 180 # tiltangle = tiltangle + 180.0 #theta
totrot = -1.0 * totrot - 180.0 # phi
tiltrot = tiltrot + 180 # tiltrot = -1.0 * tiltrot + 180.0 #psi
while totrot < 0:
totrot += 360.0
### this is the original eman part num; count is new part num
partnum = stackpartdata["particleNumber"] - 1
line = operations.spiderOutLine(count, [tiltrot, tiltangle, totrot])
eulerf.write(line)
eulerf.close()
apDisplay.printColor(
"\nFinished Euler angle doc file in " + apDisplay.timeString(time.time() - starttime), "cyan"
)
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 0.1:
apDisplay.printColor("Memory increase: %.2f MB/part" % (memdiff), "red")
return eulerfile
def close(self):
self.onClose()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
time.sleep(loadavg**2)
apParam.closeFunctionLog(functionname=self.functionname,
logfile=self.logfile, msg=(not self.quiet))
if self.quiet is False:
apDisplay.printMsg("Ended at "+time.strftime("%a, %d %b %Y %H:%M:%S"))
apDisplay.printMsg("Memory increase during run: %.3f MB"%((mem.active()-self.startmem)/1024.0))
apDisplay.printColor("Total run time:\t"+apDisplay.timeString(time.time()-self.t0),"green")
def writeToStack(self, partarray):
if self.partperiter == 0:
arrayshape = partarray.shape
partperiter = int(1e9 / (arrayshape[0] * arrayshape[1]) / 16.)
if partperiter > 4096:
partperiter = 4096
self.partperiter = partperiter
apDisplay.printMsg("Using %d particle per iteration" %
(partperiter))
stackroot = self.outstackfile[:-4]
imgnum = self.imgnum
index = imgnum % self.partperiter
### Process images
startmem = mem.active()
index = imgnum % self.partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active() - startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing single particles to file " +
stackname)
self.stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(self.stackarray, stackname, msg=False)
perpart = (time.time() - self.starttime) / imgnum
apDisplay.printColor(
"%d :: %.1fM mem :: %s/part " %
(imgnum + 1, (mem.active() - startmem) / 1024.,
apDisplay.timeString(perpart)), "blue")
self.stackarray = []
### merge particles
self.stackarray.append(partarray)
def _checkMemLeak(self):
"""
unnecessary code for determining if the program is eating memory over time
"""
### Memory leak code:
#self.stats['memlist'].append(mem.mySize()/1024)
self.stats['memlist'].append(mem.active())
memfree = mem.free()
swapfree = mem.swapfree()
minavailmem = 64 * 1024
# 64 MB, size of one image
if (memfree < minavailmem):
apDisplay.printWarning("Memory is low (" +
str(int(memfree / 1024)) +
"MB): there is probably a memory leak")
if (self.stats['count'] > 15):
memlist = self.stats['memlist'][-15:]
n = len(memlist)
gain = (memlist[n - 1] - memlist[0]) / 1024.0
sumx = n * (n - 1.0) / 2.0
sumxsq = n * (n - 1.0) * (2.0 * n - 1.0) / 6.0
sumy = 0.0
sumxy = 0.0
sumysq = 0.0
for i in range(n):
value = float(memlist[i]) / 1024.0
sumxy += float(i) * value
sumy += value
sumysq += value**2
###
stdx = math.sqrt(n * sumxsq - sumx**2)
stdy = math.sqrt(n * sumysq - sumy**2)
rho = float(n * sumxy - sumx * sumy) / float(stdx * stdy + 1e-6)
slope = float(n * sumxy - sumx * sumy) / float(n * sumxsq -
sumx * sumx)
memleak = rho * slope
###
if (self.stats['memleak'] > 3 and slope > 20 and memleak > 512
and gain > 2048):
apDisplay.printWarning("Memory leak of " +
str(round(memleak, 2)) + "MB")
elif (memleak > 32):
self.stats['memleak'] += 1
apDisplay.printWarning("substantial memory leak " +
str(round(memleak, 2)) + "MB")
print "(", str(n), round(slope,
5), round(rho, 5), round(gain, 2), ")"
def _checkMemLeak(self):
"""
unnecessary code for determining if the program is eating memory over time
"""
### Memory leak code:
#self.stats['memlist'].append(mem.mySize()/1024)
self.stats['memlist'].append(mem.active())
memfree = mem.free()
swapfree = mem.swapfree()
minavailmem = 64*1024; # 64 MB, size of one image
if(memfree < minavailmem):
apDisplay.printWarning("Memory is low ("+str(int(memfree/1024))+"MB): there is probably a memory leak")
if(self.stats['count'] > 15):
memlist = self.stats['memlist'][-15:]
n = len(memlist)
gain = (memlist[n-1] - memlist[0])/1024.0
sumx = n*(n-1.0)/2.0
sumxsq = n*(n-1.0)*(2.0*n-1.0)/6.0
sumy = 0.0; sumxy = 0.0; sumysq = 0.0
for i in range(n):
value = float(memlist[i])/1024.0
sumxy += float(i)*value
sumy += value
sumysq += value**2
###
stdx = math.sqrt(n*sumxsq - sumx**2)
stdy = math.sqrt(n*sumysq - sumy**2)
rho = float(n*sumxy - sumx*sumy)/float(stdx*stdy+1e-6)
slope = float(n*sumxy - sumx*sumy)/float(n*sumxsq - sumx*sumx)
memleak = rho*slope
###
if(self.stats['memleak'] > 3 and slope > 20 and memleak > 512 and gain > 2048):
apDisplay.printWarning("Memory leak of "+str(round(memleak,2))+"MB")
elif(memleak > 32):
self.stats['memleak'] += 1
apDisplay.printWarning("substantial memory leak "+str(round(memleak,2))+"MB")
print "(",str(n),round(slope,5),round(rho,5),round(gain,2),")"
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"):
"""
takes a selfile and creates an EMAN stack
"""
selfile = os.path.abspath(selfile)
stackfile = os.path.abspath(stackfile)
if stackfile[-4:] != ".hed":
apDisplay.printWarning("Stack file does not end in .hed")
stackfile = stackfile[:-4] + ".hed"
apDisplay.printColor("Merging files into a stack, this can take a while",
"cyan")
starttime = time.time()
if not os.path.isfile(selfile):
apDisplay.printError("selfile does not exist: " + selfile)
### Process selfile
fh = open(selfile, 'r')
filelist = []
for line in fh:
sline = line.strip()
if sline:
args = sline.split()
if (len(args) > 1):
filename = args[0].strip()
filelist.append(filename)
fh.close()
### Set variables
boxsize = apFile.getBoxSize(filelist[0])
partperiter = int(1e9 / (boxsize[0]**2) / 16.)
if partperiter > 4096:
partperiter = 4096
apDisplay.printMsg("Using %d particle per iteration" % (partperiter))
numpart = len(filelist)
if numpart < partperiter:
partperiter = numpart
### Process images
imgnum = 0
stacklist = []
stackroot = stackfile[:-4]
### get memory in kB
startmem = mem.active()
while imgnum < len(filelist):
filename = filelist[imgnum]
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active() - startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
#print "img num", imgnum
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing single particles to file " +
stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
perpart = (time.time() - starttime) / imgnum
apDisplay.printColor(
"part %d of %d :: %.1fM mem :: %s/part :: %s remain" %
(imgnum + 1, numpart, (mem.active() - startmem) / 1024.,
apDisplay.timeString(perpart),
apDisplay.timeString(perpart * (numpart - imgnum))),
"blue")
stackarray = []
### merge particles
if filetype == "mrc":
partimg = mrc.read(filename)
else:
partimg = spider.read(filename)
stackarray.append(partimg)
imgnum += 1
### write remaining particles to file
sys.stderr.write("\n")
stackname = "%s-%d.hed" % (stackroot, imgnum)
apDisplay.printMsg("writing particles to file " + stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(stackfile, warn=False)
apImagicFile.mergeStacks(stacklist, stackfile)
print stackfile
filepart = apFile.numImagesInStack(stackfile)
if filepart != numpart:
apDisplay.printError(
"number merged particles (%d) not equal number expected particles (%d)"
% (filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor(
"merged %d particles in %s" %
(imgnum, apDisplay.timeString(time.time() - starttime)), "cyan")
def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq["name"] = self.params["runname"]
selectrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
if self.params["ddstack"]:
self.other_ddstack_used = []
self.dd = apDDprocess.DDStackProcessing()
self.dd.setDDStackRun(self.params["ddstack"])
self.newddstackrun = self.dd.getDDStackRun(show_msg=True)
### stack data
stackdata = apStack.getOnlyStackData(self.params["stackid"])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params["stackid"], msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]["particle"]["selectionrun"]
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq["diam"] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq["oldselectionrun"] = oldselectrun
manualparamsq["trace"] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq["name"] = self.params["runname"]
selectrunq["hidden"] = False
selectrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
selectrunq["session"] = apStack.getSessionDataFromStackId(self.params["stackid"])
selectrunq["manparams"] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count % 100 == 0:
perpart = (time.time() - t0) / float(count + 1)
apDisplay.printColor(
"part %d of %d :: %.1fM mem :: %s/part :: %s remain"
% (
count,
numpart,
(mem.active() - startmem) / 1024.0,
apDisplay.timeString(perpart),
apDisplay.timeString(perpart * (numpart - count)),
),
"blue",
)
oldpartdata = stackpart["particle"]
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq["selectionrun"] = selectrunq
if self.params["ddstack"]:
newimagedata = self.getNewImageFromDDStack(oldpartdata["image"])
if newimagedata is False:
# no pick transferred
continue
newpartq["image"] = newimagedata
if self.params["commit"] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s" % (apDisplay.timeString(time.time() - t0)))
try:
print "====================="
result = libcv.MatchImages(image1, image2, minsize, maxsize, blur, sharpen, WoB, BoW)
print "====================="
return result
except:
return numpy.zeros([3,3], dtype=numpy.float32)
print ""
if __name__ == "__main__":
image2 = mrc.read("untilted.mrc")
image1 = mrc.read("tilted.mrc")
startmem = mem.active()
lastmem = startmem
memlist = []
for i in range(150):
result = MatchImages(image1, image2)
#checkLibCVResult(result)
#findTilt(result)
#print numpy.array(result*1000, dtype=numpy.int32)
memmeg1 = (mem.active()-startmem)/1024.0
memmeg2 = (mem.active()-lastmem)/1024.0
lastmem = mem.active()
print "-->\tMEM: "+str(int(memmeg1))+" MB "
memlist.append(memmeg2)
time.sleep(1)
memarr = numpy.asarray(memlist, dtype=numpy.float32)
print memarr
def gatherSingleFilesIntoStack(selfile, stackfile, filetype="spider"):
"""
takes a selfile and creates an EMAN stack
"""
selfile = os.path.abspath(selfile)
stackfile = os.path.abspath(stackfile)
if stackfile[-4:] != ".hed":
apDisplay.printWarning("Stack file does not end in .hed")
stackfile = stackfile[:-4]+".hed"
apDisplay.printColor("Merging files into a stack, this can take a while", "cyan")
starttime = time.time()
if not os.path.isfile(selfile):
apDisplay.printError("selfile does not exist: "+selfile)
### Process selfile
fh = open(selfile, 'r')
filelist = []
for line in fh:
sline = line.strip()
if sline:
args=sline.split()
if (len(args)>1):
filename = args[0].strip()
filelist.append(filename)
fh.close()
### Set variables
boxsize = apFile.getBoxSize(filelist[0])
partperiter = int(1e9/(boxsize[0]**2)/16.)
if partperiter > 4096:
partperiter = 4096
apDisplay.printMsg("Using %d particle per iteration"%(partperiter))
numpart = len(filelist)
if numpart < partperiter:
partperiter = numpart
### Process images
imgnum = 0
stacklist = []
stackroot = stackfile[:-4]
### get memory in kB
startmem = mem.active()
while imgnum < len(filelist):
filename = filelist[imgnum]
index = imgnum % partperiter
if imgnum % 100 == 0:
sys.stderr.write(".")
#sys.stderr.write("%03.1fM %d\n"%((mem.active()-startmem)/1024., index))
if mem.active()-startmem > 2e6:
apDisplay.printWarning("Out of memory")
if index < 1:
#print "img num", imgnum
### deal with large stacks, reset loop
if imgnum > 0:
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing single particles to file "+stackname)
stacklist.append(stackname)
apFile.removeStack(stackname, warn=False)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
perpart = (time.time()-starttime)/imgnum
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(imgnum+1, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-imgnum))), "blue")
stackarray = []
### merge particles
if filetype == "mrc":
partimg = mrc.read(filename)
else:
partimg = spider.read(filename)
stackarray.append(partimg)
imgnum += 1
### write remaining particles to file
sys.stderr.write("\n")
stackname = "%s-%d.hed"%(stackroot, imgnum)
apDisplay.printMsg("writing particles to file "+stackname)
stacklist.append(stackname)
apImagicFile.writeImagic(stackarray, stackname, msg=False)
### merge stacks
apFile.removeStack(stackfile, warn=False)
apImagicFile.mergeStacks(stacklist, stackfile)
print stackfile
filepart = apFile.numImagesInStack(stackfile)
if filepart != numpart:
apDisplay.printError("number merged particles (%d) not equal number expected particles (%d)"%
(filepart, numpart))
for stackname in stacklist:
apFile.removeStack(stackname, warn=False)
### summarize
apDisplay.printColor("merged %d particles in %s"%(imgnum, apDisplay.timeString(time.time()-starttime)), "cyan")
try:
print "====================="
result = libcv.MatchImages(image1, image2, minsize, maxsize, blur,
sharpen, WoB, BoW)
print "====================="
return result
except:
return numpy.zeros([3, 3], dtype=numpy.float32)
print ""
if __name__ == "__main__":
image2 = mrc.read("untilted.mrc")
image1 = mrc.read("tilted.mrc")
startmem = mem.active()
lastmem = startmem
memlist = []
for i in range(150):
result = MatchImages(image1, image2)
#checkLibCVResult(result)
#findTilt(result)
#print numpy.array(result*1000, dtype=numpy.int32)
memmeg1 = (mem.active() - startmem) / 1024.0
memmeg2 = (mem.active() - lastmem) / 1024.0
lastmem = mem.active()
print "-->\tMEM: " + str(int(memmeg1)) + " MB "
memlist.append(memmeg2)
time.sleep(1)
memarr = numpy.asarray(memlist, dtype=numpy.float32)
print memarr
def __init__(self,
optargs=sys.argv[1:],
quiet=False,
useglobalparams=True,
maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.clusterjobdata = None
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: " + self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: " + self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: " + self.appiondir)
self.parsePythonPath()
loadavg = os.getloadavg()[0]
if loadavg > 2.0:
apDisplay.printMsg("Load average is high " +
str(round(loadavg, 2)))
loadsquared = loadavg * loadavg
time.sleep(loadavg)
apDisplay.printMsg("New load average " +
str(round(os.getloadavg()[0], 2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs, useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
### check if user wants to print help message
if 'commit' in self.params and self.params['commit'] is True:
apDisplay.printMsg("Committing data to database")
else:
apDisplay.printWarning("Not committing data to database")
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(
2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### upload command line parameters to database
self.uploadScriptData()
### any custom init functions go here
self.onInit()
def getGoodAlignParticles(self):
includeParticle = []
tiltParticlesData = []
nopairParticle = 0
excludeParticle = 0
badmirror = 0
badscore = 0
apDisplay.printMsg("Sorting particles from classes at "+time.asctime())
count = 0
startmem = mem.active()
t0 = time.time()
if self.params['clusterid'] is not None:
### method 1: get particles from clustering data
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq['clusterstack'] = appiondata.ApClusteringStackData.direct_query(self.params['clusterid'])
clusterpartdatas = clusterpartq.query()
apDisplay.printMsg("Sorting "+str(len(clusterpartdatas))+" clustered particles")
for clustpart in clusterpartdatas:
count += 1
if count%50 == 0:
sys.stderr.write(".")
memdiff = (mem.active()-startmem)/count/1024.0
if memdiff > 3:
apDisplay.printColor("Memory increase: %d MB/part"%(memdiff), "red")
#write to text file
clustnum = clustpart['refnum']-1
if self.params['minscore'] is not None:
if ( clustpart['alignparticle']['score'] is not None
and clustpart['alignparticle']['score'] < self.params['minscore'] ):
badscore += 1
continue
elif ( clustpart['alignparticle']['spread'] is not None
and clustpart['alignparticle']['spread'] < self.params['minscore'] ):
badscore += 1
continue
if clustnum in self.classlist:
notstackpartnum = clustpart['alignparticle']['stackpart']['particleNumber']
tiltstackpartdata = apTiltPair.getStackParticleTiltPair(self.params['notstackid'],
notstackpartnum, self.params['tiltstackid'])
if tiltstackpartdata is None:
nopairParticle += 1
continue
tiltrot, theta, notrot, tiltangle = apTiltPair.getParticleTiltRotationAngles(tiltstackpartdata)
if tiltrot is None:
apDisplay.printWarning("BAD particle "+str(tiltstackpartdata))
nopairParticle += 1
continue
else:
inplane, mirror = self.getParticleInPlaneRotation(tiltstackpartdata)
if ( self.params['mirror'] == "all"
or (self.params['mirror'] == "no" and mirror is False)
or (self.params['mirror'] == "yes" and mirror is True) ):
emantiltstackpartnum = tiltstackpartdata['particleNumber']-1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(tiltstackpartdata)
if self.params['numpart'] is not None and len(includeParticle) > self.params['numpart']:
break
else:
badmirror += 1
else:
excludeParticle += 1
else:
### method 2: get particles from alignment data
alignpartq = appiondata.ApAlignParticleData()
alignpartq['alignstack'] = self.alignstackdata
alignpartdatas = alignpartq.query()
apDisplay.printMsg("Sorting "+str(len(alignpartdatas))+" aligned particles")
for alignpart in alignpartdatas:
count += 1
if count%50 == 0:
sys.stderr.write(".")
memdiff = (mem.active()-startmem)/count/1024.0
if memdiff > 3:
apDisplay.printColor("Memory increase: %d MB/part"%(memdiff), "red")
#write to text file
alignnum = alignpart['ref']['refnum']-1
if ( self.params['minscore'] is not None
and alignpart['score'] is not None
and alignpart['score'] < self.params['minscore'] ):
badscore += 1
continue
if alignnum in self.classlist:
notstackpartnum = alignpart['stackpart']['particleNumber']
tiltstackpartdata = apTiltPair.getStackParticleTiltPair(self.params['notstackid'],
notstackpartnum, self.params['tiltstackid'])
if tiltstackpartdata is None:
nopairParticle += 1
else:
inplane, mirror = self.getParticleInPlaneRotation(tiltstackpartdata)
if ( self.params['mirror'] == "all"
or (self.params['mirror'] == "no" and mirror is False)
or (self.params['mirror'] == "yes" and mirror is True) ):
emantiltstackpartnum = tiltstackpartdata['particleNumber']-1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(tiltstackpartdata)
if self.params['numpart'] is not None and len(includeParticle) > self.params['numpart']:
break
else:
badmirror += 1
else:
excludeParticle += 1
### end methods
includeParticle.sort()
### messages
if time.time()-t0 > 1.0:
apDisplay.printMsg("\nSorting time: "+apDisplay.timeString(time.time()-t0))
apDisplay.printMsg("Keeping "+str(len(includeParticle))+" and excluding \n\t"
+str(excludeParticle)+" particles with "+str(nopairParticle)+" unpaired particles")
if badmirror > 0:
apDisplay.printMsg("Particles with bad mirrors: %d"%(badmirror))
if badscore > 0:
apDisplay.printColor("Particles with bad scores: %d"%(badscore), "cyan")
if len(includeParticle) < 1:
apDisplay.printError("No particles were kept")
memdiff = (mem.active()-startmem)/count/1024.0
if memdiff > 0.1:
apDisplay.printColor("Memory increase: %.2f MB/part"%(memdiff), "red")
return includeParticle, tiltParticlesData
def getGoodAlignParticles(self):
includeParticle = []
tiltParticlesData = []
nopairParticle = 0
excludeParticle = 0
badmirror = 0
badscore = 0
apDisplay.printMsg("Sorting particles from classes at " +
time.asctime())
count = 0
startmem = mem.active()
t0 = time.time()
if self.params['clusterid'] is not None:
### method 1: get particles from clustering data
clusterpartq = appiondata.ApClusteringParticleData()
clusterpartq[
'clusterstack'] = appiondata.ApClusteringStackData.direct_query(
self.params['clusterid'])
clusterpartdatas = clusterpartq.query()
apDisplay.printMsg("Sorting " + str(len(clusterpartdatas)) +
" clustered particles")
for clustpart in clusterpartdatas:
count += 1
if count % 50 == 0:
sys.stderr.write(".")
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 3:
apDisplay.printColor(
"Memory increase: %d MB/part" % (memdiff), "red")
#write to text file
clustnum = clustpart['refnum'] - 1
if self.params['minscore'] is not None:
if (clustpart['alignparticle']['score'] is not None
and clustpart['alignparticle']['score'] <
self.params['minscore']):
badscore += 1
continue
elif (clustpart['alignparticle']['spread'] is not None
and clustpart['alignparticle']['spread'] <
self.params['minscore']):
badscore += 1
continue
if clustnum in self.classlist:
notstackpartnum = clustpart['alignparticle']['stackpart'][
'particleNumber']
tiltstackpartdata = apTiltPair.getStackParticleTiltPair(
self.params['notstackid'], notstackpartnum,
self.params['tiltstackid'])
if tiltstackpartdata is None:
nopairParticle += 1
continue
tiltrot, theta, notrot, tiltangle = apTiltPair.getParticleTiltRotationAngles(
tiltstackpartdata)
if tiltrot is None:
apDisplay.printWarning("BAD particle " +
str(tiltstackpartdata))
nopairParticle += 1
continue
else:
inplane, mirror = self.getParticleInPlaneRotation(
tiltstackpartdata)
if (self.params['mirror'] == "all" or
(self.params['mirror'] == "no" and mirror is False)
or (self.params['mirror'] == "yes"
and mirror is True)):
emantiltstackpartnum = tiltstackpartdata[
'particleNumber'] - 1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(tiltstackpartdata)
if self.params['numpart'] is not None and len(
includeParticle) > self.params['numpart']:
break
else:
badmirror += 1
else:
excludeParticle += 1
else:
### method 2: get particles from alignment data
alignpartq = appiondata.ApAlignParticleData()
alignpartq['alignstack'] = self.alignstackdata
alignpartdatas = alignpartq.query()
apDisplay.printMsg("Sorting " + str(len(alignpartdatas)) +
" aligned particles")
for alignpart in alignpartdatas:
count += 1
if count % 50 == 0:
sys.stderr.write(".")
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 3:
apDisplay.printColor(
"Memory increase: %d MB/part" % (memdiff), "red")
#write to text file
alignnum = alignpart['ref']['refnum'] - 1
if (self.params['minscore'] is not None
and alignpart['score'] is not None
and alignpart['score'] < self.params['minscore']):
badscore += 1
continue
if alignnum in self.classlist:
notstackpartnum = alignpart['stackpart']['particleNumber']
tiltstackpartdata = apTiltPair.getStackParticleTiltPair(
self.params['notstackid'], notstackpartnum,
self.params['tiltstackid'])
if tiltstackpartdata is None:
nopairParticle += 1
else:
inplane, mirror = self.getParticleInPlaneRotation(
tiltstackpartdata)
if (self.params['mirror'] == "all" or
(self.params['mirror'] == "no" and mirror is False)
or (self.params['mirror'] == "yes"
and mirror is True)):
emantiltstackpartnum = tiltstackpartdata[
'particleNumber'] - 1
includeParticle.append(emantiltstackpartnum)
tiltParticlesData.append(tiltstackpartdata)
if self.params['numpart'] is not None and len(
includeParticle) > self.params['numpart']:
break
else:
badmirror += 1
else:
excludeParticle += 1
### end methods
includeParticle.sort()
### messages
if time.time() - t0 > 1.0:
apDisplay.printMsg("\nSorting time: " +
apDisplay.timeString(time.time() - t0))
apDisplay.printMsg("Keeping " + str(len(includeParticle)) +
" and excluding \n\t" + str(excludeParticle) +
" particles with " + str(nopairParticle) +
" unpaired particles")
if badmirror > 0:
apDisplay.printMsg("Particles with bad mirrors: %d" % (badmirror))
if badscore > 0:
apDisplay.printColor("Particles with bad scores: %d" % (badscore),
"cyan")
if len(includeParticle) < 1:
apDisplay.printError("No particles were kept")
memdiff = (mem.active() - startmem) / count / 1024.0
if memdiff > 0.1:
apDisplay.printColor("Memory increase: %.2f MB/part" % (memdiff),
"red")
return includeParticle, tiltParticlesData
def __init__(self,optargs=sys.argv[1:],quiet=False,useglobalparams=True,maxnproc=None):
"""
Starts a new function and gets all the parameters
"""
### setup some expected values
self.successful_run = False
self.clusterjobdata = None
self.params = {}
sys.stdout.write("\n\n")
self.quiet = quiet
self.maxnproc = maxnproc
self.startmem = mem.active()
self.t0 = time.time()
self.createDefaultStats()
self.timestamp = apParam.makeTimestamp()
self.argdict = {}
self.optdict = {}
apDisplay.printMsg("Time stamp: "+self.timestamp)
self.functionname = apParam.getFunctionName(sys.argv[0])
apDisplay.printMsg("Function name: "+self.functionname)
self.appiondir = apParam.getAppionDirectory()
apDisplay.printMsg("Appion directory: "+self.appiondir)
hostname = apParam.getHostname()
apDisplay.printMsg("Processing hostname: "+hostname)
self.parsePythonPath()
# loadavg = os.getloadavg()[0]
# if loadavg > 2.0:
# apDisplay.printMsg("Load average is high "+str(round(loadavg,2)))
# loadsquared = loadavg*loadavg
# time.sleep(loadavg)
# apDisplay.printMsg("New load average "+str(round(os.getloadavg()[0],2)))
self.setLockname('lock')
### setup default parser: run directory, etc.
self.setParams(optargs,useglobalparams)
#if 'outdir' in self.params and self.params['outdir'] is not None:
# self.params['rundir'] = self.params['outdir']
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
### check if user wants to print help message
if 'commit' in self.params and self.params['commit'] is True:
apDisplay.printMsg("Committing data to database")
else:
apDisplay.printWarning("Not committing data to database")
self.checkConflicts()
if useglobalparams is True:
self.checkGlobalConflicts()
### setup run directory
self.setProcessingDirName()
self.setupRunDirectory()
### Start pool of threads to run subprocesses.
### Later you will use self.process_launcher.launch(...) to
### put commands into the queue.
### There is currently a timeout built into it that will cause
### the threads to die if they have no tasks after 10 seconds.
self.process_launcher = apThread.ProcessLauncher(2, self.params['rundir'])
### write function log
self.logfile = apParam.writeFunctionLog(sys.argv, msg=(not self.quiet))
### upload command line parameters to database
self.uploadScriptData()
### any custom init functions go here
self.onInit()
