def getParticlesPerCycle(self, stackfile):
"""
it more efficient to process X particles and write them to disk rather than
write each particle to disk each time.
particles are read using a memory map (numpy.memmap), so we can pretend to
continuously read all into memory
"""
### amount of free memory on machine (converted to bytes)
freememory = mem.free()*1024
self.message("Free memory: %s"%(apDisplay.bytes(freememory)))
### box size of particle
self.boxsize = apFile.getBoxSize(stackfile)[0]
self.message("Box size: %d"%(self.boxsize))
### amount of memory used per particles (4 bytes per pixel)
memperpart = self.boxsize**2 * 4.0
self.message("Memory used per part: %s"%(apDisplay.bytes(memperpart)))
### maximum number particles that fit into memory
maxpartinmem = freememory/memperpart
self.message("Max particles in memory: %d"%(maxpartinmem))
### number particles to fit into memory
partallowed = int(maxpartinmem/20.0)
self.message("Particles allowed in memory: %d"%(partallowed))
### number particles in stack
numpart = self.params['last']
if numpart > partallowed:
numcycles = math.ceil(numpart/float(partallowed))
stepsize = int(numpart/numcycles)
else:
numcycles = 1
stepsize = numpart
self.message("Particle loop num cycles: %d"%(numcycles))
self.message("Particle loop step size: %d"%(stepsize))
return stepsize
def initValues(self, stackfile, numrequest=None):
### check for stack
if not os.path.isfile(stackfile):
apDisplay.printError("stackfile does not exist: "+stackfile)
### amount of free memory on machine (converted to bytes)
self.freememory = mem.free()*1024
self.message("Free memory: %s"%(apDisplay.bytes(self.freememory)))
### box size of particle
self.boxsize = apFile.getBoxSize(stackfile)[0]
self.message("Box size: %d"%(self.boxsize))
### amount of memory used per particles (4 bytes per pixel)
self.memperpart = self.boxsize**2 * 4.0
self.message("Memory used per part: %s"%(apDisplay.bytes(self.memperpart)))
### maximum number particles that fit into memory
self.maxpartinmem = self.freememory/self.memperpart
self.message("Max particles in memory: %d"%(self.maxpartinmem))
### number particles to fit into memory
self.partallowed = int(self.maxpartinmem/20.0)
self.message("Particles allowed in memory: %d"%(self.partallowed))
### number particles in stack
numpart = apFile.numImagesInStack(stackfile)
if self.numpart is None or self.numpart > numpart:
self.numpart = numpart
if numrequest is not None and self.numpart > numrequest:
self.numpart = numrequest
self.message("Number of particles in stack: %d"%(self.numpart))
if self.numpart > self.partallowed:
numchucks = math.ceil(self.numpart/float(self.partallowed))
self.stepsize = int(self.numpart/numchucks)
else:
numchucks = 1
self.stepsize = self.numpart
self.message("Particle loop num chunks: %d"%(numchucks))
self.message("Particle loop step size: %d"%(self.stepsize))
def centerParticles(stack, mask=None, maxshift=None):
apDisplay.printMsg("Centering stack: "+stack)
stacksize = apFile.stackSize(stack)
freemem = mem.free()*1024 #convert memory to bytes
apDisplay.printMsg("file is %s, mem is %s"
%(apDisplay.bytes(stacksize), apDisplay.bytes(freemem)))
### from EMAN FAQ: need to have at least 3x as much ram as the size of the file
memsize = freemem/3.0
numfrac = int(math.ceil(stacksize/memsize))
apDisplay.printMsg("file is %s, will be split into %d fractions"
%(apDisplay.bytes(stacksize), numfrac))
for i in range(numfrac):
emancmd = "cenalignint "+stack
if numfrac > 1:
emancmd += " frac="+str(i)+"/"+str(numfrac)
if mask is not None:
emancmd += " mask="+str(mask)
if maxshift is not None:
emancmd += " maxshift="+str(maxshift)
apEMAN.executeEmanCmd(emancmd, verbose=False, showcmd=True)
return
def _checkMemLeak(self):
"""
unnecessary code for determining if the program is eating memory over time
"""
### Memory leak code:
#self.stats['memlist'].append(mem.mySize()/1024)
self.stats['memlist'].append(mem.active())
memfree = mem.free()
swapfree = mem.swapfree()
minavailmem = 64*1024; # 64 MB, size of one image
if(memfree < minavailmem):
apDisplay.printWarning("Memory is low ("+str(int(memfree/1024))+"MB): there is probably a memory leak")
if(self.stats['count'] > 15):
memlist = self.stats['memlist'][-15:]
n = len(memlist)
gain = (memlist[n-1] - memlist[0])/1024.0
sumx = n*(n-1.0)/2.0
sumxsq = n*(n-1.0)*(2.0*n-1.0)/6.0
sumy = 0.0; sumxy = 0.0; sumysq = 0.0
for i in range(n):
value = float(memlist[i])/1024.0
sumxy += float(i)*value
sumy += value
sumysq += value**2
###
stdx = math.sqrt(n*sumxsq - sumx**2)
stdy = math.sqrt(n*sumysq - sumy**2)
rho = float(n*sumxy - sumx*sumy)/float(stdx*stdy+1e-6)
slope = float(n*sumxy - sumx*sumy)/float(n*sumxsq - sumx*sumx)
memleak = rho*slope
###
if(self.stats['memleak'] > 3 and slope > 20 and memleak > 512 and gain > 2048):
apDisplay.printWarning("Memory leak of "+str(round(memleak,2))+"MB")
elif(memleak > 32):
self.stats['memleak'] += 1
apDisplay.printWarning("substantial memory leak "+str(round(memleak,2))+"MB")
print "(",str(n),round(slope,5),round(rho,5),round(gain,2),")"
