def start(self):
"""
this is the main component of the script
where all the processing is done
"""
### initialize some variables
self.runq = None
self.apix = apStack.getStackPixelSizeFromStackId(self.params['stackid'])
apDisplay.printMsg("Pixel size: %.5f"%(self.apix))
self.boxsize = apStack.getStackBoxsize(self.params['stackid'])
apDisplay.printMsg("Box size: %d"%(self.boxsize))
self.checkResults()
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.numiter = self.getNumberOfIterations()
for i in range(self.numiter):
iternum = i+1
apDisplay.printColor("\nUploading iteration %d of %d\n"%(iternum, self.numiter), "green")
self.uploadIteration(iternum)
reconrunid = apRecon.getReconRunIdFromNamePath(self.params['runname'], self.params['rundir'])
if reconrunid:
apDisplay.printMsg("calculating euler jumpers for recon="+str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(reconrunid, self.params['stackid'])
apRecon.setGoodBadParticlesFromReconId(reconrunid)
else:
apDisplay.printWarning("Could not find recon run id")
def calculateEulerJumpsAndGoodBadParticles(self, uploadIterations):
''' calculate euler jumps for entire recon, currently based on EMAN ZXZ convention '''
if self.params['commit'] is True:
reconrunid = self.refinerunq.dbid
### make table entries of good-bad particles
apRecon.setGoodBadParticlesFromReconId(reconrunid)
### verify completed refinements and iterations
refinecomplete = self.verifyNumberOfCompletedRefinements(
multiModelRefinementRun=self.multiModelRefinementRun)
if self.multiModelRefinementRun is True:
### Euler jumpers calculated from ApMultiModelRefineRunData in multi-model case, looping over values in all single-model refinements
multimodelrunid = self.multimodelq.dbid
if len(uploadIterations) > 1:
apDisplay.printMsg(
"calculating euler jumpers for multi-model refinement="
+ str(multimodelrunid))
eulerjump = apEulerJump.ApEulerJump()
### TECHNICALLY, IT DOESN'T MAKE SENSE TO PASS THE RECONRUNID, SINCE PARTICLES CAN BE JUMPING,
### BUT AS FAR AS I CAN TELL, IT DOESN'T MAKE A DIFFERENCE IN THE RESULTING QUERY, SINCE THE VARIABLE
### IS ONLY USED TO QUERY FOR STACK PARTICLES, WHICH ARE IDENTICAL IN MULTI-MODEL AND SINGLE-MODEL REFINEMENT
### CASES. THEREFORE, THE LAST RECONRUNID IS PASSES ... * DMITRY
eulerjump.calculateEulerJumpsForEntireRecon(
reconrunid,
self.runparams['stackid'],
multimodelrunid=multimodelrunid)
else:
### Euler jumpers calculated from ApRefineRunData in single-model case
if len(uploadIterations) > 1 or \
(refinecomplete.itervalues().next()[-1] == self.runparams['numiter'] and len(refinecomplete.itervalues().next())>1):
apDisplay.printMsg("calculating euler jumpers for recon=" +
str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(
reconrunid, self.runparams['stackid'])
return
def start(self):
if self.params['rundir'] is None or not os.path.isdir(
self.params['rundir']):
apDisplay.printError("upload directory does not exist: " +
str(self.params['rundir']))
### create temp directory for extracting data
self.params['tmpdir'] = os.path.join(self.params['rundir'], "temp")
apParam.createDirectory(self.params['tmpdir'], warning=True)
### make sure that the stack & model IDs exist in database
emanJobFile = self.findEmanJobFile()
self.params['stack'] = apStack.getOnlyStackData(self.params['stackid'])
self.stackmapping = apRecon.partnum2defid(self.params['stackid'])
self.params['model'] = appiondata.ApInitialModelData.direct_query(
self.params['modelid'])
self.params['boxsize'] = apStack.getStackBoxsize(
self.params['stackid'])
### parse out the refinement parameters from the log file
self.parseLogFile()
### parse out the message passing subclassification parameters from the job/log file
if self.params['package'] == 'EMAN/MsgP':
self.parseMsgPassingParams()
### convert class average files from old to new format
self.convertClassAvgFiles()
### get a list of the files in the directory
self.listFiles()
### create a refinementRun entry in the database
self.insertRefinementRun()
if self.params['euleronly'] is False:
### insert the Iteration info
for iteration in self.iterationdatas:
### if only uploading one iteration, skip to that one
if self.params['oneiter'] and int(
iteration['num']) != self.params['oneiter']:
continue
### if beginning at later iteration, skip to that one
if self.params['startiter'] and int(
iteration['num']) < self.params['startiter']:
continue
### if beginning at later iteration, skip to that one
if self.params['enditer'] and int(
iteration['num']) > self.params['enditer']:
continue
apDisplay.printColor(
"\nUploading iteration " + str(iteration['num']) + " of " +
str(len(self.iterationdatas)) + "\n", "green")
for i in range(75):
sys.stderr.write("#")
sys.stderr.write("\n")
self.insertIteration(iteration)
### calculate euler jumps
if self.params['commit'] is True:
reconrunid = self.params['refineRun'].dbid
stackid = self.params['stack'].dbid
if self.params['oneiter'] is None and len(self.iterationdatas) > 1:
apDisplay.printMsg("calculating euler jumpers for recon=" +
str(reconrunid))
eulerjump = apEulerJump.ApEulerJump()
eulerjump.calculateEulerJumpsForEntireRecon(
reconrunid, stackid)
### coran keep plot
if self.params['package'] == 'EMAN/SpiCoran':
apCoranPlot.makeCoranKeepPlot(reconrunid)
apRecon.setGoodBadParticlesFromReconId(reconrunid)
def removePtclsByJumps(self, particles, rejectlst):
eulerjump = apEulerJump.ApEulerJump()
numparts = len(particles)
apDisplay.printMsg("finding euler jumps for " + str(numparts) +
" particles")
### check symmetry
symmetry = eulerjump.getSymmetry(self.params['reconid'], msg=True)
if not re.match("^[cd][0-9]+$", symmetry.lower()) and not re.match(
"^icos", symmetry.lower()):
apDisplay.printError(
"Cannot calculate euler jumps for symmetry: " + symmetry)
return
self.params['sym'] = symmetry.lower()
### prepare file
f = open('jumps.txt', 'w', 0666)
f.write("#pnum\t")
headerlist = ('mean', 'median', 'stdev', 'min', 'max')
for key in headerlist:
f.write(key + "\t")
f.write("\n")
### get stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'])
### start loop
t0 = time.time()
medians = []
count = 0
apDisplay.printMsg("processing euler jumps for recon run=" +
str(self.params['reconid']))
for stackpart in stackparts:
count += 1
partnum = stackpart['particleNumber']
f.write('%d\t' % partnum)
jumpdata = eulerjump.getEulerJumpData(
self.params['reconid'],
stackpartid=stackpart.dbid,
stackid=self.params['stackid'],
sym=symmetry)
medians.append(jumpdata['median'])
if (jumpdata['median'] >
self.params['avgjump']) and partnum not in rejectlst:
rejectlst.append(partnum)
for key in headerlist:
f.write("%.3f\t" % (jumpdata[key]))
f.write("\n")
if count % 1000 == 0:
timeremain = (time.time() - t0) / (count + 1) * (numparts -
count)
apDisplay.printMsg(
"particle=% 5d; median jump=% 3.2f, remain time= %s" %
(partnum, jumpdata['median'],
apDisplay.timeString(timeremain)))
#f.flush()
### print stats
apDisplay.printMsg("-- median euler jumper stats --")
medians = numpy.asarray(medians, dtype=numpy.float32)
apDisplay.printMsg("mean/std :: " + str(round(medians.mean(), 2)) +
" +/- " + str(round(medians.std(), 2)))
apDisplay.printMsg("min/max :: " + str(round(medians.min(), 2)) +
" <> " + str(round(medians.max(), 2)))
perrej = round(
100.0 * float(numparts - len(rejectlst)) / float(numparts), 2)
apDisplay.printMsg("keeping " + str(numparts - len(rejectlst)) +
" of " + str(numparts) + " particles (" +
str(perrej) + "%) so far " + " in " +
apDisplay.timeString(time.time() - t0))
return rejectlst