def start(self):
stack1data=apStack.getStackParticlesFromId(self.params['sibstack1'])
stack2data=apStack.getStackParticlesFromId(self.params['sibstack2'])
path1=os.path.join(stack1data[0]['stack']['path']['path'],stack1data[0]['stack']['name'])
path2=os.path.join(stack2data[0]['stack']['path']['path'],stack2data[0]['stack']['name'])
stack2dict={}
stack2values=range(0,len(stack2data))
###This is a little hacky, but is a solution so that stack2data is only looped through once
print "making ptcl dict\n"
for ptcl in stack2values:
stack2dict[stack2data[ptcl]['particle'].dbid]=ptcl
syncstack1name=os.path.splitext(stack1data[0]['stack']['name'])[0]+'.sync1.hed'
syncstack2name=os.path.splitext(stack2data[0]['stack']['name'])[0]+'.sync2.hed'
for ptcl1 in stack1data:
ptcl1_id = ptcl1['particle'].dbid
if ptcl1_id in stack2dict.keys():
ptcl2number=stack2dict[ptcl1_id]
ptcl2=stack2data[ptcl2number]
command1='proc2d %s %s first=%d last=%d' % (path1,syncstack1name,ptcl1['particleNumber']-1, ptcl1['particleNumber']-1)
command2='proc2d %s %s first=%d last=%d' % (path2,syncstack2name,ptcl2['particleNumber']-1, ptcl2['particleNumber']-1)
print command1
print command2
os.system(command1)
os.system(command2)
print '\n'
def start(self):
stack1data=apStack.getStackParticlesFromId(self.params['sibstack1'])
stack2data=apStack.getStackParticlesFromId(self.params['sibstack2'])
path1=os.path.join(stack1data[0]['stack']['path']['path'],stack1data[0]['stack']['name'])
path2=os.path.join(stack2data[0]['stack']['path']['path'],stack2data[0]['stack']['name'])
stack2dict={}
stack2values=range(0,len(stack2data))
###This is a little hacky, but is a solution so that stack2data is only looped through once
print "making ptcl dict\n"
for ptcl in stack2values:
stack2dict[stack2data[ptcl]['particle'].dbid]=ptcl
syncstack1name=os.path.splitext(stack1data[0]['stack']['name'])[0]+'.sync1.hed'
syncstack2name=os.path.splitext(stack2data[0]['stack']['name'])[0]+'.sync2.hed'
for ptcl1 in stack1data:
ptcl1_id = ptcl1['particle'].dbid
if ptcl1_id in stack2dict.keys():
ptcl2number=stack2dict[ptcl1_id]
ptcl2=stack2data[ptcl2number]
command1='proc2d %s %s first=%d last=%d' % (path1,syncstack1name,ptcl1['particleNumber']-1, ptcl1['particleNumber']-1)
command2='proc2d %s %s first=%d last=%d' % (path2,syncstack2name,ptcl2['particleNumber']-1, ptcl2['particleNumber']-1)
print command1
print command2
os.system(command1)
os.system(command2)
print '\n'
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(
self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(
self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(
self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def start(self):
### final stack file
self.dbstackfile = os.path.join(self.params['rundir'],
self.params['stackfilename'])
if os.path.isfile(self.dbstackfile):
apDisplay.printError("A stack with name " +
self.params['stackfilename'] + " and path " +
self.params['rundir'] + " already exists.")
#self.stackid = int(self.params['stackid'])
#self.stackdata = apStack.getOnlyStackData(self.stackid)
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
originalstack = os.path.join(stackdata['path']['path'],
stackdata['name'])
#self.dbstackfile #new file
shutil.copyfile(originalstack, self.dbstackfile)
shutil.copyfile(originalstack[:-3] + "img",
self.dbstackfile[:-3] + "img")
stackpartdata = apStack.getStackParticlesFromId(self.params['stackid'])
dbids = [part.dbid for part in stackpartdata]
for i, id in enumerate(dbids):
numpy.memmap(self.dbstackfile,
dtype="float32",
offset=i * 1024 + 19 * 4)[0] = id
def getStackParticlesInOrder(params):
partorderfile = os.path.join(params['rundir'],'stackpartorder.list')
stackid = params['stackid']
if not os.path.isfile(partorderfile):
return apStack.getStackParticlesFromId(stackid)
partfile = open(partorderfile,'r')
lines = partfile.readlines()
partorder = map((lambda x:int(x[:-1])),lines)
partsort = list(partorder)
partsort.sort()
if partsort == partorder:
return apStack.getStackParticlesFromId(stackid)
apDisplay.printMsg("Preped stack has different order from the original stack. Getting information by the order of the preped stack")
stpartdatas = []
for partnum in partorder:
stpartdatas.append(apStack.getStackParticle(stackid, partnum))
return stpartdatas
def start(self):
self.stackdata = appiondata.ApStackData.direct_query(
self.params['stackid'])
if self.params['vertical'] is not True:
self.alignstackdata = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
# Path of the stack
self.stackdata = apStack.getOnlyStackData(self.params['stackid'])
fn_oldstack = os.path.join(self.stackdata['path']['path'],
self.stackdata['name'])
rotfile = None
if self.params['vertical'] is not True:
# get averaged image:
self.alignstackdata = appiondata.ApAlignStackData.direct_query(
self.params['alignstackid'])
avgimg = os.path.join(self.alignstackdata['path']['path'],
self.alignstackdata['avgmrcfile'])
# Convert averaged aligned mrcfile to spider
spiavg = os.path.join(self.params['rundir'], "avg.spi")
emancmd = "proc2d %s %s spiderswap edgenorm" % (avgimg, spiavg)
apEMAN.executeEmanCmd(emancmd, verbose=True)
# find rotation for vertical alignment
rot = self.findRotation(spiavg)
apDisplay.printMsg("found average rotation: %.2f" % rot)
rotlist = self.getInplaneRotations()
rotfile = self.createRotationSpiList(rotlist, rot)
# Convert the original stack to spider
spistack = self.convertStackToSpider(fn_oldstack)
# boxmask the particles
spimaskfile = "masked" + self.timestamp + ".spi"
self.boxMask(spistack, spimaskfile, rotfile)
# Convert the spider stack to imagic
imgstack = self.convertStackToImagic(spimaskfile)
# Create average MRC
apStack.averageStack(imgstack)
# Clean up
apDisplay.printMsg("deleting temporary processing files")
os.remove(spistack)
os.remove(spimaskfile)
# Upload results
if self.params['commit'] is True:
oldstackparts = apStack.getStackParticlesFromId(
self.params['stackid'])
apStack.commitMaskedStack(self.params,
oldstackparts,
newname='start.hed')
time.sleep(1)
return
def removePtclsByJumps(self, particles, rejectlst):
eulerjump = apEulerJump.ApEulerJump()
numparts = len(particles)
apDisplay.printMsg("finding euler jumps for "+str(numparts)+" particles")
### check symmetry
symmetry = eulerjump.getSymmetry(self.params['reconid'], msg=True)
if not re.match("^[cd][0-9]+$", symmetry.lower()) and not re.match("^icos", symmetry.lower()):
apDisplay.printError("Cannot calculate euler jumps for symmetry: "+symmetry)
return
self.params['sym']=symmetry.lower()
### prepare file
f = open('jumps.txt','w', 0666)
f.write("#pnum\t")
headerlist = ('mean', 'median', 'stdev', 'min', 'max')
for key in headerlist:
f.write(key+"\t")
f.write("\n")
### get stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'])
### start loop
t0 = time.time()
medians = []
count = 0
apDisplay.printMsg("processing euler jumps for recon run="+str(self.params['reconid']))
for stackpart in stackparts:
count += 1
partnum = stackpart['particleNumber']
f.write('%d\t' % partnum)
jumpdata = eulerjump.getEulerJumpData(self.params['reconid'], stackpartid=stackpart.dbid, stackid=self.params['stackid'], sym=symmetry)
medians.append(jumpdata['median'])
if (jumpdata['median'] > self.params['avgjump']) and partnum not in rejectlst:
rejectlst.append(partnum)
for key in headerlist:
f.write("%.3f\t" % (jumpdata[key]))
f.write("\n")
if count % 1000 == 0:
timeremain = (time.time()-t0)/(count+1)*(numparts-count)
apDisplay.printMsg("particle=% 5d; median jump=% 3.2f, remain time= %s" % (partnum, jumpdata['median'],
apDisplay.timeString(timeremain)))
#f.flush()
### print stats
apDisplay.printMsg("-- median euler jumper stats --")
medians = numpy.asarray(medians, dtype=numpy.float32)
apDisplay.printMsg("mean/std :: "+str(round(medians.mean(),2))+" +/- "
+str(round(medians.std(),2)))
apDisplay.printMsg("min/max :: "+str(round(medians.min(),2))+" <> "
+str(round(medians.max(),2)))
perrej = round(100.0*float(numparts-len(rejectlst))/float(numparts),2)
apDisplay.printMsg("keeping "+str(numparts-len(rejectlst))+" of "+str(numparts)
+" particles ("+str(perrej)+"%) so far "
+" in "+apDisplay.timeString(time.time()-t0))
return rejectlst
def getStackParticlesInOrder(params):
partorderfile = os.path.join(params['rundir'], 'stackpartorder.list')
stackid = params['stackid']
if not os.path.isfile(partorderfile):
return apStack.getStackParticlesFromId(stackid)
partfile = open(partorderfile, 'r')
lines = partfile.readlines()
partorder = map((lambda x: int(x[:-1])), lines)
partsort = list(partorder)
partsort.sort()
if partsort == partorder:
return apStack.getStackParticlesFromId(stackid)
apDisplay.printMsg(
"Preped stack has different order from the original stack. Getting information by the order of the preped stack"
)
stpartdatas = []
for partnum in partorder:
stpartdatas.append(apStack.getStackParticle(stackid, partnum))
return stpartdatas
def start(self):
self.stackdata = apStack.getOnlyStackData(self.params['stackid'],
msg=False)
# creating a keepfile, fixed filename
self.params['keepfile'] = os.path.join(self.params['newstackpath'],
"keepfile.lst")
#path to the old stack
oldstack = os.path.join(self.stackdata['path']['path'],
self.stackdata['name'])
#path to the new stack. the stack path will be provided by the db in the future
newstack = os.path.join(self.params['newstackpath'],
self.params['newstack'])
#messy way to count the number of particles in a stack
h = open(newstack, 'r')
numimg = 0
while h.read(1024):
numimg += 1
#have to use this function to make sure i get the same particle number like in the download
stackpartdata = apStack.getStackParticlesFromId(self.params['stackid'])
#since the keepfile has to be a proc2d like file, i create a dictionary to transfer the
#uniqe particle id into the stack position. I have to decrement 1 to make it count from 0
#to numing
partdict = {}
dbids = [(part.dbid, part['particleNumber']) for part in stackpartdata]
for part in dbids:
partdict[int(part[0])] = int(part[1] - 1)
#writing the keepfile
f = open(self.params['keepfile'], 'w')
for i in range(0, numimg):
partnumber = partdict[int(
numpy.memmap(newstack,
dtype="float32",
offset=i * 1024 + 19 * 4)[0])]
f.write('%d\n' % partnumber)
f.close()
newcreatestack = os.path.join(self.params['rundir'],
self.params['newstack'])
apStack.makeNewStack(oldstack,
newcreatestack,
self.params['keepfile'],
bad=True)
apStack.commitSubStack(self.params,
self.params['newstack'],
sorted=False)
apStack.averageStack(stack=newcreatestack)
newstackid = apStack.getStackIdFromPath(newcreatestack)
def start(self):
stackparts = apStack.getStackParticlesFromId(self.params['stackid'])
stackdata = apStack.getOnlyStackData(self.params['stackid'])
newname = stackdata['name']
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
newstack = os.path.join(self.params['rundir'], newname)
# calculate slop and intercept from the four points given
slope = (self.params['maxy'] - self.params['miny']) / (
self.params['maxx'] - self.params['minx'])
intercept = self.params['miny'] - (slope * self.params['minx'])
# print slope
# print intercept
numparticles = 0
self.params['keepfile'] = os.path.join(
self.params['rundir'], "keepfile-" + self.timestamp + ".list")
f = open(self.params['keepfile'], 'w')
for stackpart in stackparts:
#print str(stackpart['particleNumber'])+","+ str(stackpart['mean'])+","+str(stackpart['stdev'])
if stackpart['mean'] > self.params['minx'] and stackpart[
'mean'] < self.params['maxx']:
#print str(stackpart['particleNumber'])+","+ str(stackpart['mean'])+","+str(stackpart['stdev'])
calcY = slope * stackpart['mean'] + intercept
if (calcY >= stackpart['stdev'] and self.params['keepabove'] is not True) or \
(calcY <= stackpart['stdev'] and self.params['keepabove'] is True):
emanpartnum = stackpart['particleNumber'] - 1
f.write('%i\n' % emanpartnum)
numparticles += 1
f.close()
self.params['description'] += (
(" ... %d particle substack of stackid %d" %
(numparticles, self.params['stackid'])))
#create the new sub stack
apStack.makeNewStack(oldstack,
newstack,
self.params['keepfile'],
bad=True)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
apStack.commitSubStack(self.params, newname, oldstackparts=stackparts)
apStack.averageStack(stack=newstack)
# stack mean plot
newstackid = apStack.getStackIdFromPath(newstack)
apDisplay.printMsg("creating Stack Mean Plot montage for stackid")
apStackMeanPlot.makeStackMeanPlot(newstackid)
def start(self):
#find stack
stackparticles = apStack.getStackParticlesFromId(
self.params['stackid'])
if self.params['logsplit']:
#stacklist = oldLogSplit(self.params['logstart'], len(stackparticles), self.params['logdivisions'])
stacklist = evenLogSplit(self.params['logstart'],
len(stackparticles))
elif self.params['nptcls']:
stacklist = [self.params['nptcls']]
else:
apDisplay.printError("Please specify nptlcs or logsplit")
oldstackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(oldstackdata['path']['path'],
oldstackdata['name'])
#create run directory
if self.params['rundir'] is None:
path = oldstackdata['path']['path']
path = os.path.split(os.path.abspath(path))[0]
self.params['rundir'] = path
apDisplay.printMsg("Out directory: " + self.params['rundir'])
origdescription = self.params['description']
for stack in stacklist:
self.params['description'] = (
origdescription +
(" ... split %d particles from original stackid=%d" %
(stack, self.params['stackid'])))
workingdir = os.path.join(self.params['rundir'], str(stack))
#check for previously commited stacks
newstack = os.path.join(workingdir, self.params['stackname'])
apStack.checkForPreviousStack(newstack)
#create rundir and change to that directory
apDisplay.printMsg("Run directory: " + workingdir)
apParam.createDirectory(workingdir)
os.chdir(workingdir)
#create random list
lstfile = makeRandomLst(stack, stackparticles, self.params)
#shutil.copy(lstfile, workingdir)
#make new stack
apStack.makeNewStack(oldstack, newstack, lstfile)
#apStack.makeNewStack(lstfile, self.params['stackname'])
#commit new stack
self.params['keepfile'] = os.path.abspath(lstfile)
self.params['rundir'] = os.path.abspath(workingdir)
apStack.commitSubStack(self.params)
def start(self):
#find stack
stackparticles = apStack.getStackParticlesFromId(self.params['stackid'])
if self.params['logsplit']:
#stacklist = oldLogSplit(self.params['logstart'], len(stackparticles), self.params['logdivisions'])
stacklist = evenLogSplit(self.params['logstart'], len(stackparticles))
elif self.params['nptcls']:
stacklist = [self.params['nptcls']]
else:
apDisplay.printError("Please specify nptlcs or logsplit")
oldstackdata = apStack.getOnlyStackData(self.params['stackid'])
oldstack = os.path.join(oldstackdata['path']['path'], oldstackdata['name'])
#create run directory
if self.params['rundir'] is None:
path = oldstackdata['path']['path']
path = os.path.split(os.path.abspath(path))[0]
self.params['rundir'] = path
apDisplay.printMsg("Out directory: "+self.params['rundir'])
origdescription=self.params['description']
for stack in stacklist:
self.params['description'] = (
origdescription+
(" ... split %d particles from original stackid=%d"
% (stack, self.params['stackid']))
)
workingdir = os.path.join(self.params['rundir'], str(stack))
#check for previously commited stacks
newstack = os.path.join(workingdir ,self.params['stackname'])
apStack.checkForPreviousStack(newstack)
#create rundir and change to that directory
apDisplay.printMsg("Run directory: "+workingdir)
apParam.createDirectory(workingdir)
os.chdir(workingdir)
#create random list
lstfile = makeRandomLst(stack, stackparticles, self.params)
#shutil.copy(lstfile, workingdir)
#make new stack
apStack.makeNewStack(oldstack, newstack, lstfile)
#apStack.makeNewStack(lstfile, self.params['stackname'])
#commit new stack
self.params['keepfile'] = os.path.abspath(lstfile)
self.params['rundir'] = os.path.abspath(workingdir)
apStack.commitSubStack(self.params)
def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
### stack data
stackdata = apStack.getOnlyStackData(self.params['stackid'])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]['particle']['selectionrun']
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq['oldselectionrun'] = oldselectrun
manualparamsq['trace'] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['hidden'] = False
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrunq['session'] = apStack.getSessionDataFromStackId(self.params['stackid'])
selectrunq['manparams'] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count%100 == 0:
perpart = (time.time()-t0)/float(count+1)
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(count, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-count))), "blue")
oldpartdata = stackpart['particle']
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq['selectionrun'] = selectrunq
if self.params['commit'] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s"%(apDisplay.timeString(time.time()-t0)))
def start(self):
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
if self.params['vertical'] is not True:
self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
# Path of the stack
self.stackdata = apStack.getOnlyStackData(self.params['stackid'])
fn_oldstack = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
rotfile = None
if self.params['vertical'] is not True:
# get averaged image:
self.alignstackdata = appiondata.ApAlignStackData.direct_query(self.params['alignstackid'])
avgimg = os.path.join(self.alignstackdata['path']['path'], self.alignstackdata['avgmrcfile'])
# Convert averaged aligned mrcfile to spider
spiavg = os.path.join(self.params['rundir'],"avg.spi")
emancmd = "proc2d %s %s spiderswap edgenorm"%(avgimg,spiavg)
apEMAN.executeEmanCmd(emancmd, verbose=True)
# find rotation for vertical alignment
rot = self.findRotation(spiavg)
apDisplay.printMsg("found average rotation: %.2f"%rot)
rotlist = self.getInplaneRotations()
rotfile = self.createRotationSpiList(rotlist,rot)
# Convert the original stack to spider
spistack = self.convertStackToSpider(fn_oldstack)
# boxmask the particles
spimaskfile = "masked"+self.timestamp+".spi"
self.boxMask(spistack,spimaskfile,rotfile)
# Convert the spider stack to imagic
imgstack = self.convertStackToImagic(spimaskfile)
# Create average MRC
apStack.averageStack(imgstack)
# Clean up
apDisplay.printMsg("deleting temporary processing files")
os.remove(spistack)
os.remove(spimaskfile)
# Upload results
if self.params['commit'] is True:
oldstackparts = apStack.getStackParticlesFromId(self.params['stackid'])
apStack.commitMaskedStack(self.params, oldstackparts, newname='start.hed')
time.sleep(1)
return
def start(self):
stackparts = apStack.getStackParticlesFromId(self.params['stackid'])
stackdata = apStack.getOnlyStackData(self.params['stackid'])
newname = stackdata['name']
oldstack = os.path.join(stackdata['path']['path'], stackdata['name'])
newstack = os.path.join(self.params['rundir'], newname)
# calculate slop and intercept from the four points given
slope = (self.params['maxy'] - self.params['miny']) / (self.params['maxx'] - self.params['minx'])
intercept = self.params['miny'] - (slope*self.params['minx'])
# print slope
# print intercept
numparticles = 0
self.params['keepfile'] = os.path.join(self.params['rundir'], "keepfile-"+self.timestamp+".list")
f=open(self.params['keepfile'],'w')
for stackpart in stackparts:
#print str(stackpart['particleNumber'])+","+ str(stackpart['mean'])+","+str(stackpart['stdev'])
if stackpart['mean'] > self.params['minx'] and stackpart['mean'] < self.params['maxx']:
#print str(stackpart['particleNumber'])+","+ str(stackpart['mean'])+","+str(stackpart['stdev'])
calcY = slope*stackpart['mean']+intercept
if (calcY >= stackpart['stdev'] and self.params['keepabove'] is not True) or \
(calcY <= stackpart['stdev'] and self.params['keepabove'] is True):
emanpartnum = stackpart['particleNumber']-1
f.write('%i\n' % emanpartnum)
numparticles+=1
f.close()
self.params['description'] +=(
(" ... %d particle substack of stackid %d"
% (numparticles, self.params['stackid']))
)
#create the new sub stack
apStack.makeNewStack(oldstack, newstack, self.params['keepfile'], bad=True)
if not os.path.isfile(newstack):
apDisplay.printError("No stack was created")
apStack.commitSubStack(self.params, newname, oldstackparts=stackparts)
apStack.averageStack(stack=newstack)
# stack mean plot
newstackid = apStack.getStackIdFromPath(newstack)
apDisplay.printMsg("creating Stack Mean Plot montage for stackid")
apStackMeanPlot.makeStackMeanPlot(newstackid)
def calculateEulerJumpsForEntireRecon(self, reconrunid, stackid=None, sym=None, multimodelrunid=None):
if sym is None:
sym = self.getSymmetry(reconrunid)
if re.match("^icos", sym.lower()):
apDisplay.printWarning("Processing Icos symmetry "+sym)
if not re.match("^[cd][0-9]+$", sym.lower()) and not re.match("^icos", sym.lower()):
apDisplay.printError("Cannot calculate euler jumps for symmetry: "+sym)
return
### get stack particles
if stackid is None:
stackid = apStack.getStackIdFromRecon(reconrunid, msg=False)
stackparts = apStack.getStackParticlesFromId(stackid)
stackparts.sort(self.sortStackParts)
numparts = len(stackparts)
### start loop
t0 = time.time()
medians = []
count = 0
miscount = 0
for stackpart in stackparts:
count += 1
if multimodelrunid is None:
jumpdata = self.getEulerJumpData(reconrunid, stackpartid=stackpart.dbid, stackid=stackid, sym=sym)
else:
jumpdata = self.getEulerJumpData(reconrunid, stackpartid=stackpart.dbid, stackid=stackid, sym=sym, multimodelrunid=multimodelrunid)
if jumpdata is None:
if miscount < 25:
continue
else:
break
medians.append(jumpdata['median'])
if count % 500 == 0:
timeremain = (time.time()-t0)/(count+1)*(numparts-count)
print ("particle=% 5d; median jump=% 3.2f, remain time= %s" % (stackpart['particleNumber'], jumpdata['median'],
apDisplay.timeString(timeremain)))
if len(medians) > 0:
### print stats
apDisplay.printMsg("complete "+str(len(stackparts))+" particles in "+apDisplay.timeString(time.time()-t0))
print "-- median euler jumper stats --"
medians = numpy.asarray(medians, dtype=numpy.float32)
print ("mean/std :: "+str(round(medians.mean(),2))+" +/- "
+str(round(medians.std(),2)))
print ("min/max :: "+str(round(medians.min(),2))+" <> "
+str(round(medians.max(),2)))
else:
apDisplay.printWarning("no Euler jumpers inserted into the database, make sure that the angles are read by the recon uploader")
return
def commitStack(self, stackid):
startpart = self.partnum
stackq = appiondata.ApStackData()
oldstackdata = apStack.getOnlyStackData(stackid)
stackq['name'] = self.params['stackfilename']
stackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
stackq['description'] = self.params[
'description'] + " ... combined stack ids " + str(
self.params['stacks'])
stackq['substackname'] = self.params['runname']
stackq['hidden'] = False
stackq['pixelsize'] = self.newpixelsize * 1e-10
stackq['boxsize'] = self.newboxsize
rinstackdata = apStack.getRunsInStack(stackid)
for run in rinstackdata:
rinstackq = appiondata.ApRunsInStackData()
rinstackq['stack'] = stackq
rinstackq['stackRun'] = run['stackRun']
rinstackq.insert()
stpartsdata = apStack.getStackParticlesFromId(stackid)
apDisplay.printMsg("inserting " + str(len(stpartsdata)) +
" particles into DB")
for particle in stpartsdata:
stpartq = appiondata.ApStackParticleData()
stpartq['particleNumber'] = self.partnum
stpartq['stack'] = stackq
stpartq['stackRun'] = particle['stackRun']
stpartq['particle'] = particle['particle']
stpartq.insert()
self.partnum += 1
if self.partnum % 1000 == 0:
sys.stderr.write(".")
sys.stderr.write("\n")
apDisplay.printMsg("commited particles " + str(startpart) + "-" +
str(self.partnum))
return
def start(self):
self.stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
# creating a keepfile, fixed filename
self.params['keepfile'] = os.path.join(self.params['newstackpath'],"keepfile.lst")
#path to the old stack
oldstack = os.path.join(self.stackdata['path']['path'], self.stackdata['name'])
#path to the new stack. the stack path will be provided by the db in the future
newstack = os.path.join(self.params['newstackpath'], self.params['newstack'])
#messy way to count the number of particles in a stack
h = open(newstack, 'r')
numimg = 0
while h.read(1024):
numimg += 1
#have to use this function to make sure i get the same particle number like in the download
stackpartdata = apStack.getStackParticlesFromId(self.params['stackid'])
#since the keepfile has to be a proc2d like file, i create a dictionary to transfer the
#uniqe particle id into the stack position. I have to decrement 1 to make it count from 0
#to numing
partdict = {}
dbids = [(part.dbid,part['particleNumber']) for part in stackpartdata]
for part in dbids:
partdict[int(part[0])] = int(part[1]-1)
#writing the keepfile
f = open(self.params['keepfile'], 'w')
for i in range(0,numimg):
partnumber = partdict[int(numpy.memmap(newstack, dtype="float32", offset=i*1024+19*4)[0])]
f.write('%d\n' % partnumber)
f.close()
newcreatestack = os.path.join(self.params['rundir'],self.params['newstack'])
apStack.makeNewStack(oldstack, newcreatestack, self.params['keepfile'], bad=True)
apStack.commitSubStack(self.params, self.params['newstack'], sorted=False)
apStack.averageStack(stack=newcreatestack)
newstackid = apStack.getStackIdFromPath(newcreatestack)
def start(self):
### final stack file
self.dbstackfile = os.path.join( self.params['rundir'], self.params['stackfilename'] )
if os.path.isfile(self.dbstackfile):
apDisplay.printError("A stack with name "+self.params['stackfilename']+" and path "
+self.params['rundir']+" already exists.")
#self.stackid = int(self.params['stackid'])
#self.stackdata = apStack.getOnlyStackData(self.stackid)
stackdata = apStack.getOnlyStackData(self.params['stackid'], msg=False)
originalstack = os.path.join(stackdata['path']['path'], stackdata['name'])
#self.dbstackfile #new file
shutil.copyfile(originalstack,self.dbstackfile)
shutil.copyfile(originalstack[:-3]+"img",self.dbstackfile[:-3]+"img")
stackpartdata = apStack.getStackParticlesFromId(self.params['stackid'])
dbids = [part.dbid for part in stackpartdata]
for i,id in enumerate(dbids):
numpy.memmap(self.dbstackfile, dtype="float32", offset=i*1024+19*4)[0] = id
def commitStack(self, stackid):
startpart = self.partnum
stackq = appiondata.ApStackData()
oldstackdata = apStack.getOnlyStackData(stackid)
stackq['name'] = self.params['stackfilename']
stackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
stackq['description'] = self.params['description']+" ... combined stack ids "+str(self.params['stacks'])
stackq['substackname'] = self.params['runname']
stackq['hidden'] = False
stackq['pixelsize'] = self.newpixelsize*1e-10
stackq['boxsize'] = self.newboxsize
rinstackdata = apStack.getRunsInStack(stackid)
for run in rinstackdata:
rinstackq = appiondata.ApRunsInStackData()
rinstackq['stack'] = stackq
rinstackq['stackRun'] = run['stackRun']
rinstackq.insert()
stpartsdata = apStack.getStackParticlesFromId(stackid)
apDisplay.printMsg("inserting "+str(len(stpartsdata))+" particles into DB")
for particle in stpartsdata:
stpartq = appiondata.ApStackParticleData()
stpartq['particleNumber'] = self.partnum
stpartq['stack'] = stackq
stpartq['stackRun'] = particle['stackRun']
stpartq['particle'] = particle['particle']
stpartq.insert()
self.partnum += 1
if self.partnum % 1000 == 0:
sys.stderr.write(".")
sys.stderr.write("\n")
apDisplay.printMsg("commited particles "+str(startpart)+"-"+str(self.partnum))
return
def getExistingStackInfo(self):
stackfile=os.path.join(self.params['rundir'], self.params['single'])
stackid = apStack.getStackIdFromPath(stackfile)
numdbpart = len(apStack.getStackParticlesFromId(stackid))
if numdbpart == 0:
self.params['continue'] = False
apDisplay.printWarning("file exists but no particles in database, deleting stack file")
apFile.removeStack(stackfile)
return 0
### we now have particles in the database
if self.params['continue'] is False:
apDisplay.printWarning("particles exist in database, must force continue")
self.params['continue'] = True
### we better have the same number of particles in the file and the database
numfilepart = apFile.numImagesInStack(stackfile)
if numfilepart != numdbpart:
apDisplay.printError("database and file have different number of particles, \n"+
"create a new stack this one is corrupt")
return numfilepart
def start(self):
self.params['output_fileformat'] = 'mrc'
newstackname = 'framealigned.hed'
stackdata = apStack.getStackParticlesFromId(self.params['stackid'])
stackrundata = apStack.getOnlyStackData(self.params['stackid'])
apix = stackrundata['pixelsize'] * 1e10
kev = stackdata[0]['particle']['image']['scope']['high tension'] / 1000
origstackpath = os.path.join(stackrundata['path']['path'],
stackrundata['name'])
boxsize = stackdata[0]['stackRun']['stackParams']['boxSize']
binning = stackdata[0]['stackRun']['stackParams']['bin']
#determine camera type
cameratype = stackdata[0]['particle']['image']['camera']['ccdcamera'][
'name']
if self.params['override_camera'] is not None:
cameratype = self.params['override_camera']
#create sorted boxfiles
imagedict = {}
masterlist = []
for particle in stackdata:
parentimage = particle['particle']['image']['filename']
if parentimage in imagedict.keys():
imagedict[parentimage].append(particle['particle'])
else:
imagedict[parentimage] = []
imagedict[parentimage].append(particle['particle'])
index = len(imagedict[parentimage]) - 1
masterlist.append({
'particle': particle,
'key': parentimage,
'index': index
})
#print masterlist
for key in imagedict:
particlelst = imagedict[key]
parentimage = key
framespath = particlelst[0]['image']['session']['frame path']
print cameratype
if 'Gatan' in cameratype:
#prepare frames
print framespath
#prepare frame directory
framespathname = os.path.join(self.params['rundir'],
parentimage + '.frames')
if os.path.exists(framespathname):
pass
else:
os.mkdir(framespathname)
print framespathname
mrcframestackname = parentimage + '.frames.mrc'
print mrcframestackname
nframes = particlelst[0]['image']['camera']['nframes']
print "Extracting frames for", mrcframestackname
for n in range(nframes):
a = mrc.read(os.path.join(framespath, mrcframestackname),
n)
numpil.write(a,
imfile=os.path.join(framespathname,
'RawImage_%d.tif' % (n)),
format='tiff')
elif 'DE' in cameratype:
framespathname = os.path.join(framespath,
parentimage + '.frames')
print os.getcwd()
print framespathname
#generate DE script call
if os.path.exists(framespathname):
print "found frames for", parentimage
nframes = particlelst[0]['image']['camera']['nframes']
boxname = parentimage + '.box'
boxpath = os.path.join(framespathname, boxname)
shiftdata = {'scale': 1, 'shiftx': 0, 'shifty': 0}
#flatfield references
brightrefpath = particlelst[0]['image']['bright']['session'][
'image path']
brightrefname = particlelst[0]['image']['bright']['filename']
brightnframes = particlelst[0]['image']['bright']['camera'][
'nframes']
darkrefpath = particlelst[0]['image']['dark']['session'][
'image path']
darkrefname = particlelst[0]['image']['dark']['filename']
darknframes = particlelst[0]['image']['dark']['camera'][
'nframes']
brightref = os.path.join(brightrefpath, brightrefname + '.mrc')
darkref = os.path.join(darkrefpath, darkrefname + '.mrc')
print brightref
print darkref
apBoxer.processParticleData(particle['particle']['image'],
boxsize, particlelst, shiftdata,
boxpath)
print framespathname
#set appion specific options
self.params['gainreference_filename'] = brightref
self.params['gainreference_framecount'] = brightnframes
self.params['darkreference_filename'] = darkref
self.params['darkreference_framecount'] = darknframes
self.params['input_framecount'] = nframes
self.params['boxes_fromfiles'] = 1
#self.params['run_verbosity']=3
self.params['output_invert'] = 0
#self.params['radiationdamage_apix=']=apix
self.params['radiationdamage_voltage'] = kev
#self.params['boxes_boxsize']=boxsize
outpath = os.path.join(self.params['rundir'], key)
if os.path.exists(outpath):
shutil.rmtree(outpath)
os.mkdir(outpath)
command = ['deProcessFrames.py']
keys = self.params.keys()
keys.sort()
for key in keys:
param = self.params[key]
#print key, param, type(param)
if param == None or param == '':
pass
else:
option = '--%s=%s' % (key, param)
command.append(option)
command.append(outpath)
command.append(framespathname)
print command
if self.params['dryrun'] is False:
subprocess.call(command)
#recreate particle stack
for n, particledict in enumerate(masterlist):
parentimage = particledict['key']
correctedpath = os.path.join(self.params['rundir'], parentimage)
print correctedpath
if os.path.exists(correctedpath):
correctedparticle = glob.glob(
os.path.join(correctedpath,
('%s.*.region_%03d.*' %
(parentimage, particledict['index']))))
print os.path.join(correctedpath,
('%s.*.region_%03d.*' %
(parentimage, particledict['index'])))
print correctedparticle
#sys.exit()
command = ['proc2d', correctedparticle[0], newstackname]
if self.params['output_rotation'] != 0:
command.append('rot=%d' % self.params['output_rotation'])
if self.params['show_DE_command'] is True:
print command
subprocess.call(command)
else:
print "did not find frames for ", parentimage
command = [
'proc2d', origstackpath, newstackname, ('first=%d' % n),
('last=%d' % n)
]
print command
if self.params['dryrun'] is False:
subprocess.call(command)
#upload stack
#make keep file
self.params['keepfile'] = 'keepfile.txt'
f = open(self.params['keepfile'], 'w')
for n in range(len(masterlist)):
f.write('%d\n' % (n))
f.close()
apStack.commitSubStack(self.params, newname=newstackname)
apStack.averageStack(stack=newstackname)
print "Done!!!!"
def start(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
nptcls=len(stackPartList)
currentImageId = stackPartList[0]['particle']['image'].dbid
count = 0
imagenumber=1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
#create list of dictionaries that will be passed to starFile maker later
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n"%(count, nptcls))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning("particle number in database is not in sync")
partParams = {}
partParams['ptclnum'] = count
### get image data
imagedata = stackPart['particle']['image']
if imagedata.dbid != currentImageId:
imagenumber+=1
currentImageId=imagedata.dbid
partParams['filmNum'] = imagenumber
#print partParams['filmNum']
partParams['kv'] = imagedata['scope']['high tension']/1000.0
partParams['cs'] =imagedata['scope']['tem']['cs']*1000
### get CTF data from image
ctfdata = ctfdb.getBestCtfValue(imagedata, msg=False, sortType='maxconf')
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1']*1e10)
partParams['defocus2'] = abs(ctfdata['defocus2']*1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata['amplitude_contrast']
else:
apDisplay.printError("No ctf information for particle %d in image %s"%(count, imagedata['filename']))
partParamsList.append(partParams)
###now make star file
#first make header
star = starFile.StarFile(self.params['outstar'])
labels = ['_rlnImageName', '_rlnMicrographName',
'_rlnDefocusU', '_rlnDefocusV', '_rlnDefocusAngle', '_rlnVoltage',
'_rlnSphericalAberration', '_rlnAmplitudeContrast',
]
valueSets = [] #list of strings for star file
###now make particle data
for partParams in partParamsList:
relionDataLine = ("%[email protected] mic%d %.6f %.6f %.6f %d %.6f %.6f"
%( partParams['ptclnum'], partParams['filmNum'],
partParams['defocus2'], partParams['defocus1'], partParams['angle_astigmatism'],
partParams['kv'], partParams['cs'], partParams['amplitude_contrast'],
))
valueSets.append(relionDataLine)
star.buildLoopFile( "data_", labels, valueSets )
star.write()
def getStackParticleParams(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
if 'last' not in self.params:
self.params['last'] = len(stackPartList)
firstImageId = stackPartList[0]['particle']['image'].dbid
count = 0
lastImageId = -1
lastCtfData = None
lastKv = -1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
t0 = time.time()
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n"%(count, self.params['last']))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning("particle number in database is not in sync")
if count > self.params['last']:
break
partParams = {}
partParams['ptclnum'] = count
partParams['filmNum'] = self.getFilmNumber(stackPart, firstImageId)
#print partParams['filmNum']
### get image data
imagedata = stackPart['particle']['image']
if self.originalStackData.defocpair is True:
imagedata = apDefocalPairs.getDefocusPair(imagedata)
if lastImageId == imagedata.dbid:
ctfdata = lastCtfData
partParams['kv'] = lastKv
else:
ctfdata = ctfdb.getBestCtfValue(imagedata, msg=False, method=self.params['ctfmethod'])
partParams['kv'] = imagedata['scope']['high tension']/1000.0
lastCtfData = ctfdata
lastImageId = imagedata.dbid
lastKv = partParams['kv']
### get CTF data from image
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1']*1e10)
partParams['defocus2'] = abs(ctfdata['defocus2']*1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata['amplitude_contrast']
else:
apDisplay.printWarning("No ctf information for particle %d in image %d"%(count, imagedata.dbid))
partParams['defocus1'] = 0.1
partParams['defocus2'] = 0.1
partParams['angle_astigmatism'] = 0.0
partParams['amplitude_contrast'] = 0.07
if self.params['reconiterid'] is not None:
eulerDict = self.getStackParticleEulersForIteration(stackPart)
partParams.update(eulerDict)
partParamsList.append(partParams)
print "no class %d ; mismatch %d"%(self.noClassification, self.mismatch)
sys.stderr.write("\ndone in %s\n\n"%(apDisplay.timeString(time.time()-t0)))
return partParamsList
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(
self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '" + newdbname + "'",
"green")
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['expweight'] is False:
apDisplay.printWarning(
"Exposure weighting is turned off, make sure this is what you want"
)
if self.params['localavg'] is False:
apDisplay.printWarning(
"Trajectory local averaging is turned off, make sure this is what you want"
)
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(
self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(
self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(
self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# pixel size info
self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(
self.params['stackid'])
self.params['box'] = self.stackdata['boxsize']
self.params['particleradius'] = self.params[
'particleradius'] / self.params['apix']
if self.params['particleradius'] > self.params['box'] / 2.0:
apDisplay.printWarning(
"specified particle radius greater than box radius, \
setting particle radius to 0.8 * boxsize")
# micrograph & frame info
frames = qimage['use frames']
nframes = len(frames)
if self.params['framelastali'] is None:
self.params['framelastali'] = frames[-1]
if self.params['framelastave'] is None:
self.params['framelastave'] = frames[-1]
# microscope kV
self.params['kv'] = qimage['scope']['high tension'] / 1000.0
# query exposure per frame, if not set here
if self.params['total_dose'] is not None:
dose = self.params['total_dose']
else:
try:
dose = apDatabase.getDoseFromImageData(qimage)
except:
apDisplay.printError(
"dose not specified and not in database, please specify explicitly"
)
if self.params['expperframe'] is None and self.params[
'expweight'] is True:
if dose is not None:
self.params['expperframe'] = dose / nframes
else:
apDisplay.printError(
"exposure per frame needs to be specified, cannot find in database"
)
# dimensions
self.params['framex'] = int(
apDatabase.getDimensionsFromImageData(qimage)['x'])
self.params['framey'] = int(
apDatabase.getDimensionsFromImageData(qimage)['y'])
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def start(self):
### default parameters
starfile = self.params['starfile']
images = self.params['images']
dbnum = self.params['dbnum']
ctfrunid = self.params['ctfrunid']
stackid = self.params['stackid']
voltage = self.params['voltage']
cs = self.params['cs']
wgh = self.params['wgh']
### particles, angles
appiondata.sinedon.setConfig('appiondata', db="ap%d" % dbnum)
particledata = apStack.getStackParticlesFromId(stackid)
### write Relion starfile header
sf = open(starfile, "w")
sf.write("data_images\n")
sf.write("loop_\n")
sf.write("_rlnImageName\n")
sf.write("_rlnMicrographName\n")
sf.write("_rlnDefocusU\n")
sf.write("_rlnDefocusV\n")
sf.write("_rlnDefocusAngle\n")
sf.write("_rlnVoltage\n")
sf.write("_rlnSphericalAberration\n")
sf.write("_rlnAmplitudeContrast\n")
### write info to starfile
olddx = 0
micn = 0
oldimgid = None
for i in range(len(particledata)):
if i % 1000 == 0:
print "done with %d particles" % i
### CTF info
image = particledata[i]['particle']['image']
imgid = image.dbid
try:
ctf = ctfdb.getCtfValueForCtfRunId(image, ctfrunid, msg=False)
dx = ctf['defocus1'] * 10e9
dy = ctf['defocus2'] * 10e9
astig = ctf['angle_astigmatism']
except:
ctf = None
# print "didn't find CTF values for image ", image.dbid
if ctf is None:
if oldimgid != imgid:
print "particle %d: " % i, "getting best value for image: %d" % imgid
ctf = ctfdb.getBestCtfValueForImage(image, msg=False, method='ctffind')
dx = ctf[0]['defocus1'] * 10e9
dy = ctf[0]['defocus2'] * 10e9
astig = ctf[0]['angle_astigmatism']
oldctf = ctf
oldimgid = imgid
else:
try:
ctf = oldctf
dx = oldctf[0]['defocus1'] * 10e9
dy = oldctf[0]['defocus2'] * 10e9
astig = oldctf[0]['angle_astigmatism']
except:
apDisplay.printError("no CTF information for image")
if dx != olddx:
micn += 1
olddx = dx
### write input Relion parameters
sf.write("%06d@%s%10d%12.3f%12.3f%12.3f%8.3f%8.3f%8.3f\n"
% (i+1, images, micn, dx, dy, astig, voltage, cs, wgh)
)
sf.close()
sys.exit()
#parse params
params = {}
params['stackid'] = int(sys.argv[1])
params['newstackname'] = sys.argv[2]
params['bin'] = int(sys.argv[3])
params['newstackpath'] = os.getcwd()
params['description'] = "stackid %d was scaled by a factor of %d" % (
params['stackid'], params['bin'])
#check for multiple runs in stack
runs_in_stack = apStack.getRunsInStack(params['stackid'])
if len(runs_in_stack) > 1:
print "scalestack.py can't scale this stack because it is a combination of multiple makestack runs."
print "Instead, use makestack to create a new single scaled stack"
sys.exit()
#get stackdata
stackdata = apStack.getStackParticlesFromId(params['stackid'])
#do operations on stack
print "Scaling stack"
scaleStack(stackdata, params)
#commit new stack to db
print "Commiting new stack to db"
commitScaledStack(stackdata, params)
print "Done!"
def makeStackMeanPlot(stackid, gridpoints=16):
if gridpoints > 30:
apDisplay.printError("Too large of a grid")
apDisplay.printMsg("creating Stack Mean Plot montage for stackid: "+str(stackid))
t0 = time.time()
### big stacks are too slow
boxsize = apStack.getStackBoxsize(stackid)
bin = 1
if boxsize is not None:
while boxsize/bin > 128:
bin+=1
apDisplay.printMsg("binning stack by "+str(bin))
stackdata = apStack.getOnlyStackData(stackid, msg=False)
stackfile = os.path.join(stackdata['path']['path'], stackdata['name'])
partdatas = apStack.getStackParticlesFromId(stackid, msg=False)
#check only first 100 particles for now
#partdatas = partdatas[:500]
apFile.removeFile("montage"+str(stackid)+".png")
### find limits
limits = {'minmean': 1e12, 'maxmean': -1e12, 'minstdev': 1e12, 'maxstdev': -1e12,}
for partdata in partdatas:
if partdata['mean'] is None:
continue
mean = partdata['mean']
stdev = partdata['stdev']
if mean < limits['minmean']:
limits['minmean'] = mean
if mean > limits['maxmean']:
limits['maxmean'] = mean
if stdev < limits['minstdev']:
limits['minstdev'] = stdev
if stdev > limits['maxstdev']:
limits['maxstdev'] = stdev
if limits['minmean'] > 1e11:
apDisplay.printWarning("particles have no mean values in database")
return
apDisplay.printMsg(str(limits))
### create particle bins
partlists = {}
for i in range(gridpoints):
for j in range(gridpoints):
key = ("%02dx%02d"%(i,j))
partlists[key] = []
### sort paritcles into bins
for partdata in partdatas:
key = meanStdevToKey(partdata['mean'], partdata['stdev'], limits, gridpoints)
partnum = int(partdata['particleNumber'])
partlists[key].append(partnum)
printPlot(partlists, gridpoints)
### createStackAverages
keys = partlists.keys()
keys.sort()
count = 0
backs = "\b\b\b\b\b\b\b\b\b\b\b"
montagestack = "montage"+str(stackid)+".hed"
apFile.removeStack(montagestack)
mystack = []
for key in keys:
count += 1
sys.stderr.write(backs+backs+backs+backs)
sys.stderr.write("% 3d of % 3d, %s: % 6d"%(count, len(keys), key, len(partlists[key])))
avgimg = averageSubStack(partlists[key], stackfile, bin)
if avgimg is not False:
mystack.append(avgimg)
apImagicFile.writeImagic(mystack, montagestack)
sys.stderr.write("\n")
assemblePngs(keys, str(stackid), montagestack)
apDisplay.printMsg("/bin/mv -v montage"+str(stackid)+".??? "+stackdata['path']['path'])
apDisplay.printMsg("finished in "+apDisplay.timeString(time.time()-t0))
def start(self):
self.params['output_fileformat'] = 'mrc'
newstackname='framealigned.hed'
stackdata=apStack.getStackParticlesFromId(self.params['stackid'])
stackrundata=apStack.getOnlyStackData(self.params['stackid'])
apix=stackrundata['pixelsize']*1e10
kev=stackdata[0]['particle']['image']['scope']['high tension']/1000
origstackpath=os.path.join(stackrundata['path']['path'],stackrundata['name'])
boxsize=stackdata[0]['stackRun']['stackParams']['boxSize']
binning=stackdata[0]['stackRun']['stackParams']['bin']
#determine camera type
cameratype=stackdata[0]['particle']['image']['camera']['ccdcamera']['name']
if self.params['override_camera'] is not None:
cameratype=self.params['override_camera']
#create sorted boxfiles
imagedict={}
masterlist=[]
for particle in stackdata:
parentimage=particle['particle']['image']['filename']
if parentimage in imagedict.keys():
imagedict[parentimage].append(particle['particle'])
else:
imagedict[parentimage]=[]
imagedict[parentimage].append(particle['particle'])
index=len(imagedict[parentimage])-1
masterlist.append({'particle':particle,'key':parentimage,'index':index})
#print masterlist
for key in imagedict:
particlelst=imagedict[key]
parentimage=key
framespath=particlelst[0]['image']['session']['frame path']
print cameratype
if 'Gatan' in cameratype:
#prepare frames
print framespath
#prepare frame directory
framespathname=os.path.join(self.params['rundir'],parentimage+'.frames')
if os.path.exists(framespathname):
pass
else:
os.mkdir(framespathname)
print framespathname
mrcframestackname=parentimage+'.frames.mrc'
print mrcframestackname
nframes=particlelst[0]['image']['camera']['nframes']
print "Extracting frames for", mrcframestackname
for n in range(nframes):
a=mrc.read(os.path.join(framespath,mrcframestackname),n)
numpil.write(a,imfile=os.path.join(framespathname,'RawImage_%d.tif' % (n)), format='tiff')
elif 'DE' in cameratype:
framespathname=os.path.join(framespath,parentimage+'.frames')
print os.getcwd()
print framespathname
#generate DE script call
if os.path.exists(framespathname):
print "found frames for", parentimage
nframes=particlelst[0]['image']['camera']['nframes']
boxname=parentimage + '.box'
boxpath=os.path.join(framespathname,boxname)
shiftdata={'scale':1,'shiftx':0,'shifty':0}
#flatfield references
brightrefpath=particlelst[0]['image']['bright']['session']['image path']
brightrefname=particlelst[0]['image']['bright']['filename']
brightnframes=particlelst[0]['image']['bright']['camera']['nframes']
darkrefpath=particlelst[0]['image']['dark']['session']['image path']
darkrefname=particlelst[0]['image']['dark']['filename']
darknframes=particlelst[0]['image']['dark']['camera']['nframes']
brightref=os.path.join(brightrefpath,brightrefname+'.mrc')
darkref=os.path.join(darkrefpath,darkrefname+'.mrc')
print brightref
print darkref
apBoxer.processParticleData(particle['particle']['image'],boxsize,particlelst,shiftdata,boxpath)
print framespathname
#set appion specific options
self.params['gainreference_filename']=brightref
self.params['gainreference_framecount']=brightnframes
self.params['darkreference_filename']=darkref
self.params['darkreference_framecount']=darknframes
self.params['input_framecount']=nframes
self.params['boxes_fromfiles']=1
#self.params['run_verbosity']=3
self.params['output_invert']=0
#self.params['radiationdamage_apix=']=apix
self.params['radiationdamage_voltage']=kev
#self.params['boxes_boxsize']=boxsize
outpath=os.path.join(self.params['rundir'],key)
if os.path.exists(outpath):
shutil.rmtree(outpath)
os.mkdir(outpath)
command=['deProcessFrames.py']
keys=self.params.keys()
keys.sort()
for key in keys:
param=self.params[key]
#print key, param, type(param)
if param == None or param=='':
pass
else:
option='--%s=%s' % (key,param)
command.append(option)
command.append(outpath)
command.append(framespathname)
print command
if self.params['dryrun'] is False:
subprocess.call(command)
#recreate particle stack
for n,particledict in enumerate(masterlist):
parentimage=particledict['key']
correctedpath=os.path.join(self.params['rundir'],parentimage)
print correctedpath
if os.path.exists(correctedpath):
correctedparticle=glob.glob(os.path.join(correctedpath,('%s.*.region_%03d.*' % (parentimage,particledict['index']))))
print os.path.join(correctedpath,('%s.*.region_%03d.*' % (parentimage,particledict['index'])))
print correctedparticle
#sys.exit()
command=['proc2d',correctedparticle[0], newstackname]
if self.params['output_rotation'] !=0:
command.append('rot=%d' % self.params['output_rotation'])
if self.params['show_DE_command'] is True:
print command
subprocess.call(command)
else:
print "did not find frames for ", parentimage
command=['proc2d', origstackpath, newstackname,('first=%d' % n), ('last=%d' % n)]
print command
if self.params['dryrun'] is False:
subprocess.call(command)
#upload stack
#make keep file
self.params['keepfile']='keepfile.txt'
f=open(self.params['keepfile'],'w')
for n in range(len(masterlist)):
f.write('%d\n' % (n))
f.close()
apStack.commitSubStack(self.params, newname=newstackname)
apStack.averageStack(stack=newstackname)
print "Done!!!!"
def generateParticleParams(self):
paramfile = os.path.join(self.params['rundir'], 'params.iter000.par')
apDisplay.printMsg("Creating parameter file: " + paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if os.path.isfile(paramfile):
f = open(paramfile, 'r')
numparam = len(f.readlines())
if numparam > self.params['last']:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning(
"Param file exists with too few particles: %d vs %d" %
(numparam, numpart))
stackdata = apStack.getStackParticlesFromId(self.params['stackid'])
particleparams = {}
f = open(paramfile, 'w')
f.write(
"C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n"
)
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if (count % 200 == 0):
estime = (time.time() - t0) * (numpart - count) / float(count)
apDisplay.printMsg("particle %d -- %s remain" %
(count, apDisplay.timeString(estime)))
if count > self.params['last']:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
'ptclnum': particle['particleNumber'],
'psi': 0.0,
'theta': 0.0,
'phi': 0.0,
'df1': -1.0, # workaround if no ctf correction
'df2': -1.0, # set defocus to -1.0 Angstroms
'angast': 0.0,
'mag': 10000, # workaround to get around dstep
'shx': 0.0,
'shy': 0.0,
'film': 1,
'presa': 0.0,
'dpres': 0.0,
}
imagedata = particle['particle']['image']
if self.params['noctf'] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(
imagedata, msg=False, method=self.params['ctfmethod'])
if ctfdata is not None:
### use defocus and astigmatism values
particleparams['df1'] = abs(ctfdata['defocus1'] * 1e10)
particleparams['df2'] = abs(ctfdata['defocus2'] * 1e10)
particleparams['angast'] = ctfdata['angle_astigmatism']
# if using parameters from previous reconstruction
if self.params['reconiterid'] is not None:
emaneuler = self.getStackParticleEulersForIteration(
particle['particleNumber'])
frealigneulers = apFrealign.convertEmanEulersToFrealign(
emaneuler, sym=self.params['sym'])
particleparams['psi'] = frealigneulers['psi']
particleparams['theta'] = frealigneulers['theta']
particleparams['phi'] = frealigneulers['phi']
particleparams['shx'] = emaneuler['shiftx']
particleparams['shy'] = emaneuler['shifty']
if emaneuler['mirror'] is True:
particleparams['shx'] *= -1
self.writeParticleParamLine(particleparams, f)
f.close()
return paramfile
def generateParticleParams(self):
paramfile = os.path.join(self.params["rundir"], "params.iter000.par")
apDisplay.printMsg("Creating parameter file: " + paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params["stackid"])
if os.path.isfile(paramfile):
f = open(paramfile, "r")
numparam = len(f.readlines())
if numparam > self.params["last"]:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning("Param file exists with too few particles: %d vs %d" % (numparam, numpart))
stackdata = apStack.getStackParticlesFromId(self.params["stackid"])
particleparams = {}
f = open(paramfile, "w")
f.write("C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n")
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if count % 200 == 0:
estime = (time.time() - t0) * (numpart - count) / float(count)
apDisplay.printMsg("particle %d -- %s remain" % (count, apDisplay.timeString(estime)))
if count > self.params["last"]:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
"ptclnum": particle["particleNumber"],
"psi": 0.0,
"theta": 0.0,
"phi": 0.0,
"df1": -1.0, # workaround if no ctf correction
"df2": -1.0, # set defocus to -1.0 Angstroms
"angast": 0.0,
"mag": 10000, # workaround to get around dstep
"shx": 0.0,
"shy": 0.0,
"film": 1,
"presa": 0.0,
"dpres": 0.0,
}
imagedata = particle["particle"]["image"]
if self.params["noctf"] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(
imagedata, msg=False, method=self.params["ctfmethod"]
)
if ctfdata is not None:
### use defocus and astigmatism values
particleparams["df1"] = abs(ctfdata["defocus1"] * 1e10)
particleparams["df2"] = abs(ctfdata["defocus2"] * 1e10)
particleparams["angast"] = ctfdata["angle_astigmatism"]
# if using parameters from previous reconstruction
if self.params["reconiterid"] is not None:
emaneuler = self.getStackParticleEulersForIteration(particle["particleNumber"])
frealigneulers = apFrealign.convertEmanEulersToFrealign(emaneuler, sym=self.params["sym"])
particleparams["psi"] = frealigneulers["psi"]
particleparams["theta"] = frealigneulers["theta"]
particleparams["phi"] = frealigneulers["phi"]
particleparams["shx"] = emaneuler["shiftx"]
particleparams["shy"] = emaneuler["shifty"]
if emaneuler["mirror"] is True:
particleparams["shx"] *= -1
self.writeParticleParamLine(particleparams, f)
f.close()
return paramfile
def getStackParticleParams(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
if 'last' not in self.params:
self.params['last'] = len(stackPartList)
firstImageId = stackPartList[0]['particle']['image'].dbid
count = 0
lastImageId = -1
lastCtfData = None
lastKv = -1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
t0 = time.time()
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n" %
(count, self.params['last']))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning(
"particle number in database is not in sync")
if count > self.params['last']:
break
partParams = {}
partParams['ptclnum'] = count
partParams['filmNum'] = self.getFilmNumber(stackPart, firstImageId)
#print partParams['filmNum']
### get image data
imagedata = stackPart['particle']['image']
if self.originalStackData.defocpair is True:
imagedata = apDefocalPairs.getDefocusPair(imagedata)
if lastImageId == imagedata.dbid:
ctfdata = lastCtfData
partParams['kv'] = lastKv
else:
ctfdata = ctfdb.getBestCtfValue(
imagedata, msg=False, method=self.params['ctfmethod'])
partParams['kv'] = imagedata['scope']['high tension'] / 1000.0
lastCtfData = ctfdata
lastImageId = imagedata.dbid
lastKv = partParams['kv']
### get CTF data from image
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1'] * 1e10)
partParams['defocus2'] = abs(ctfdata['defocus2'] * 1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata[
'amplitude_contrast']
else:
apDisplay.printWarning(
"No ctf information for particle %d in image %d" %
(count, imagedata.dbid))
partParams['defocus1'] = 0.1
partParams['defocus2'] = 0.1
partParams['angle_astigmatism'] = 0.0
partParams['amplitude_contrast'] = 0.07
if self.params['reconiterid'] is not None:
eulerDict = self.getStackParticleEulersForIteration(stackPart)
partParams.update(eulerDict)
partParamsList.append(partParams)
print "no class %d ; mismatch %d" % (self.noClassification,
self.mismatch)
sys.stderr.write("\ndone in %s\n\n" %
(apDisplay.timeString(time.time() - t0)))
return partParamsList
sys.exit()
#parse params
params={}
params['stackid']=int(sys.argv[1])
params['newstackname']=sys.argv[2]
params['bin']=int(sys.argv[3])
params['newstackpath']=os.getcwd()
params['description']="stackid %d was scaled by a factor of %d" % (params['stackid'],params['bin'])
#check for multiple runs in stack
runs_in_stack=apStack.getRunsInStack(params['stackid'])
if len(runs_in_stack) > 1:
print "scalestack.py can't scale this stack because it is a combination of multiple makestack runs."
print "Instead, use makestack to create a new single scaled stack"
sys.exit()
#get stackdata
stackdata=apStack.getStackParticlesFromId(params['stackid'])
#do operations on stack
print "Scaling stack"
scaleStack(stackdata,params)
#commit new stack to db
print "Commiting new stack to db"
commitScaledStack(stackdata,params)
print "Done!"
def start(self):
subtomorunq = appiondata.ApSubTomogramRunData()
subtomorundata = subtomorunq.direct_query(self.params['subtomoId'])
volshape, totalbin, pixelsize = apTomo.getSubvolumeInfo(subtomorundata)
if volshape is None:
apDisplay.printError('No subvolume exists for the subtomoId')
sessionname = subtomorundata['session']['name']
stackq = appiondata.ApStackData()
stackdata = stackq.direct_query(self.params['stackId'])
diameter = apStack.getStackParticleDiameter(stackdata)
diameterpixel = diameter * 1e-10 / pixelsize
halfwidth = diameterpixel / 4
ztolerance = halfwidth
zbackgroundrange = max(((volshape[0] - diameterpixel * 3) / 2, 10))
if self.params['commit']:
avgrundata = apTomo.insertTomoAverageRun(
self.params['runname'],
self.params['rundir'],
subtomorundata,
stackdata,
halfwidth,
self.params['description'],
)
profiles = {}
sumvol = numpy.zeros(volshape)
substacktype, conditionstackdata = apStack.findSubStackConditionData(
stackdata)
if substacktype in ['clustersub', 'alignsub']:
alignstack = apStack.getAlignStack(substacktype,
conditionstackdata)
alignpackage = apAlignment.getAlignPackage(alignstack['alignrun'])
stackprtls = apStack.getStackParticlesFromId(stackdata.dbid)
i = 0
for stackp in stackprtls:
alignp = apAlignment.getAlignParticle(stackp, alignstack)
shift = apAlignment.getAlignShift(alignp, alignpackage)
subtomodata = apTomo.getSubTomogramData(subtomorundata, stackp)
subtomofile = os.path.join(subtomodata['path']['path'],
subtomodata['name'] + '.rec')
subvolume = apTomo.getTomoVolume(subtomodata)
if subvolume is not None:
zcenter = volshape[0] / 2
profile = apTomo.getParticleCenterZProfile(
subvolume, shift, halfwidth, zbackgroundrange)
subtomoid = subtomodata.dbid
profiles[subtomoid] = profile
center = apTomo.gaussianCenter(profile)
if center > zcenter - ztolerance and center < zcenter + ztolerance:
i += 1
shiftz = zcenter - center
transformedvolume = apTomo.transformTomo(
subvolume, subtomofile, alignpackage, alignp,
shiftz, totalbin)
## write transformed mrc file to check the result
filename = os.path.join(
self.params['rundir'],
'./transformed%05d.mrc' % subtomoid)
mrc.write(transformedvolume, filename)
sumvol += transformedvolume
t = numpy.sum(transformedvolume, axis=0)
filename = os.path.join(self.params['rundir'],
'./p%05d.mrc' % subtomoid)
mrc.write(transformedvolume, filename)
if self.params['commit']:
apTomo.insertTomoAvgParticle(
avgrundata, subtomodata, alignp, shiftz)
if i < 1:
apDisplay.printError('no subtomogram qualifies for averaging')
else:
avgvol = sumvol / i
avgvolfilename = sessionname + "_" + self.params['runname'] + ".mrc"
avgvolpath = os.path.join(self.params['rundir'], avgvolfilename)
mrc.write(avgvol, avgvolpath)
if not os.path.isfile(avgvolpath):
apDisplay.printError("tomogram not exist")
apTomo.makeMovie(avgvolpath, self.params['maxsize'])
apTomo.makeProjection(avgvolpath, self.params['maxsize'])
proshape = profile.shape
for id in profiles.keys():
out = open('profile_%05d.txt' % id, 'w')
for z in range(0, proshape[0]):
str = "%5d\t" % z
str += "%6.3f\t" % profiles[id][z]
str += "\n"
out.write(str)
out.close()
def start(self):
subtomorunq = appiondata.ApSubTomogramRunData()
subtomorundata = subtomorunq.direct_query(self.params['subtomoId'])
volshape,totalbin,pixelsize = apTomo.getSubvolumeInfo(subtomorundata)
if volshape is None:
apDisplay.printError('No subvolume exists for the subtomoId')
sessionname = subtomorundata['session']['name']
stackq = appiondata.ApStackData()
stackdata = stackq.direct_query(self.params['stackId'])
diameter = apStack.getStackParticleDiameter(stackdata)
diameterpixel = diameter * 1e-10 / pixelsize
halfwidth = diameterpixel / 4
ztolerance = halfwidth
zbackgroundrange = max(((volshape[0] - diameterpixel*3)/2,10))
if self.params['commit']:
avgrundata = apTomo.insertTomoAverageRun(self.params['runname'],
self.params['rundir'],
subtomorundata,
stackdata,
halfwidth,
self.params['description'],
)
profiles = {}
sumvol = numpy.zeros(volshape)
substacktype,conditionstackdata = apStack.findSubStackConditionData(stackdata)
if substacktype in ['clustersub','alignsub']:
alignstack = apStack.getAlignStack(substacktype,conditionstackdata)
alignpackage = apAlignment.getAlignPackage(alignstack['alignrun'])
stackprtls = apStack.getStackParticlesFromId(stackdata.dbid)
i = 0
for stackp in stackprtls:
alignp = apAlignment.getAlignParticle(stackp,alignstack)
shift = apAlignment.getAlignShift(alignp,alignpackage)
subtomodata = apTomo.getSubTomogramData(subtomorundata,stackp)
subtomofile = os.path.join(subtomodata['path']['path'],subtomodata['name']+'.rec')
subvolume = apTomo.getTomoVolume(subtomodata)
if subvolume is not None:
zcenter = volshape[0] / 2
profile = apTomo.getParticleCenterZProfile(subvolume,shift,halfwidth,zbackgroundrange)
subtomoid = subtomodata.dbid
profiles[subtomoid] = profile
center = apTomo.gaussianCenter(profile)
if center > zcenter - ztolerance and center < zcenter + ztolerance:
i += 1
shiftz = zcenter - center
transformedvolume = apTomo.transformTomo(subvolume,subtomofile,alignpackage,alignp,shiftz,totalbin)
## write transformed mrc file to check the result
filename = os.path.join(self.params['rundir'],'./transformed%05d.mrc' %subtomoid)
mrc.write(transformedvolume,filename)
sumvol += transformedvolume
t = numpy.sum(transformedvolume,axis=0)
filename = os.path.join(self.params['rundir'],'./p%05d.mrc' %subtomoid)
mrc.write(transformedvolume,filename)
if self.params['commit']:
apTomo.insertTomoAvgParticle(avgrundata,subtomodata,alignp,shiftz)
if i < 1:
apDisplay.printError('no subtomogram qualifies for averaging')
else:
avgvol = sumvol / i
avgvolfilename = sessionname+"_"+self.params['runname']+".mrc"
avgvolpath = os.path.join(self.params['rundir'],avgvolfilename)
mrc.write(avgvol,avgvolpath)
if not os.path.isfile(avgvolpath):
apDisplay.printError("tomogram not exist")
apTomo.makeMovie(avgvolpath,self.params['maxsize'])
apTomo.makeProjection(avgvolpath,self.params['maxsize'])
proshape = profile.shape
for id in profiles.keys():
out = open('profile_%05d.txt'%id,'w')
for z in range(0,proshape[0]):
str = "%5d\t" % z
str += "%6.3f\t" % profiles[id][z]
str += "\n"
out.write(str)
out.close()
def checkConflicts(self):
### setup correct database after we have read the project id
if 'projectid' in self.params and self.params['projectid'] is not None:
apDisplay.printMsg("Using split database")
# use a project database
newdbname = apProject.getAppionDBFromProjectId(self.params['projectid'])
sinedon.setConfig('appiondata', db=newdbname)
apDisplay.printColor("Connected to database: '"+newdbname+"'", "green")
if self.params['stackid'] is None:
apDisplay.printError("stackid was not defined")
if self.params['expweight'] is False:
apDisplay.printWarning("Exposure weighting is turned off, make sure this is what you want")
if self.params['localavg'] is False:
apDisplay.printWarning("Trajectory local averaging is turned off, make sure this is what you want")
# DD processes
self.dd = apDDprocess.DDStackProcessing()
print self.dd
# get stack data
self.stackdata = appiondata.ApStackData.direct_query(self.params['stackid'])
self.stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
self.sessiondata = apStack.getSessionDataFromStackId(self.params['stackid'])
# query image
qimage = self.stackparts[0]['particle']['image']
# pixel size info
self.params['apix'] = apStack.getMicrographPixelSizeFromStackId(self.params['stackid'])
self.params['box'] = self.stackdata['boxsize']
self.params['particleradius'] = self.params['particleradius'] / self.params['apix']
if self.params['particleradius'] > self.params['box'] / 2.0:
apDisplay.printWarning("specified particle radius greater than box radius, \
setting particle radius to 0.8 * boxsize")
# micrograph & frame info
frames = qimage['use frames']
nframes = len(frames)
if self.params['framelastali'] is None:
self.params['framelastali'] = frames[-1]
if self.params['framelastave'] is None:
self.params['framelastave'] = frames[-1]
# microscope kV
self.params['kv'] = qimage['scope']['high tension']/1000.0
# query exposure per frame, if not set here
if self.params['total_dose'] is not None:
dose = self.params['total_dose']
else:
try:
dose = apDatabase.getDoseFromImageData(qimage)
except:
apDisplay.printError("dose not specified and not in database, please specify explicitly")
if self.params['expperframe'] is None and self.params['expweight'] is True:
if dose is not None:
self.params['expperframe'] = dose / nframes
else:
apDisplay.printError("exposure per frame needs to be specified, cannot find in database")
# dimensions
self.params['framex'] = int(apDatabase.getDimensionsFromImageData(qimage)['x'])
self.params['framey'] = int(apDatabase.getDimensionsFromImageData(qimage)['y'])
# DD info
self.dd.setImageData(qimage)
self.dd.setDDStackRun(self.params['ddstackid'])
self.ddstackpath = self.dd.getDDStackRun()['path']['path']
def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq["name"] = self.params["runname"]
selectrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
if self.params["ddstack"]:
self.other_ddstack_used = []
self.dd = apDDprocess.DDStackProcessing()
self.dd.setDDStackRun(self.params["ddstack"])
self.newddstackrun = self.dd.getDDStackRun(show_msg=True)
### stack data
stackdata = apStack.getOnlyStackData(self.params["stackid"])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params["stackid"], msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]["particle"]["selectionrun"]
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq["diam"] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq["oldselectionrun"] = oldselectrun
manualparamsq["trace"] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq["name"] = self.params["runname"]
selectrunq["hidden"] = False
selectrunq["path"] = appiondata.ApPathData(path=os.path.abspath(self.params["rundir"]))
selectrunq["session"] = apStack.getSessionDataFromStackId(self.params["stackid"])
selectrunq["manparams"] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count % 100 == 0:
perpart = (time.time() - t0) / float(count + 1)
apDisplay.printColor(
"part %d of %d :: %.1fM mem :: %s/part :: %s remain"
% (
count,
numpart,
(mem.active() - startmem) / 1024.0,
apDisplay.timeString(perpart),
apDisplay.timeString(perpart * (numpart - count)),
),
"blue",
)
oldpartdata = stackpart["particle"]
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq["selectionrun"] = selectrunq
if self.params["ddstack"]:
newimagedata = self.getNewImageFromDDStack(oldpartdata["image"])
if newimagedata is False:
# no pick transferred
continue
newpartq["image"] = newimagedata
if self.params["commit"] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s" % (apDisplay.timeString(time.time() - t0)))
def calculateEulerJumpsForEntireRecon(self,
reconrunid,
stackid=None,
sym=None,
multimodelrunid=None):
if sym is None:
sym = self.getSymmetry(reconrunid)
if re.match("^icos", sym.lower()):
apDisplay.printWarning("Processing Icos symmetry " + sym)
if not re.match("^[cd][0-9]+$", sym.lower()) and not re.match(
"^icos", sym.lower()):
apDisplay.printError(
"Cannot calculate euler jumps for symmetry: " + sym)
return
### get stack particles
if stackid is None:
stackid = apStack.getStackIdFromRecon(reconrunid, msg=False)
stackparts = apStack.getStackParticlesFromId(stackid)
stackparts.sort(self.sortStackParts)
numparts = len(stackparts)
### start loop
t0 = time.time()
medians = []
count = 0
miscount = 0
for stackpart in stackparts:
count += 1
if multimodelrunid is None:
jumpdata = self.getEulerJumpData(reconrunid,
stackpartid=stackpart.dbid,
stackid=stackid,
sym=sym)
else:
jumpdata = self.getEulerJumpData(
reconrunid,
stackpartid=stackpart.dbid,
stackid=stackid,
sym=sym,
multimodelrunid=multimodelrunid)
if jumpdata is None:
if miscount < 25:
continue
else:
break
medians.append(jumpdata['median'])
if count % 500 == 0:
timeremain = (time.time() - t0) / (count + 1) * (numparts -
count)
print("particle=% 5d; median jump=% 3.2f, remain time= %s" %
(stackpart['particleNumber'], jumpdata['median'],
apDisplay.timeString(timeremain)))
if len(medians) > 0:
### print stats
apDisplay.printMsg("complete " + str(len(stackparts)) +
" particles in " +
apDisplay.timeString(time.time() - t0))
print "-- median euler jumper stats --"
medians = numpy.asarray(medians, dtype=numpy.float32)
print("mean/std :: " + str(round(medians.mean(), 2)) + " +/- " +
str(round(medians.std(), 2)))
print("min/max :: " + str(round(medians.min(), 2)) + " <> " +
str(round(medians.max(), 2)))
else:
apDisplay.printWarning(
"no Euler jumpers inserted into the database, make sure that the angles are read by the recon uploader"
)
return
def start(self):
"""
for each particle in the stack, get the information that RELION needs
"""
stackPartList = apStack.getStackParticlesFromId(self.params['stackid'])
nptcls = len(stackPartList)
currentImageId = stackPartList[0]['particle']['image'].dbid
count = 0
imagenumber = 1
partParamsList = []
sys.stderr.write("reading stack particle data\n")
#create list of dictionaries that will be passed to starFile maker later
for stackPart in stackPartList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
if count % 10000 == 0:
sys.stderr.write("\nparticle %d of %d\n" % (count, nptcls))
# extra particle number information not read by Relion
if count != stackPart['particleNumber']:
apDisplay.printWarning(
"particle number in database is not in sync")
partParams = {}
partParams['ptclnum'] = count
### get image data
imagedata = stackPart['particle']['image']
if imagedata.dbid != currentImageId:
imagenumber += 1
currentImageId = imagedata.dbid
partParams['filmNum'] = imagenumber
#print partParams['filmNum']
partParams['kv'] = imagedata['scope']['high tension'] / 1000.0
partParams['cs'] = imagedata['scope']['tem']['cs'] * 1000
### get CTF data from image
ctfdata = ctfdb.getBestCtfValue(imagedata,
msg=False,
sortType='maxconf')
if ctfdata is not None:
# use defocus & astigmatism values
partParams['defocus1'] = abs(ctfdata['defocus1'] * 1e10)
partParams['defocus2'] = abs(ctfdata['defocus2'] * 1e10)
partParams['angle_astigmatism'] = ctfdata['angle_astigmatism']
partParams['amplitude_contrast'] = ctfdata[
'amplitude_contrast']
else:
apDisplay.printError(
"No ctf information for particle %d in image %s" %
(count, imagedata['filename']))
partParamsList.append(partParams)
###now make star file
#first make header
star = starFile.StarFile(self.params['outstar'])
labels = [
'_rlnImageName',
'_rlnMicrographName',
'_rlnDefocusU',
'_rlnDefocusV',
'_rlnDefocusAngle',
'_rlnVoltage',
'_rlnSphericalAberration',
'_rlnAmplitudeContrast',
]
valueSets = [] #list of strings for star file
###now make particle data
for partParams in partParamsList:
relionDataLine = (
"%[email protected] mic%d %.6f %.6f %.6f %d %.6f %.6f" % (
partParams['ptclnum'],
partParams['filmNum'],
partParams['defocus2'],
partParams['defocus1'],
partParams['angle_astigmatism'],
partParams['kv'],
partParams['cs'],
partParams['amplitude_contrast'],
))
valueSets.append(relionDataLine)
star.buildLoopFile("data_", labels, valueSets)
star.write()
def removePtclsByJumps(self, particles, rejectlst):
eulerjump = apEulerJump.ApEulerJump()
numparts = len(particles)
apDisplay.printMsg("finding euler jumps for " + str(numparts) +
" particles")
### check symmetry
symmetry = eulerjump.getSymmetry(self.params['reconid'], msg=True)
if not re.match("^[cd][0-9]+$", symmetry.lower()) and not re.match(
"^icos", symmetry.lower()):
apDisplay.printError(
"Cannot calculate euler jumps for symmetry: " + symmetry)
return
self.params['sym'] = symmetry.lower()
### prepare file
f = open('jumps.txt', 'w', 0666)
f.write("#pnum\t")
headerlist = ('mean', 'median', 'stdev', 'min', 'max')
for key in headerlist:
f.write(key + "\t")
f.write("\n")
### get stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'])
### start loop
t0 = time.time()
medians = []
count = 0
apDisplay.printMsg("processing euler jumps for recon run=" +
str(self.params['reconid']))
for stackpart in stackparts:
count += 1
partnum = stackpart['particleNumber']
f.write('%d\t' % partnum)
jumpdata = eulerjump.getEulerJumpData(
self.params['reconid'],
stackpartid=stackpart.dbid,
stackid=self.params['stackid'],
sym=symmetry)
medians.append(jumpdata['median'])
if (jumpdata['median'] >
self.params['avgjump']) and partnum not in rejectlst:
rejectlst.append(partnum)
for key in headerlist:
f.write("%.3f\t" % (jumpdata[key]))
f.write("\n")
if count % 1000 == 0:
timeremain = (time.time() - t0) / (count + 1) * (numparts -
count)
apDisplay.printMsg(
"particle=% 5d; median jump=% 3.2f, remain time= %s" %
(partnum, jumpdata['median'],
apDisplay.timeString(timeremain)))
#f.flush()
### print stats
apDisplay.printMsg("-- median euler jumper stats --")
medians = numpy.asarray(medians, dtype=numpy.float32)
apDisplay.printMsg("mean/std :: " + str(round(medians.mean(), 2)) +
" +/- " + str(round(medians.std(), 2)))
apDisplay.printMsg("min/max :: " + str(round(medians.min(), 2)) +
" <> " + str(round(medians.max(), 2)))
perrej = round(
100.0 * float(numparts - len(rejectlst)) / float(numparts), 2)
apDisplay.printMsg("keeping " + str(numparts - len(rejectlst)) +
" of " + str(numparts) + " particles (" +
str(perrej) + "%) so far " + " in " +
apDisplay.timeString(time.time() - t0))
return rejectlst
def start(self):
### default parameters
starfile = self.params['starfile']
images = self.params['images']
dbnum = self.params['dbnum']
ctfrunid = self.params['ctfrunid']
stackid = self.params['stackid']
voltage = self.params['voltage']
cs = self.params['cs']
wgh = self.params['wgh']
### particles, angles
appiondata.sinedon.setConfig('appiondata', db="ap%d" % dbnum)
particledata = apStack.getStackParticlesFromId(stackid)
### write Relion starfile header
sf = open(starfile, "w")
sf.write("data_images\n")
sf.write("loop_\n")
sf.write("_rlnImageName\n")
sf.write("_rlnMicrographName\n")
sf.write("_rlnDefocusU\n")
sf.write("_rlnDefocusV\n")
sf.write("_rlnDefocusAngle\n")
sf.write("_rlnVoltage\n")
sf.write("_rlnSphericalAberration\n")
sf.write("_rlnAmplitudeContrast\n")
### write info to starfile
olddx = 0
micn = 0
oldimgid = None
for i in range(len(particledata)):
if i % 1000 == 0:
print "done with %d particles" % i
### CTF info
image = particledata[i]['particle']['image']
imgid = image.dbid
try:
ctf = ctfdb.getCtfValueForCtfRunId(image, ctfrunid, msg=False)
dx = ctf['defocus1'] * 10e9
dy = ctf['defocus2'] * 10e9
astig = ctf['angle_astigmatism']
except:
ctf = None
# print "didn't find CTF values for image ", image.dbid
if ctf is None:
if oldimgid != imgid:
print "particle %d: " % i, "getting best value for image: %d" % imgid
ctf = ctfdb.getBestCtfValueForImage(image,
msg=False,
method='ctffind')
dx = ctf[0]['defocus1'] * 10e9
dy = ctf[0]['defocus2'] * 10e9
astig = ctf[0]['angle_astigmatism']
oldctf = ctf
oldimgid = imgid
else:
try:
ctf = oldctf
dx = oldctf[0]['defocus1'] * 10e9
dy = oldctf[0]['defocus2'] * 10e9
astig = oldctf[0]['angle_astigmatism']
except:
apDisplay.printError("no CTF information for image")
if dx != olddx:
micn += 1
olddx = dx
### write input Relion parameters
sf.write("%06d@%s%10d%12.3f%12.3f%12.3f%8.3f%8.3f%8.3f\n" %
(i + 1, images, micn, dx, dy, astig, voltage, cs, wgh))
sf.close()
def generateParticleParams(self):
paramfile = os.path.join(self.params['rundir'], 'params.iter000.par')
apDisplay.printMsg("Creating parameter file: "+paramfile)
numpart = apStack.getNumberStackParticlesFromId(self.params['stackid'])
if os.path.isfile(paramfile):
f = open(paramfile, 'r')
numparam = len(f.readlines())
if numparam > self.params['last']:
apDisplay.printMsg("Param file exists")
return paramfile
else:
apDisplay.printWarning("Param file exists with too few particles: %d vs %d"%(numparam,numpart))
stackdata = apStack.getStackParticlesFromId(self.params['stackid'])
particleparams={}
f = open(paramfile, 'w')
f.write("C PSI THETA PHI SHX SHY MAG FILM DF1 DF2 ANGAST PRESA\n")
apDisplay.printMsg("Writing out particle parameters")
count = 0
t0 = time.time()
self.noeulers = 0
for particle in stackdata:
count += 1
if (count % 200 == 0):
estime = (time.time() - t0) * (numpart-count) / float(count)
apDisplay.printMsg("particle %d -- %s remain"%(count, apDisplay.timeString(estime)))
if count > self.params['last']:
break
# defaults
## Niko says that if the defocus is negative it will not perform CTF correction
## But it will also not correct the amplitudes
particleparams = {
'ptclnum': particle['particleNumber'],
'psi': 0.0,
'theta': 0.0,
'phi': 0.0,
'df1': -1.0, # workaround if no ctf correction
'df2': -1.0, # set defocus to -1.0 Angstroms
'angast': 0.0,
'mag': 10000, # workaround to get around dstep
'shx': 0.0,
'shy': 0.0,
'film': 1,
'presa': 0.0,
'dpres': 0.0,
}
imagedata = particle['particle']['image']
if self.params['noctf'] is False:
ctfdata, confidence = ctfdb.getBestCtfValueForImage(imagedata, msg=False, method=self.params['ctfmethod'])
if ctfdata is not None:
### use defocus and astigmatism values
particleparams['df1'] = abs(ctfdata['defocus1']*1e10)
particleparams['df2'] = abs(ctfdata['defocus2']*1e10)
particleparams['angast'] = ctfdata['angle_astigmatism']
# if using parameters from previous reconstruction
if self.params['reconiterid'] is not None:
emaneuler = self.getStackParticleEulersForIteration(particle['particleNumber'])
frealigneulers = apFrealign.convertEmanEulersToFrealign(emaneuler, sym=self.params['sym'])
particleparams['psi'] = frealigneulers['psi']
particleparams['theta'] = frealigneulers['theta']
particleparams['phi'] = frealigneulers['phi']
particleparams['shx'] = emaneuler['shiftx']
particleparams['shy'] = emaneuler['shifty']
if emaneuler['mirror'] is True:
particleparams['shx'] *= -1
self.writeParticleParamLine(particleparams,f)
f.close()
return paramfile
