def insertAlignParticlesIntoDatabase(self, level):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
D = self.getClassificationAtLevel(level)
for ref in D:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref + 1
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = self.params['maxiter']
if ref in self.resdict:
refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info ... NOTE: ALIGNMENT PARAMETERS ARE NOT SAVED IN XMIPP 2.4
for partnum in D[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum) + 1
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'],
int(partnum) +
1) ### particle numbering starts with 0!!!!!!!
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
### insert bad particles
if len(self.badpartlist) > 0:
for p in self.badpartlist:
refq = appiondata.ApAlignReferenceData()
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = self.params['maxiter']
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(p) + 1
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'],
int(p) + 1)
alignpartq['stackpart'] = stackpartdata
alignpartq['bad'] = 1
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " +
str(self.params['numpart']) + " particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def insertParticlesIntoDatabase(self, stackid, partlist):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in partlist:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = partdict['refnum']
refq['iteration'] = self.lastiter
refsearch = "part" + self.params['timestamp'] + "_ref*" + str(
partdict['refnum']) + "*"
refbase = os.path.splitext(glob.glob(refsearch)[0])[0]
refq['mrcfile'] = refbase + ".mrc"
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if partdict['refnum'] in self.resdict:
refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
if not os.path.isfile(reffile):
emancmd = "proc2d " + refbase + ".xmp " + refbase + ".mrc"
apEMAN.executeEmanCmd(emancmd, verbose=False)
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: " +
reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partdict['partnum']
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(stackid,
partdict['partnum'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['inplane']
alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
alignpartq['spread'] = partdict['spread']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " + str(count) +
" particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def insertParticlesIntoDatabase(self, stackid, partList):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in partList:
count += 1
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = partdict['refnum']
if refq['refnum'] < 1:
apDisplay.printError("Classes must start with 1")
refq['iteration'] = self.lastiter
#refq['mrcfile'] = refbase+".mrc" ### need to create mrcs of each class???
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
if partdict['refnum'] in self.resdict:
refq['ssnr_resolution'] = self.resdict[partdict['refnum']]
refq['isac_generation'] = partdict['generation']
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partdict['partnum']
if alignpartq['partnum'] < 1:
apDisplay.printError("Particles must start with 1")
alignpartq['alignstack'] = self.alignstackdata
if self.params['alignstackid'] is None:
stackpartdata = apStack.getStackParticle(
stackid, partdict['origPartNum'])
else:
stackpartdata = apStack.getStackParticleFromAlignParticle(
self.params['alignstackid'], partdict['origPartNum'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['inplane']
alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
alignpartq['score'] = partdict['peak']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " + str(count) +
" particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def insertParticlesIntoDatabase(self, partlist, insert=False):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for particle in partlist:
count += 1
### get all particle parameters
partnum = count
rotation = particle[0] * -1 ### in spider negative rotation is clockwise
shiftx = particle[1]
shifty = particle[2]
ccc = particle[3]
refnum = int(particle[4])
mirror = particle[5]
if count % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['imagicfile'] = "references.hed"
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['templatestack'] = appiondata.ApTemplateStackData.direct_query(self.params['templateStackId'])
reffile = os.path.join(self.params['rundir'], refq['imagicfile'])
if not os.path.isfile(reffile):
apDisplay.printError("could not find reference file: "+reffile)
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = self.alignstackdata
stackpartdata = apStack.getStackParticle(self.params['stackId'], partnum)
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = shiftx
alignpartq['yshift'] = shifty
alignpartq['rotation'] = rotation
alignpartq['mirror'] = mirror
alignpartq['ref'] = refq
alignpartq['correlation'] = ccc
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor("\ninserted "+str(inserted)+" of "+str(count)+" particles into the database in "
+apDisplay.timeString(time.time()-t0), "cyan")
return
def insertAlignParticlesIntoDatabase(self):
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for ref in self.classD:
### setup reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = ref
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refq['iteration'] = 10
# if ref in self.resdict:
# refq['ssnr_resolution'] = self.resdict[ref]
### setup particle info
for partnum in self.classD[ref]:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = int(partnum)
alignpartq['alignstack'] = self.alignstackdata
### particle numbering starts with 1 in SIMPLE
stackpartdata = apStack.getStackParticle(
self.runparams['stackid'], int(partnum))
alignpartq['stackpart'] = stackpartdata
alignpartq['ref'] = refq
try:
alignpartq['xshift'] = self.alignD[partnum - 1]['shx']
except:
pass
try:
alignpartq['yshift'] = self.alignD[partnum - 1]['shy']
except:
pass
try:
alignpartq['mirror'] = self.alignD[partnum - 1]['mirror']
except:
pass
alignpartq['rotation'] = self.alignD[partnum - 1]['rot']
### insert
if self.params['commit'] is True:
inserted += 1
alignpartq.insert()
apDisplay.printColor(
"\ninserted " + str(inserted) + " of " +
str(self.params['numpart']) + " particles into the database in " +
apDisplay.timeString(time.time() - t0), "cyan")
return
def insertRefBasedRun(self, partlist, imagicstack, insert=False):
apDisplay.printMsg("committing results to DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ref based run
refrunq = appiondata.ApRefBasedRunData()
refrunq['runname'] = self.params['runname']
refrunq['xysearch'] = self.params['xysearch']
refrunq['xystep'] = self.params['xystep']
refrunq['first_ring'] = self.params['firstring']
refrunq['last_ring'] = self.params['lastring']
refrunq['num_iter'] = self.params['numiter']
refrunq['invert_templs'] = self.params['inverttemplates']
refrunq['num_templs'] = self.params['numtemplate']
#refrunq['csym', int),
refrunq['run_seconds'] = self.params['runtime']
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['refbasedrun'] = refrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = imagicstack
alignstackq['avgmrcfile'] = "average.mrc"
emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter'])
apEMAN.executeEmanCmd(emancmd)
alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter']))
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: "+imagicfile)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = self.stack['apix']*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### insert reference data
reflist = []
for j in range(self.params['numiter']):
iternum = j+1
for i in range(len(self.templatelist)):
refnum = i+1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum))
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
if refnum in self.resdict:
refq['ssnr_resolution'] = self.resdict[refnum]
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
#refq['varmrcfile', str),
#refq['frc_resolution', float),
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template']-1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
alignpartq['score'] = partdict['score']
alignpartq['mirror'] = partdict['mirror']
if insert is True:
alignpartq.insert()
return
def insertEdIterRun(self, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + runparams['runname'] +
"' and path already exist in database")
### setup ed-iter run
editrunq = appiondata.ApEdIterRunData()
editrunq['runname'] = self.params['runname']
editrunq['radius'] = self.params['partrad']
editrunq['num_iter'] = self.params['numiter']
#editrunq['freealigns'] = self.params['freealigns'] ### no longer an option
editrunq['invert_templs'] = self.params['inverttemplates']
editrunq['num_templs'] = self.params['numtemplate']
#editrunq['csym', int),
editrunq['run_seconds'] = self.runtime
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['editerrun'] = editrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup aligned stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
# stack of aligned particles
alignstackq['imagicfile'] = "alignstack.hed"
# final class averages
alignstackq['refstackfile'] = "avg.hed"
# averaged results
alignstackq['avgmrcfile'] = "average.mrc"
# check to be sure dbpaths to files are correct
refstackfile = os.path.join(
self.params['rundir'],
alignstackq['refstackfile']) #final averages
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile']) #averaged results
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['iteration'] = self.params['numiter']
alignstackq['boxsize'] = math.floor(self.stack['boxsize'] /
self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert(
) #alignstackq contains alignrunq which contains editrunq
### setup data from each iteration
apDisplay.printColor(
"Inserting average and variance images from each iteration, please wait",
"cyan")
reflist = []
for j in range(self.params['numiter']):
iternum = j + 1
for i in range(len(self.templatelist)):
refnum = i + 1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
#refq['frc_resolution'] = #(float)
avgname = os.path.join(self.params['rundir'],
"r%02d/avg%03d" % (iternum, refnum))
varname = os.path.join(self.params['rundir'],
"r%02d/var%03d" % (iternum, refnum))
if os.path.isfile(avgname + ".spi"):
emancmd = "proc2d " + avgname + ".spi " + avgname + ".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['mrcfile'] = (avgname + ".mrc")
emancmd = "proc2d " + varname + ".spi " + varname + ".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['varmrcfile'] = (varname + ".mrc")
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
else:
reflist.append(None)
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
partlist = self.readApshDocFile("apshdoc.spi", "apshdoc.pickle")
for partdict in partlist:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template'] - 1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['rotation'] = partdict['rot']
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['mirror'] = partdict['mirror']
alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert() #insert each particle
return
def commitToDatabase(self):
"""
insert the results into the database
"""
### expected result for an alignment run:
### 1. aligned particle stack in IMAGIC
### 2. rotation, shift, and quality parameters for each particle
### 3. which particles belongs to which class
### 4. stack file with the class averages
alignedstack = os.path.join(self.params['rundir'], "ptcl.hed")
refstack = os.path.join(self.params['rundir'], "iter.final.hed")
averagemrc = os.path.join(self.params['rundir'], "average.mrc")
apStack.averageStack(alignedstack, averagemrc)
particlemapping = self.determineClassOwnership()
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError(
"Run name '" + runparams['runname'] +
"' and path already exisclassmappingt in database")
### setup eman refine2d run
emanrefinetwodq = appiondata.ApEMANRefine2dRunData()
emanrefinetwodq['runname'] = self.params['runname']
emanrefinetwodq['run_seconds'] = time.time() - self.t0
emanrefinetwodq['num_iters'] = self.params['numiter']
emanrefinetwodq['num_classes'] = self.params['numclasses']
### finish alignment run
alignrunq['refine2drun'] = emanrefinetwodq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stackalignimagicfile
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = os.path.basename(alignedstack)
alignstackq['avgmrcfile'] = os.path.basename(averagemrc)
alignstackq['refstackfile'] = os.path.basename(refstack)
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printErrrefqor("could not find reference stack file: " +
refstackfile)
### continue setting values
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = apFile.numImagesInStack(
alignimagicfilepath)
### inserting particles and references
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for emanpartnum in range(self.params['numpart']):
partnum = emanpartnum + 1
if partnum % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refnum = particlemapping[emanpartnum]
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### TODO: create mrc file
#refq['mrcfile'] = refbase+".mrc"
#reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
#if not os.path.isfile(reffile):
# emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc"
# apEMAN.executeEmanCmd(emancmd, verbose=False)
#if not os.path.isfile(reffile):
# apDisplay.printError("could not find reference file: "+reffile)
### TODO: get resolution
#refq['ssnr_resolution'] = TODO
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partnum)
alignpartq['stackpart'] = stackpartdata
### TODO: get the alignment parameters
#alignpartq['xshift'] = partdict['xshift']
#alignpartq['yshift'] = partdict['yshift']
#alignpartq['rotation'] = partdict['inplane']
#alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
### TODO: get the score
#alignpartq['score'] = partdict['score']
### insert
if self.params['commit'] is True:
alignpartq.insert()
return
def insertParticlesIntoDatabase(self, partlist, partrefdict):
# insert particle alignment information into database
count = 0
inserted = 0
t0 = time.time()
apDisplay.printColor("\nPreparing to insert particle alignment data, please wait", "cyan")
# get path data
pathq = appiondata.ApPathData()
pathq['path'] = os.path.abspath(self.params['rundir'])
pathdata = pathq.query(results=1)
pathid = pathdata[0].dbid
# align run id
alignrunid = self.alignstackdata['alignrun'].dbid
# get stack particle ids
stackpdbdict={}
sqlcmd = "SELECT particleNumber,DEF_id "+ \
"FROM ApStackParticleData "+ \
"WHERE `REF|ApStackData|stack`=%i"%(self.params['stackid'])
results = sinedon.directq.complexMysqlQuery('appiondata',sqlcmd)
for part in results:
pnum = int(part['particleNumber'])
stackpdbdict[pnum]=int(part['DEF_id'])
apDisplay.printColor("found "+str(len(results))+" particles in "
+apDisplay.timeString(time.time()-t0), "cyan")
t0 = time.time()
apDisplay.printColor("\nInserting class averages into database","cyan")
# insert reference image data
reflistvals = []
for i in range(1,max(partrefdict.values())+1):
sqlvals="(%i,%i,'%s',%i,%i)"%( \
i,self.params['currentiter'], \
os.path.basename(self.params['currentcls'])+".img", \
alignrunid,pathid)
reflistvals.append(sqlvals)
sqlcmd = "INSERT INTO `ApAlignReferenceData` ("+ \
"`refnum`,`iteration`,`imagicfile`,"+ \
"`REF|ApAlignRunData|alignrun`,`REF|ApPathData|path`) "
sqlcmd += "VALUES "+",".join(reflistvals)
sinedon.directq.complexMysqlQuery('appiondata',sqlcmd)
# get DEF_ids from inserted references
refq = appiondata.ApAlignReferenceData()
refq['iteration'] = self.params['currentiter']
refq['imagicfile'] = os.path.basename(self.params['currentcls'])+".img"
refq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = self.alignstackdata['alignrun']
refresults = refq.query()
# save DEF_ids to dictionary
refdbiddict={}
for ref in refresults:
refdbiddict[ref['refnum']]=ref.dbid
apDisplay.printColor("inserted "+str(len(refdbiddict))+" class averages in "
+apDisplay.timeString(time.time()-t0), "cyan")
t0 = time.time()
apDisplay.printColor("\nAssembling database insertion command","cyan")
partlistvals = []
for partdict in partlist:
count += 1
if count % (len(partlist)/100) == 0:
pleft = int(float(count)/len(partlist)*100)
perpart = (time.time()-t0)/count
tleft = (len(partlist)-count)*perpart
sys.stderr.write("%3i%% complete, %s left \r"%(pleft,apDisplay.timeString(tleft)))
partnum = int(partdict['partnum'])
refnum = partrefdict[partnum]
refnum_dbid = refdbiddict[refnum]
stackpart_dbid = stackpdbdict[partnum]
sqlvals = "(%i,%i,%i,%s,%s,%s,%i,%s,%i)"%( \
partdict['partnum'], alignrunid, stackpart_dbid, \
partdict['xshift'], partdict['yshift'], \
partdict['inplane'], partdict['mirror'], \
partdict['cc'],refnum_dbid)
partlistvals.append(sqlvals)
sys.stderr.write("100% complete\t\n")
apDisplay.printColor("Inserting particle information into database","cyan")
# start big insert cmd
sqlstart = "INSERT INTO `ApAlignParticleData` (" + \
"`partnum`,`REF|ApAlignStackData|alignstack`," + \
"`REF|ApStackParticleData|stackpart`," + \
"`xshift`,`yshift`,`rotation`,`mirror`," + \
"`correlation`,`REF|ApAlignReferenceData|ref`) " + \
"VALUES "
# break up command into groups of 100K inserts
# this is a workaround for the max_allowed_packet at 16MB
n = 100000
sqlinserts = [partlistvals[i:i+n] \
for i in range(0, len(partlistvals), n)]
for sqlinsert in sqlinserts:
sqlcmd=sqlstart+",".join(sqlinsert)
sinedon.directq.complexMysqlQuery('appiondata',sqlcmd)
apDisplay.printColor("\nInserted "+ str(count)+" particles into the database in "
+ apDisplay.timeString(time.time()-t0), "cyan")
def insertNoRefRun(self, imagicstack, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + self.params['runname'] +
"' and path already exist in database")
# create a norefParam object
norefq = appiondata.ApSpiderNoRefRunData()
norefq['runname'] = self.params['runname']
norefq['particle_diam'] = 2.0 * self.params['partrad']
norefq['first_ring'] = self.params['firstring']
norefq['last_ring'] = self.params['lastring']
norefq['init_method'] = self.params['initmethod']
norefq['run_seconds'] = self.runtime
### finish alignment run
alignrunq['norefrun'] = norefq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['description'] = self.params['description']
# STOP HERE
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = os.path.basename(imagicstack)
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = 0
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: " + imagicfile)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average file: " + avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = math.floor(self.stack['boxsize'] /
self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### create reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = 1
refq['iteration'] = 0
refq['mrcfile'] = "template.mrc"
#refpath = os.path.abspath(os.path.join(self.params['rundir'], "alignment"))
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in self.partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = refq
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
#alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert()
return
