def insertAlignStackRunIntoDatabase(self, alignimagicfile, refstackfile):
apDisplay.printMsg("Inserting SIMPLE Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.runparams['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
# if uniquerun:
# apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database")
### finish alignment run
alignrunq['simplerun'] = self.simpleqdata
alignrunq['hidden'] = False
alignrunq['runname'] = self.runparams['runname']
alignrunq['description'] = self.runparams['description']
# alignrunq['lp_filt'] = self.runparams['lowpass']
# alignrunq['hp_filt'] = self.runparams['highpass']
alignrunq['bin'] = self.runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = refstackfile
alignstackq['iteration'] = 10 ### hardcoded in SIMPLE
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(
self.runparams['stackid'])
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = self.apix
alignstackq['description'] = self.runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def insertMaxLikeJob(self):
maxjobq = appiondata.ApMaxLikeJobData()
maxjobq['runname'] = self.params['runname']
maxjobq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
maxjobdatas = maxjobq.query(results=1)
if maxjobdatas:
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignrundata = alignrunq.query(results=1)
if maxjobdatas[0]['finished'] is True or alignrundata:
apDisplay.printError(
"This run name already exists as finished in the database, please change the runname"
)
maxjobq[
'REF|projectdata|projects|project'] = apProject.getProjectIdFromStackId(
self.params['stackid'])
maxjobq['timestamp'] = self.timestamp
maxjobq['finished'] = False
maxjobq['hidden'] = False
if self.params['commit'] is True:
maxjobq.insert()
self.params['maxlikejobid'] = maxjobq.dbid
print "self.params['maxlikejobid']", self.params['maxlikejobid']
return
def insertAlignStackRunIntoDatabase(self, alignimagicfile):
apDisplay.printMsg("Inserting CL2D Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
# if uniquerun:
# apDisplay.printError("Run name '"+self.runparams['runname']+"' and path already exist in database")
### setup cl2d run
cl2dq = appiondata.ApCL2DRunData()
cl2dq['runname'] = self.runparams['runname']
cl2dq['run_seconds'] = self.runparams['runtime']
cl2dq['fast'] = self.runparams['fast']
self.cl2dqdata=cl2dq
### finish alignment run
alignrunq['cl2drun'] = cl2dq
alignrunq['hidden'] = False
alignrunq['runname'] = self.runparams['runname']
alignrunq['description'] = self.runparams['description']
alignrunq['lp_filt'] = self.runparams['lowpass']
alignrunq['hp_filt'] = self.runparams['highpass']
alignrunq['bin'] = self.runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
if self.runparams['align'] is True: ### option to create aligned stack
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_level_%02d_.hed"%(self.Nlevels-1)
alignstackq['iteration'] = self.runparams['maxiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
if self.runparams['align'] is True: ### option to create aligned stack
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(self.runparams['stackid'])
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(self.runparams['stackid'])*self.runparams['bin']
alignstackq['description'] = self.runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def insertRunIntoDatabase(self):
apDisplay.printMsg("Inserting Topology Rep Run into DB")
### set up alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+self.params['runname']+"' and path already exist in database")
### set up topology rep run
toprepq = appiondata.ApTopolRepRunData()
toprepq['runname'] = self.params['runname']
toprepq['mask'] = self.params['mask']
toprepq['itermult'] = self.params['itermult']
toprepq['learn'] = self.params['learn']
toprepq['ilearn'] = self.params['ilearn']
toprepq['age'] = self.params['maxage']
toprepq['mramethod'] = self.params['mramethod']
toprepq['job'] = self.params['topoljob']
### finish alignment run
alignrunq['topreprun'] = toprepq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### set up alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = "mrastack.hed"
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = os.path.basename(self.params['currentcls'])+".hed"
alignstackq['iteration'] = self.params['currentiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: "+imagicfile)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = self.stackdata
alignstackq['boxsize'] = math.floor(self.workingboxsize)
alignstackq['pixelsize'] = self.stack['apix']*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
def checkRunNamePath(self):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + runparams['runname'] +
"' and path already exist in database")
def getAlignRunIdFromName(alignrunname):
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = alignrunname
alignrundatas = alignrunq.query()
if not alignrundatas:
return None
if len(alignrundatas) == 1:
### simpe case
alignrunid = alignrundatas[0].dbid
else:
for alignrundata in alignrundatas:
print alignrundata
apDisplay.printError("Found too many align runs for specified criteria")
apDisplay.printMsg("Found align run id %d"%(alignrunid))
return alignrunid
def insertTopolRepJob(self):
topoljobq = appiondata.ApTopolRepJobData()
topoljobq['runname'] = self.params['runname']
topoljobq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
topoljobdatas = topoljobq.query(results=1)
if topoljobdatas:
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignrundata = alignrunq.query(results=1)
if topoljobdatas[0]['finished'] is True or alignrundata:
apDisplay.printError("This run name already exists as finished in the database, please change the runname")
topoljobq['REF|projectdata|projects|project'] = apProject.getProjectIdFromStackId(self.params['stackid'])
topoljobq['timestamp'] = self.timestamp
topoljobq['finished'] = False
topoljobq['hidden'] = False
if self.params['commit'] is True:
topoljobq.insert()
self.params['topoljob'] = topoljobq
return
def insertRefBasedRun(self, partlist, imagicstack, insert=False):
apDisplay.printMsg("committing results to DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup ref based run
refrunq = appiondata.ApRefBasedRunData()
refrunq['runname'] = self.params['runname']
refrunq['xysearch'] = self.params['xysearch']
refrunq['xystep'] = self.params['xystep']
refrunq['first_ring'] = self.params['firstring']
refrunq['last_ring'] = self.params['lastring']
refrunq['num_iter'] = self.params['numiter']
refrunq['invert_templs'] = self.params['inverttemplates']
refrunq['num_templs'] = self.params['numtemplate']
#refrunq['csym', int),
refrunq['run_seconds'] = self.params['runtime']
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['refbasedrun'] = refrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = imagicstack
alignstackq['avgmrcfile'] = "average.mrc"
emancmd = "proc2d templatestack%02d.spi templatestack%02d.hed"%(self.params['numiter'],self.params['numiter'])
apEMAN.executeEmanCmd(emancmd)
alignstackq['refstackfile'] = ("templatestack%02d.hed"%(self.params['numiter']))
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: "+imagicfile)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = self.boxsize
alignstackq['pixelsize'] = self.stack['apix']*self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### insert reference data
reflist = []
for j in range(self.params['numiter']):
iternum = j+1
for i in range(len(self.templatelist)):
refnum = i+1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refq['mrcfile'] = ("templateavg%02d-%02d.mrc"%(iternum,refnum))
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
if refnum in self.resdict:
refq['ssnr_resolution'] = self.resdict[refnum]
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
#refq['varmrcfile', str),
#refq['frc_resolution', float),
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait", "cyan")
for partdict in partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template']-1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'], partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
alignpartq['score'] = partdict['score']
alignpartq['mirror'] = partdict['mirror']
if insert is True:
alignpartq.insert()
return
def insertEdIterRun(self, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + runparams['runname'] +
"' and path already exist in database")
### setup ed-iter run
editrunq = appiondata.ApEdIterRunData()
editrunq['runname'] = self.params['runname']
editrunq['radius'] = self.params['partrad']
editrunq['num_iter'] = self.params['numiter']
#editrunq['freealigns'] = self.params['freealigns'] ### no longer an option
editrunq['invert_templs'] = self.params['inverttemplates']
editrunq['num_templs'] = self.params['numtemplate']
#editrunq['csym', int),
editrunq['run_seconds'] = self.runtime
### finish alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['editerrun'] = editrunq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
### setup aligned stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
# stack of aligned particles
alignstackq['imagicfile'] = "alignstack.hed"
# final class averages
alignstackq['refstackfile'] = "avg.hed"
# averaged results
alignstackq['avgmrcfile'] = "average.mrc"
# check to be sure dbpaths to files are correct
refstackfile = os.path.join(
self.params['rundir'],
alignstackq['refstackfile']) #final averages
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile']) #averaged results
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['iteration'] = self.params['numiter']
alignstackq['boxsize'] = math.floor(self.stack['boxsize'] /
self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert(
) #alignstackq contains alignrunq which contains editrunq
### setup data from each iteration
apDisplay.printColor(
"Inserting average and variance images from each iteration, please wait",
"cyan")
reflist = []
for j in range(self.params['numiter']):
iternum = j + 1
for i in range(len(self.templatelist)):
refnum = i + 1
templateid = self.templatelist[i]
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = refnum
refq['iteration'] = iternum
refq['template'] = apTemplate.getTemplateFromId(templateid)
refpath = os.path.join(self.params['rundir'], "templates")
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(refpath))
refq['alignrun'] = alignrunq
#refq['frc_resolution'] = #(float)
avgname = os.path.join(self.params['rundir'],
"r%02d/avg%03d" % (iternum, refnum))
varname = os.path.join(self.params['rundir'],
"r%02d/var%03d" % (iternum, refnum))
if os.path.isfile(avgname + ".spi"):
emancmd = "proc2d " + avgname + ".spi " + avgname + ".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['mrcfile'] = (avgname + ".mrc")
emancmd = "proc2d " + varname + ".spi " + varname + ".mrc"
apEMAN.executeEmanCmd(emancmd)
refq['varmrcfile'] = (varname + ".mrc")
if insert is True:
refq.insert()
if iternum == self.params['numiter']:
reflist.append(refq)
else:
reflist.append(None)
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
partlist = self.readApshDocFile("apshdoc.spi", "apshdoc.pickle")
for partdict in partlist:
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = reflist[partdict['template'] - 1]
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['rotation'] = partdict['rot']
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['mirror'] = partdict['mirror']
alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert() #insert each particle
return
def commitToDatabase(self):
"""
insert the results into the database
"""
### expected result for an alignment run:
### 1. aligned particle stack in IMAGIC
### 2. rotation, shift, and quality parameters for each particle
### 3. which particles belongs to which class
### 4. stack file with the class averages
alignedstack = os.path.join(self.params['rundir'], "ptcl.hed")
refstack = os.path.join(self.params['rundir'], "iter.final.hed")
averagemrc = os.path.join(self.params['rundir'], "average.mrc")
apStack.averageStack(alignedstack, averagemrc)
particlemapping = self.determineClassOwnership()
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError(
"Run name '" + runparams['runname'] +
"' and path already exisclassmappingt in database")
### setup eman refine2d run
emanrefinetwodq = appiondata.ApEMANRefine2dRunData()
emanrefinetwodq['runname'] = self.params['runname']
emanrefinetwodq['run_seconds'] = time.time() - self.t0
emanrefinetwodq['num_iters'] = self.params['numiter']
emanrefinetwodq['num_classes'] = self.params['numclasses']
### finish alignment run
alignrunq['refine2drun'] = emanrefinetwodq
alignrunq['hidden'] = False
alignrunq['runname'] = self.params['runname']
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stackalignimagicfile
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = os.path.basename(alignedstack)
alignstackq['avgmrcfile'] = os.path.basename(averagemrc)
alignstackq['refstackfile'] = os.path.basename(refstack)
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printErrrefqor("could not find reference stack file: " +
refstackfile)
### continue setting values
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = apFile.numImagesInStack(
alignimagicfilepath)
### inserting particles and references
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for emanpartnum in range(self.params['numpart']):
partnum = emanpartnum + 1
if partnum % 100 == 0:
sys.stderr.write(".")
### setup reference
refq = appiondata.ApAlignReferenceData()
refnum = particlemapping[emanpartnum]
refq['refnum'] = refnum
refq['iteration'] = self.params['numiter']
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### TODO: create mrc file
#refq['mrcfile'] = refbase+".mrc"
#reffile = os.path.join(self.params['rundir'], refq['mrcfile'])
#if not os.path.isfile(reffile):
# emancmd = "proc2d "+refstack+".xmp "+refstack+".mrc"
# apEMAN.executeEmanCmd(emancmd, verbose=False)
#if not os.path.isfile(reffile):
# apDisplay.printError("could not find reference file: "+reffile)
### TODO: get resolution
#refq['ssnr_resolution'] = TODO
### setup particle
alignpartq = appiondata.ApAlignParticleData()
alignpartq['partnum'] = partnum
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partnum)
alignpartq['stackpart'] = stackpartdata
### TODO: get the alignment parameters
#alignpartq['xshift'] = partdict['xshift']
#alignpartq['yshift'] = partdict['yshift']
#alignpartq['rotation'] = partdict['inplane']
#alignpartq['mirror'] = partdict['mirror']
alignpartq['ref'] = refq
### TODO: get the score
#alignpartq['score'] = partdict['score']
### insert
if self.params['commit'] is True:
alignpartq.insert()
return
def insertRunIntoDatabase(self, alignimagicfile, runparams):
apDisplay.printMsg("Inserting MaxLike Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = runparams['runname']
alignrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '"+runparams['runname']+"' and path already exist in database")
### setup max like run
maxlikeq = appiondata.ApMaxLikeRunData()
maxlikeq['runname'] = runparams['runname']
maxlikeq['run_seconds'] = runparams['runtime']
#maxlikeq['mask_diam'] = 2.0*runparams['maskrad']
maxlikeq['fast'] = runparams['fast']
maxlikeq['fastmode'] = runparams['fastmode']
maxlikeq['mirror'] = runparams['mirror']
maxlikeq['student'] = bool(runparams['student'])
maxlikeq['init_method'] = "xmipp default"
maxlikeq['job'] = self.getMaxLikeJob(runparams)
### finish alignment run
alignrunq['maxlikerun'] = maxlikeq
alignrunq['hidden'] = False
alignrunq['runname'] = runparams['runname']
alignrunq['description'] = runparams['description']
alignrunq['lp_filt'] = runparams['lowpass']
alignrunq['hp_filt'] = runparams['highpass']
alignrunq['bin'] = runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = alignimagicfile
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "part"+self.params['timestamp']+"_average.hed"
alignstackq['iteration'] = self.lastiter
alignstackq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'], alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: "+alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'], alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: "+avgmrcfile)
refstackfile = os.path.join(self.params['rundir'], alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: "+refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(runparams['stackid'])*runparams['bin']
alignstackq['description'] = runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = runparams['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def insertRunIntoDatabase(self, alignedPartStack, alignedClassStack,
runparams):
apDisplay.printMsg("Inserting ISAC Run into DB")
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = runparams['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + runparams['runname'] +
"' and path already exist in database")
### setup ISAC like run
isacq = appiondata.ApSparxISACRunData()
isacq['runname'] = runparams['runname']
isacq['job'] = self.getISACJobData(runparams)
### finish alignment run
alignrunq['isacrun'] = isacq
alignrunq['hidden'] = False
alignrunq['runname'] = runparams['runname']
alignrunq['description'] = runparams['description']
alignrunq['lp_filt'] = runparams['lowpass']
alignrunq['hp_filt'] = runparams['highpass']
alignrunq['bin'] = runparams['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = alignedPartStack
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = alignedClassStack
alignstackq['iteration'] = self.lastiter
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
alignimagicfilepath = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(alignimagicfilepath):
apDisplay.printError("could not find stack file: " +
alignimagicfilepath)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average mrc file: " +
avgmrcfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(runparams['stackid'])
alignstackq['boxsize'] = apFile.getBoxSize(alignimagicfilepath)[0]
alignstackq['pixelsize'] = apStack.getStackPixelSizeFromStackId(
self.params['stackid']) * self.params['bin']
alignstackq['description'] = runparams['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.numPart
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def insertAlignmentRun(self, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + self.params['runname'] +
"' and path already exist in database")
### setup Reference preparation parameters, if given
MRAq = appiondata.ApMultiRefAlignRunData()
if self.params['refs'] is True:
MRAq['lowpass_refs'] = self.params['lowpass_refs']
MRAq['thresh_refs'] = self.params['thresh_refs']
MRAq['maskrad_refs'] = self.params['maskrad_refs']
### setup Multi Reference Alignment Run
MRAq['mirror'] = self.params['mirror']
MRAq['center'] = self.params['center']
MRAq['alignment_type'] = self.params['alignment_type']
if self.params['alignment_type'].lower() == "all":
MRAq['first_alignment'] = self.params['first_alignment']
if self.params['alignment_type'].lower() == "brute_force":
MRAq['num_orientations'] = self.params['num_orientations']
MRAq['max_shift_orig'] = self.params['max_shift_orig']
if self.params['center'] is True:
MRAq['max_shift_this'] = self.params['max_shift_this']
MRAq['samp_param'] = self.params['samp_param']
MRAq['min_radius'] = self.params['minrad']
MRAq['max_radius'] = self.params['minrad']
MRAq['numiter'] = self.params['numiter']
### finish alignment run
alignrunq['imagicMRA'] = MRAq
alignrunq['hidden'] = False
alignrunq['description'] = self.params['description']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['bin'] = self.params['bin']
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['imagicfile'] = "alignstack.hed"
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['refstackfile'] = "references.hed"
alignstackq['iteration'] = self.params['numiter']
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
alignstackq['alignrun'] = alignrunq
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: " + imagicfile)
refstackfile = os.path.join(self.params['rundir'],
alignstackq['refstackfile'])
if not os.path.isfile(refstackfile):
apDisplay.printError("could not find reference stack file: " +
refstackfile)
alignstackq['stack'] = apStack.getOnlyStackData(self.params['stackId'])
alignstackq['boxsize'] = self.params['boxsize']
alignstackq['pixelsize'] = self.params['apix']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
### insert
if self.params['commit'] is True:
alignstackq.insert()
self.alignstackdata = alignstackq
return
def insertNoRefRun(self, imagicstack, insert=False):
### setup alignment run
alignrunq = appiondata.ApAlignRunData()
alignrunq['runname'] = self.params['runname']
alignrunq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
uniquerun = alignrunq.query(results=1)
if uniquerun:
apDisplay.printError("Run name '" + self.params['runname'] +
"' and path already exist in database")
# create a norefParam object
norefq = appiondata.ApSpiderNoRefRunData()
norefq['runname'] = self.params['runname']
norefq['particle_diam'] = 2.0 * self.params['partrad']
norefq['first_ring'] = self.params['firstring']
norefq['last_ring'] = self.params['lastring']
norefq['init_method'] = self.params['initmethod']
norefq['run_seconds'] = self.runtime
### finish alignment run
alignrunq['norefrun'] = norefq
alignrunq['hidden'] = False
alignrunq['bin'] = self.params['bin']
alignrunq['hp_filt'] = self.params['highpass']
alignrunq['lp_filt'] = self.params['lowpass']
alignrunq['description'] = self.params['description']
# STOP HERE
### setup alignment stack
alignstackq = appiondata.ApAlignStackData()
alignstackq['alignrun'] = alignrunq
alignstackq['imagicfile'] = os.path.basename(imagicstack)
alignstackq['avgmrcfile'] = "average.mrc"
alignstackq['alignrun'] = alignrunq
alignstackq['iteration'] = 0
alignstackq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
### check to make sure files exist
imagicfile = os.path.join(self.params['rundir'],
alignstackq['imagicfile'])
if not os.path.isfile(imagicfile):
apDisplay.printError("could not find stack file: " + imagicfile)
avgmrcfile = os.path.join(self.params['rundir'],
alignstackq['avgmrcfile'])
if not os.path.isfile(avgmrcfile):
apDisplay.printError("could not find average file: " + avgmrcfile)
alignstackq['stack'] = self.stack['data']
alignstackq['boxsize'] = math.floor(self.stack['boxsize'] /
self.params['bin'])
alignstackq['pixelsize'] = self.stack['apix'] * self.params['bin']
alignstackq['description'] = self.params['description']
alignstackq['hidden'] = False
alignstackq['num_particles'] = self.params['numpart']
if insert is True:
alignstackq.insert()
### create reference
refq = appiondata.ApAlignReferenceData()
refq['refnum'] = 1
refq['iteration'] = 0
refq['mrcfile'] = "template.mrc"
#refpath = os.path.abspath(os.path.join(self.params['rundir'], "alignment"))
refq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
refq['alignrun'] = alignrunq
### insert particle data
apDisplay.printColor("Inserting particle alignment data, please wait",
"cyan")
for partdict in self.partlist:
### see apSpider.alignment.alignStack() for more info
"""
partdict.keys()
'num': int(data[0]), #SPIDER NUMBERING: 1,2,3,...
'template': int(abs(templatenum)), #SPIDER NUMBERING: 1,2,3,...
'mirror': checkMirror(templatenum),
'score': float(data[3]),
'rot': float(data[4]),
'xshift': float(data[5]),
'yshift': float(data[6]),
"""
alignpartq = appiondata.ApAlignParticleData()
alignpartq['ref'] = refq
alignpartq['partnum'] = partdict['num']
alignpartq['alignstack'] = alignstackq
stackpartdata = apStack.getStackParticle(self.params['stackid'],
partdict['num'])
alignpartq['stackpart'] = stackpartdata
alignpartq['xshift'] = partdict['xshift']
alignpartq['yshift'] = partdict['yshift']
alignpartq['rotation'] = partdict['rot']
#alignpartq['score'] = partdict['score']
if insert is True:
alignpartq.insert()
return
