def start(self):
### check for existing run
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrundata = selectrunq.query(readimages=False)
if selectrundata:
apDisplay.printError("Runname already exists")
### stack data
stackdata = apStack.getOnlyStackData(self.params['stackid'])
### stack particles
stackparts = apStack.getStackParticlesFromId(self.params['stackid'], msg=True)
stackparts.reverse()
### selection run for first particle
oldselectrun = stackparts[0]['particle']['selectionrun']
### set selection run
manualparamsq = appiondata.ApManualParamsData()
manualparamsq['diam'] = self.getDiamFromSelectionRun(oldselectrun)
manualparamsq['oldselectionrun'] = oldselectrun
manualparamsq['trace'] = False
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq['hidden'] = False
selectrunq['path'] = appiondata.ApPathData(path=os.path.abspath(self.params['rundir']))
selectrunq['session'] = apStack.getSessionDataFromStackId(self.params['stackid'])
selectrunq['manparams'] = manualparamsq
### insert particles
apDisplay.printMsg("Inserting particles into database")
count = 0
t0 = time.time()
startmem = mem.active()
numpart = len(stackparts)
for stackpart in stackparts:
count += 1
if count > 10 and count%100 == 0:
perpart = (time.time()-t0)/float(count+1)
apDisplay.printColor("part %d of %d :: %.1fM mem :: %s/part :: %s remain"%
(count, numpart, (mem.active()-startmem)/1024. , apDisplay.timeString(perpart),
apDisplay.timeString(perpart*(numpart-count))), "blue")
oldpartdata = stackpart['particle']
newpartq = appiondata.ApParticleData(initializer=oldpartdata)
newpartq['selectionrun'] = selectrunq
if self.params['commit'] is True:
newpartq.insert()
apDisplay.printMsg("Completed in %s"%(apDisplay.timeString(time.time()-t0)))
def checkPreviousTemplateRun(self):
### check if we have a previous selection run
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq[
'session'] = sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
rundatas = selectrunq.query(results=1)
if not rundatas:
return True
rundata = rundatas[0]
### check if we have a previous template run
templaterunq = appiondata.ApTemplateRunData(selectionrun=rundata)
templatedatas = templaterunq.query()
if not templatedatas:
return True
### make sure of using same number of templates
if len(self.params['templateIds']) != len(templatedatas):
apDisplay.printError("different number of templates from last run")
### make sure we have same rotation parameters
for i, templateid in enumerate(self.params['templateIds']):
templaterunq = appiondata.ApTemplateRunData()
templaterunq['selectionrun'] = rundata
templaterunq[
'template'] = appiondata.ApTemplateImageData.direct_query(
templateid)
### this is wrong only check last template run not this run
queryresult = templaterunq.query(results=1)
if queryresult:
templatedata = queryresult[0]
return True
def getNumSelectionRunsFromSession(sessionname):
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
selectq = appiondata.ApSelectionRunData()
selectq['session'] = sessiondata
selectdatas = selectq.query()
if not selectdatas:
return 0
return len(selectdatas)
def getRecentSelectionIdFromSession(sessionname):
sessiondata = apDatabase.getSessionDataFromSessionName(sessionname)
selectq = appiondata.ApSelectionRunData()
selectq['session'] = sessiondata
selectdatas = selectq.query(results=1)
if not selectdatas:
return None
return selectdatas[0].dbid
def getImagePixelSizeFromContourId(contourid):
# This returns pixelsize in Angstrom
q = appiondata.ApSelectionRunData()
rundata = q.direct_query(contourid)
q = appiondata.ApContourData(selectionrun=rundata)
r = q.query(results=1)
lastimagedata = r[0]['image']
apix = apDatabase.getPixelSize(lastimagedata)
return apix
def insertCenterSelectionRunData(self, sessiondata, selectionrunname):
manparamsq = appiondata.ApManualParamsData()
manparamsq['bin'] = self.fullbin
manparamsq['tomocenter'] = True
runq = appiondata.ApSelectionRunData(session=sessiondata,
name=selectionrunname,
manparams=manparamsq)
runq.insert()
self.params['selexonId'] = runq.dbid
def getContourPickerDataFileName(sessionname,contourid):
q = appiondata.ApSelectionRunData()
rundata = q.direct_query(contourid)
if rundata:
contourrunpath = rundata['path']['path']
datafilepath = os.path.join(contourrunpath,'contourpickerData-'+sessionname+'.txt')
if os.path.isfile(datafilepath):
return datafilepath
return False
def commitSizingRun(params, method=None):
q = appiondata.ApSelectionRunData()
tracerundata = q.direct_query(params['contourid'])
pathq = appiondata.ApPathData()
pathq['path'] = params['rundir']
q = q.direct_query(params['contourid'])
q = appiondata.ApSizingRunData()
q['name'] = params['runname']
q['tracerun'] = tracerundata
q['path'] = pathq
q['method'] = method
q.insert()
return q
def getParticlesForImageFromRunName(imgdata, runname):
"""
returns particles for a given image and selection run name
"""
srunq = appiondata.ApSelectionRunData()
srunq['name'] = runname
srunq['session'] = imgdata['session']
ptclq = appiondata.ApParticleData()
ptclq['image'] = imgdata
ptclq['selectionrun'] = srunq
particles = ptclq.query()
return particles
def checkPreviousTemplateRun(self):
### check if we have a previous selection run
selectrunq = appiondata.ApSelectionRunData()
selectrunq['name'] = self.params['runname']
selectrunq[
'session'] = sessiondata = apDatabase.getSessionDataFromSessionName(
self.params['sessionname'])
rundatas = selectrunq.query(results=1)
if not rundatas:
return True
rundata = rundatas[0]
### check if we have a previous template run
templaterunq = appiondata.ApTemplateRunData(selectionrun=rundata)
templatedatas = templaterunq.query()
if not templatedatas:
return True
### make sure of using same number of templates
if len(self.params['templateIds']) != len(templatedatas):
apDisplay.printError("different number of templates from last run")
### make sure we have same rotation parameters
for i, templateid in enumerate(self.params['templateIds']):
templaterunq = appiondata.ApTemplateRunData()
templaterunq['selectionrun'] = rundata
templaterunq[
'template'] = appiondata.ApTemplateImageData.direct_query(
templateid)
### this is wrong only check last template run not this run
templatedata = templaterunq.query(results=1)[0]
if (templatedata['range_start'] !=
self.params["startang" + str(i + 1)]
or templatedata['range_end'] !=
self.params["endang" + str(i + 1)]
or templatedata['range_incr'] !=
self.params["incrang" + str(i + 1)]):
print i + 1, templateid
print templatedata['range_start'], self.params["startang" +
str(i + 1)]
print templatedata['range_end'], self.params["endang" +
str(i + 1)]
print templatedata['range_incr'], self.params["incrang" +
str(i + 1)]
apDisplay.printWarning(
"different template search ranges from last run")
return True
def start(self):
apDisplay.printMsg("\n\n")
### get particles
refrun = appiondata.ApRefineRunData.direct_query(
self.params['reconid'])
refiterq = appiondata.ApRefineIterData()
refiterq['refineRun'] = refrun
refiterq['iteration'] = self.params['iternum']
refiterdatas = refiterq.query(results=1)
refpartq = appiondata.ApRefineParticleData()
refpartq['refineIter'] = refiterdatas[0]
#this gets lots of data
refpartdatas = refpartq.query()
### get session
firstrefpart = refpartdatas[0]
firstpart = firstrefpart['particle']['particle']
sessiondata = firstpart['selectionrun']['session']
### create a selection run
runq = appiondata.ApSelectionRunData()
for key in firstpart['selectionrun'].keys():
runq[key] = firstpart['selectionrun'][key]
runq['name'] = self.params['runname']
runq['session'] = sessiondata
pathq = appiondata.ApPathData()
pathq['path'] = self.params['rundir']
runq['path'] = pathq
runq['description'] = (
"Corrected particles from refine id %d iter %d and selection %d" %
(self.params['reconid'], self.params['iternum'],
firstpart['selectionrun'].dbid))
count = 0
for refpartdata in refpartdatas:
count += 1
if count % 10 == 0:
sys.stderr.write(".")
partdata = refpartdata['particle']['particle']
newpartq = appiondata.ApParticleData()
for key in partdata.keys():
newpartq[key] = partdata[key]
newpartq['xcoord'] = partdata['xcoord'] + refpartdata['shiftx']
newpartq['ycoord'] = partdata['ycoord'] + refpartdata['shifty']
newpartq['selectionrun'] = runq
newpartq.insert()
def insertManualParams(self):
runq = appiondata.ApSelectionRunData()
runq['name'] = self.params['runname']
runq['session'] = self.sessiondata
runq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
rundatas = runq.query(results=1)
if rundatas and rundatas[0]['params']['diam'] != self.params['diam']:
apDisplay.printError("upload diameter not the same as last run")
manparams = appiondata.ApSelectionParamsData()
manparams['diam'] = self.params['diam']
runq['params'] = manparams
if self.params['commit'] is True:
apDisplay.printColor(
"Inserting manual selection run into database", "green")
runq.insert()
return runq
def insertSubTomoRun(sessiondata,
selectionrunid,
stackid,
name,
invert=False,
subbin=1):
if selectionrunid:
qpick = appiondata.ApSelectionRunData()
pickdata = qpick.direct_query(selectionrunid)
else:
pickdata = None
if stackid:
qstack = appiondata.ApStackData()
stackdata = qstack.direct_query(stackid)
else:
stackdata = None
qrun = appiondata.ApSubTomogramRunData(session=sessiondata,
pick=pickdata,
stack=stackdata,
runname=name,
invert=invert,
subbin=subbin)
return publish(qrun)
def commitRunToDatabase(self, sessiondata, insert=True):
### this dict maps the contents of self.params to the sinedon object
dbmap = {
'diam': 'diam',
'bin': 'bin',
'lp': 'lp_filt',
'hp': 'hp_filt',
'pixlimit': 'pixel_value_limit',
'invert': 'invert',
#'thresh': 'threshold', # major bug, do not overwrite dict variables!!!
'thresh': 'manual_thresh',
'maxthresh': 'max_threshold',
'maxpeaks': 'max_peaks',
'maxsize': 'maxsize',
'median': 'median',
'defocpair': 'defocal_pairs',
'overlapmult': 'overlapmult',
'helicalinsert': 'helicalinsert',
'helicalstep': 'helicalstep',
}
paramQuery = self.getParticleParamsData()
#insert common parameters
for pkey, dbkey in dbmap.items():
if (dbkey in paramQuery and pkey in self.params
and self.params[pkey] is not None):
paramQuery[dbkey] = self.params[pkey]
runq = appiondata.ApSelectionRunData()
runq['name'] = self.params['runname']
runq['session'] = sessiondata
rundatas = runq.query(results=1)
if rundatas:
#get previous params
if isinstance(paramQuery, appiondata.ApSelectionParamsData):
paramData = rundatas[0]['params']
elif isinstance(paramQuery, appiondata.ApDogParamsData):
paramData = rundatas[0]['dogparams']
elif isinstance(paramQuery, appiondata.ApManualParamsData):
paramData = rundatas[0]['manparams']
elif isinstance(paramQuery, appiondata.ApTiltAlignParamsData):
paramData = rundatas[0]['tiltparams']
else:
apDisplay.printError(
"selection run does not have valid parameter data\n")
#make sure all params are the same as previous session
if not paramData:
apDisplay.printError("No parameters\n")
else:
for key in paramQuery:
if paramData[key] != paramQuery[key] and paramData[
key] is not None:
try:
data_dbid = paramData[key].dbid
query_dbid = paramQuery[key].dbid
if data_dbid != query_dbid:
apDisplay.printWarning(
str(key) + ":" +
str(paramQuery[key].dbid) +
" not equal to " +
str(paramData[key].dbid))
apDisplay.printError(
"All parameters for a picker run name must be identical\n"
)
except:
apDisplay.printWarning(
str(key) + ":" + str(paramQuery[key]) +
" not equal to " + str(paramData[key]))
apDisplay.printError(
"All parameters for a picker run name must be identical\n"
)
#if I made it here all parameters are the same, so it isn't necessary to commit
return rundatas[0]
#set params for run
if isinstance(paramQuery, appiondata.ApSelectionParamsData):
runq['params'] = paramQuery
elif isinstance(paramQuery, appiondata.ApDogParamsData):
runq['dogparams'] = paramQuery
elif isinstance(paramQuery, appiondata.ApManualParamsData):
runq['manparams'] = paramQuery
elif isinstance(paramQuery, appiondata.ApTiltAlignParamsData):
runq['tiltparams'] = paramQuery
else:
apDisplay.printError(
"self.getParticleParamsData() did not return valid parameter data\n"
)
#create path
runq['path'] = appiondata.ApPathData(
path=os.path.abspath(self.params['rundir']))
#add description
if self.params['description']:
runq['description'] = self.params['description']
if insert is True:
runq.insert()
return runq
def insertTiltTransform(imgdata1, imgdata2, tiltparams, params):
#First we need to sort imgdata
#'07aug30b_a_00013gr_00010sq_v01_00002sq_v01_00016en_00'
#'07aug30b_a_00013gr_00010sq_v01_00002sq_01_00016en_01'
#last two digits confer order, but then the transform changes...
bin = params['bin']
### first find the runid
runq = appiondata.ApSelectionRunData()
runq['name'] = params['runname']
runq['session'] = imgdata1['session']
rundatas = runq.query(results=1, readimages=False)
if not rundatas:
apDisplay.printError("could not find runid in database")
### the order is specified by 1,2; so don't change it let makestack figure it out
for imgdata in (imgdata1, imgdata2):
for index in ("1","2"):
transq = appiondata.ApImageTiltTransformData()
transq["image"+index] = imgdata
transq['tiltrun'] = rundatas[0]
transdata = transq.query(readimages=False)
if transdata:
apDisplay.printWarning("Transform values already in database for "+imgdata['filename'])
return transdata[0]
### prepare the insertion
transq = appiondata.ApImageTiltTransformData()
transq['image1'] = imgdata1
transq['image2'] = imgdata2
transq['tiltrun'] = rundatas[0]
dbdict = tiltPickerToDbNames(tiltparams)
if dbdict is None:
return None
#Can I do for key in appiondata.ApImageTiltTransformData() ro transq???
for key in ('image1_x','image1_y','image1_rotation','image2_x','image2_y','image2_rotation','scale_factor','tilt_angle', 'overlap'):
if key not in dbdict:
apDisplay.printError("Key: "+key+" was not found in transformation data")
for key,val in dbdict.items():
#print key
if re.match("image[12]_[xy]", key):
transq[key] = round(val*bin,2)
else:
transq[key] = val
#print i,v
### this overlap is wrong because the images are binned by 'bin' and now we give it the full image
"""
imgShape1 = numpy.asarray(imgdata1['image'].shape, dtype=numpy.int8)/params['bin']
image1 = numpy.ones(imgShape1)
imgShape2 = numpy.asarray(imgdata2['image'].shape, dtype=numpy.int8)/params['bin']
image2 = numpy.ones(imgShape2)
bestOverlap, tiltOverlap = apTiltTransform.getOverlapPercent(image1, image2, tiltparams)
print "image overlaps", bestOverlap, tiltOverlap
transq['overlap'] = round(bestOverlap,5)
"""
apDisplay.printMsg("Inserting transform between "+apDisplay.short(imgdata1['filename'])+\
" and "+apDisplay.short(imgdata2['filename'])+" into database")
transq.insert()
apDisplay.printMsg("done")
return transq
def getSelectionRunDataFromName(imgdata, runname):
srunq = appiondata.ApSelectionRunData()
srunq['name'] = runname
srunq['session'] = imgdata['session']
selectionrundata = srunq.query()
return selectionrundata[0]
from appionlib import appiondata
import leginon.leginondata
import sys
if __name__ == "__main__":
if len(sys.argv) < 3:
print "usage: ./pcount.py <session name> <particle selection run name>\n\n"
sys.exit(1)
ses_name = sys.argv[1]
run_name = sys.argv[2]
# query objects
qsession = leginon.leginondata.SessionData(name=ses_name)
qrun = appiondata.ApSelectionRunData(session=qsession, name=run_name)
qpart = appiondata.ApParticleData(selectionrun=qrun)
# all particles
particles = qpart.query()
# Find assessment of image that each particle is from.
# This is messy because we want to query images without reading MRCs.
assessments = {}
akeep = []
areject = []
anone = []
for particle in particles:
imgref = particle.special_getitem('image', dereference=False)
imgid = imgref.dbid
if imgid not in assessments:
def insertParticlePeaks(peaktree, imgdata, runname, msg=False):
"""
takes an image data object (imgdata) and inserts particles into DB from peaktree
"""
#INFO
sessiondata = imgdata['session']
imgname = imgdata['filename']
#GET RUN DATA
runq = appiondata.ApSelectionRunData()
runq['name'] = runname
runq['session'] = sessiondata
selectionruns = runq.query(results=1)
if not selectionruns:
apDisplay.printError("could not find selection run in database")
### WRITE PARTICLES TO DATABASE
count = 0
t0 = time.time()
for peakdict in peaktree:
particlesq = appiondata.ApParticleData()
particlesq['selectionrun'] = selectionruns[0]
particlesq['image'] = imgdata
if 'template' in peakdict and peakdict['template'] is not None:
particlesq[
'template'] = appiondata.ApTemplateImageData.direct_query(
peakdict['template'])
for key in 'correlation', 'peakmoment', 'peakstddev', 'peakarea', 'label':
if key in peakdict and peakdict[key] is not None:
if isinstance(peakdict[key], float):
### limit decimals
particlesq[key] = round(peakdict[key], 6)
else:
particlesq[key] = peakdict[key]
### must be integers
particlesq['xcoord'] = int(round(peakdict['xcoord']))
particlesq['ycoord'] = int(round(peakdict['ycoord']))
if 'angle' in peakdict:
particlesq['angle'] = peakdict['angle']
if 'helixnum' in peakdict:
particlesq['helixnum'] = peakdict['helixnum']
if 'helicalstep' in peakdict:
particlesq['helicalstep'] = peakdict['helicalstep']
if 'diameter' in peakdict and peakdict['diameter'] is not None:
peakdict['diameter'] = round(peakdict['diameter'], 6)
if 'peakarea' in peakdict and peakdict[
'peakarea'] is not None and peakdict['peakarea'] > 0:
peakhasarea = True
else:
apDisplay.printWarning("peak has no area")
peakhasarea = False
if 'correlation' in peakdict and peakdict[
'correlation'] is not None and peakdict['correlation'] > 2:
apDisplay.printWarning("peak has correlation greater than 2.0")
### INSERT VALUES
if peakhasarea is True:
presult = particlesq.query()
if not presult:
count += 1
particlesq.insert()
if msg is True:
apDisplay.printMsg("inserted " + str(count) + " of " +
str(len(peaktree)) + " peaks into database" +
" in " + apDisplay.timeString(time.time() - t0))
return
def runRelionPreprocess(self, newstackroot):
'''
1. Use stackIntoPicks.py to extract the particle locations from the selected stack.
2. Run makestack2.py without ctf correction or normalization using the stackIntoPicks result as the Particles run.
3. Run relion_preprocess with rescale and norm. Outputs .mrcs file.
Neil: most of these steps could be done more generally
'''
apDisplay.printWarning("Making a new stack from original images")
# Build the stackIntoPicks command
apDisplay.printMsg(
'Extracting the particle locations from your selected stack.')
newstackrunname = self.params['runname'] + "_particles"
newstackrundir = self.params['rundir']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
sessionid = int(self.params['expid'])
cmd = "stackIntoPicks.py "
cmd += (" --stackid=%d --projectid=%d --runname=%s --rundir=%s " %
(stackid, projectid, newstackrunname, newstackrundir))
cmd += (" --commit --expId=%d --jobtype=stackintopicks " % (sessionid))
# Run the command
logfilepath = os.path.join(newstackrundir, 'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Build the makestack2 command
'''
This is the command we want to make a stack from a stackIntoPicks run
makestack2.py
--single=start.hed --selectionid=130 --invert --boxsize=320 --bin=2
--description="made from stackrun1 using stackintopicks"
--runname=stack66 --rundir=/ami/data00/appion/zz07jul25b/stacks/stack66
--commit --preset=en --projectid=303 --session=zz07jul25b
--no-rejects --no-wait --continue --expid=8556 --jobtype=makestack2
--ppn=1 --nodes=1 --walltime=240 --jobid=2016
'''
apDisplay.printMsg(
'Using selected stack particle locations to make a Relion ready stack....'
)
# Get the ID of the stackIntoPicks run we just created
#apParticle.getSelectionIdFromName(runname, sessionname)
runq = appiondata.ApSelectionRunData()
runq['name'] = newstackrunname
runq['session'] = leginondata.SessionData.direct_query(
self.params['expid'])
rundatas = runq.query(results=1)
print rundatas
if rundatas:
selectionid = rundatas[0].dbid
else:
apDisplay.printError(
"Error creating Relion ready stack. Could not find stackIntoPicks.py data in database.\n"
)
# Gather all the makestack parameters
totalpart = self.originalStackData.numpart
numpart = totalpart if not self.params['last'] else min(
self.params['last'], totalpart)
stackpathname = os.path.basename(self.originalStackData.path)
newstackrunname = self.params['runname']
newstackrundir = self.params['rundir']
newstackimagicfile = os.path.join(newstackrundir, 'start.hed')
presetname = self.originalStackData.preset
# binning is combination of the original binning of the stack and the preparation binnning
bin = self.originalStackData.bin * self.params['bin']
unbinnedboxsize = self.stack['boxsize'] * self.originalStackData.bin
lowpasstext = setArgText(
'lowpass',
(self.params['lowpass'], self.originalStackData.lowpass), False)
highpasstext = setArgText(
'highpass',
(self.params['highpass'], self.originalStackData.highpass), True)
partlimittext = setArgText('partlimit', (numpart, ), False)
xmipp_normtext = setArgText('xmipp-normalize',
(self.params['xmipp-norm'], ), True)
sessionid = int(self.params['expid'])
sessiondata = apDatabase.getSessionDataFromSessionId(sessionid)
sessionname = sessiondata['name']
projectid = self.params['projectid']
stackid = self.originalStackData.stackid
reversetext = '--reverse' if self.originalStackData.reverse else ''
defoctext = '--defocpair' if self.originalStackData.defocpair else ''
inverttext = '--no-invert' if not self.invert else ''
# Build the makestack2 command
cmd = "makestack2.py "
cmd += (" --single=%s --selectionid=%d %s --boxsize=%d --bin=%d " %
(os.path.basename(newstackimagicfile), selectionid, inverttext,
unbinnedboxsize, bin))
cmd += (
" --description='Relion refinestack based on %s(id=%d)' --projectid=%d "
% (stackpathname, stackid, projectid))
cmd += (
" --preset=%s --runname=%s --rundir=%s --session=%s --expId=%d " %
(presetname, newstackrunname, newstackrundir, sessionname,
sessionid))
cmd += " --no-wait --no-commit --no-continue --jobtype=makestack2 "
# Run the command
logfilepath = os.path.join(newstackrundir, 'relionstackrun.log')
returncode = self.runAppionScriptInSubprocess(cmd, logfilepath)
if returncode > 0:
apDisplay.printError('Error in Relion specific stack making')
# Clean up
boxfiles = glob.glob("*.box")
for boxfile in boxfiles:
apFile.removeFile(boxfile)
# Make sure our new stack params reflects the changes made
# Use the same complex equation as in eman clip
clipsize = self.calcClipSize(self.stack['boxsize'], self.params['bin'])
self.stack['boxsize'] = clipsize / self.params['bin']
self.stack['apix'] = self.stack['apix'] * self.params['bin']
self.stack['file'] = newstackroot + '.hed'
# Run Relion pre-process command
'''
Setup the follow relion preprocess command to normalize the new stack:
relion_preprocess \
--o particles --norm --bg_radius 60 --white_dust -1 --black_dust -1 \
--operate_on /ami/data00/appion/zz07jul25b/stacks/stack63_no_xmipp_norm/start.hed
'''
apDisplay.printMsg(
'Running Relion preprocessing to normalize your Relion Ready stack....'
)
bg_radius = math.floor((self.stack['boxsize'] / 2) - 1)
relioncmd = "relion_preprocess "
relioncmd += " --o particles --norm --bg_radius %d " % (bg_radius)
relioncmd += " --white_dust -1 --black_dust -1 --operate_on %s " % (
newstackimagicfile)
apParam.runCmd(relioncmd, package="Relion", verbose=True, showcmd=True)
self.stack['file'] = 'particles.mrcs'
def createStackData(self):
apDisplay.printColor("Starting upload of stack", "blue")
pathq = appiondata.ApPathData()
pathq['path'] = self.params['rundir']
manq = appiondata.ApManualParamsData()
manq['diam'] = self.params['diameter']
manq['trace'] = False
selectq = appiondata.ApSelectionRunData()
selectq['name'] = 'fakestack_' + self.params['runname']
selectq['hidden'] = True
selectq['path'] = pathq
selectq['session'] = self.sessiondata
selectq['manparams'] = manq
stackq = appiondata.ApStackData()
stackq['name'] = "start.hed"
stackq['path'] = pathq
stackq['description'] = self.params['description']
stackq['hidden'] = False
stackq['pixelsize'] = self.params['apix'] * 1e-10
stackq['boxsize'] = self.boxsize
stackq['centered'] = False
stackparamq = appiondata.ApStackParamsData()
stackparamq['boxSize'] = self.boxsize
stackparamq['bin'] = 1
stackparamq['phaseFlipped'] = self.params['ctfcorrect']
if self.params['ctfcorrect'] is True:
stackparamq['fileType'] = "manual"
stackparamq['fileType'] = "imagic"
stackparamq['normalized'] = self.params['normalize']
stackparamq['lowpass'] = 0
stackparamq['highpass'] = 0
stackrunq = appiondata.ApStackRunData()
stackrunq['stackRunName'] = self.params['runname']
stackrunq['stackParams'] = stackparamq
stackrunq['selectionrun'] = selectq
stackrunq['session'] = self.sessiondata
runsinstackq = appiondata.ApRunsInStackData()
runsinstackq['stack'] = stackq
runsinstackq['stackRun'] = stackrunq
if self.params['commit'] is True:
runsinstackq.insert()
if 'syncstackid' in self.params.keys(
) and self.params['syncstackid']:
stackdata = runsinstackq['stack']
stack2data = apStack.getOnlyStackData(
self.params['syncstackid'])
syncq = appiondata.ApSyncStackData(
stack1=stackdata,
stack2=stack2data,
synctype=self.params['synctype'])
syncq.insert()
### for each particle
sys.stderr.write("Starting particle upload")
for i in range(self.numpart):
if i % 100 == 0:
sys.stderr.write(".")
partq = appiondata.ApParticleData()
partq['image'] = None #We have no image, see if this works???
partq['selectionrun'] = selectq
partq['xcoord'] = int(i % 1000)
partq['ycoord'] = int(i / 1000)
partq['diameter'] = self.params['diameter']
stackpartq = appiondata.ApStackParticleData()
stackpartq['particleNumber'] = i + 1
stackpartq['stack'] = stackq
stackpartq['stackRun'] = stackrunq
stackpartq['particle'] = partq
stackpartq['mean'] = 0.0
stackpartq['stdev'] = 1.0
if self.params['commit'] is True:
stackpartq.insert()
sys.stderr.write("\n")
return
def insertParticlePeakPairs(peaktree1, peaktree2, peakerrors, imgdata1,
imgdata2, transdata, runname):
"""
takes both image data (imgdata) and inserts particle pairs into DB from peaktrees
"""
#INFO
sessiondata = imgdata1['session']
legimgid1 = int(imgdata1.dbid)
legimgid2 = int(imgdata2.dbid)
imgname1 = imgdata1['filename']
imgname2 = imgdata2['filename']
#CHECK ARRAY LENGTHS
len1 = len(peaktree1)
len2 = len(peaktree2)
len3 = len(peakerrors)
if len1 != len2 or len2 != len3:
apDisplay.printError("insertParticlePeakPairs particle arrays must have the same length "+\
str(len1)+" "+str(len2)+" "+str(len3))
#GET RUN DATA
runq = appiondata.ApSelectionRunData()
runq['name'] = runname
runq['session'] = sessiondata
selectionruns = runq.query(results=1)
if not selectionruns:
apDisplay.printError("could not find selection run in database")
#GET TRANSFORM DATA
transq = appiondata.ApImageTiltTransformData()
transq['image1'] = imgdata1
transq['image2'] = imgdata2
transq['tiltrun'] = selectionruns[0]
transids = transq.query(results=1)
if not transids:
apDisplay.printError("could not find transform id in database")
### WRITE PARTICLES TO DATABASE
count = 0
t0 = time.time()
last50 = time.time()
apDisplay.printMsg("looping over " + str(len(peaktree1)) + " particles")
for i in range(len(peaktree1)):
remaining_peaks = len(peaktree1) - count
if count and remaining_peaks and remaining_peaks % 50 == 0:
#sys.stderr.write("<"+str(len(peaktree1)-count))
print(
"%d particles remain, %s time remains, %s per particle, %s last 50 particles"
% (
len(peaktree1) - count,
apDisplay.timeString(
(time.time() - t0) / count * (remaining_peaks)),
apDisplay.timeString((time.time() - t0) / count),
apDisplay.timeString(time.time() - last50),
))
last50 = time.time()
peakdict1 = peaktree1[i]
peakdict2 = peaktree2[i]
error = peakerrors[i]
partq1 = appiondata.ApParticleData()
partq1['selectionrun'] = selectionruns[0]
partq1['image'] = imgdata1
partq1['xcoord'] = peakdict1['xcoord']
partq1['ycoord'] = peakdict1['ycoord']
partq1['peakarea'] = 1
partq2 = appiondata.ApParticleData()
partq2['selectionrun'] = selectionruns[0]
partq2['image'] = imgdata2
partq2['xcoord'] = peakdict2['xcoord']
partq2['ycoord'] = peakdict2['ycoord']
partq2['peakarea'] = 1
# I do NOT have to check if particles already exist, because this is a NEW selectionrun
partpairq = appiondata.ApTiltParticlePairData()
partpairq['particle1'] = partq1
partpairq['particle2'] = partq2
#NEED TO LOOK UP TRANSFORM DATA
partpairq['transform'] = transdata
#NEED TO CALCULATE ERROR, ALWAYS POSITIVE
partpairq['error'] = error
#presult = partpairq.query()
#if not presult:
count += 1
partq1.insert(force=True)
partq2.insert(force=True)
partpairq.insert(force=True)
apDisplay.printMsg("inserted " + str(count) + " of " +
str(len(peaktree1)) + " peaks into database" + " in " +
apDisplay.timeString(time.time() - t0))
return
