def _insertAllSteps(self):
"""for each micrograph insert the steps to preprocess it
"""
self._defineBasicParams()
# Write pos files for each micrograph
firstStepId = self._insertFunctionStep('writePosFilesStep')
# For each micrograph insert the steps
#run in parallel
deps = []
for mic in self.inputMics:
localDeps = [firstStepId]
micrographToExtract = mic.getFileName()
baseMicName = removeBaseExt(mic.getFileName())
if self.ctfRelations.hasValue():
micName = mic.getMicName()
mic.setCTF(self.ctfDict[micName])
# If downsample type is 'other' perform a downsample
downFactor = self.downFactor.get()
if self.downsampleType == OTHER and abs(downFactor - 1.) > 0.0001:
fnDownsampled = self._getTmpPath(baseMicName+"_downsampled.xmp")
args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier"
localDeps = [self._insertRunJobStep("xmipp_transform_downsample", args % locals(),prerequisites=localDeps)]
micrographToExtract = fnDownsampled
# If remove dust
if self.doRemoveDust:
fnNoDust = self._getTmpPath(baseMicName+"_noDust.xmp")
thresholdDust = self.thresholdDust.get() #TODO: remove this extra variable
args=" -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f"
localDeps = [self._insertRunJobStep("xmipp_transform_filter", args % locals(),prerequisites=localDeps)]
micrographToExtract = fnNoDust
#self._insertFunctionStep('getCTF', micId, baseMicName, micrographToExtract)
#FIXME: Check only if mic has CTF when implemented ok
#if self.doFlip or mic.hasCTF():
fnCTF=None
if self.ctfRelations.hasValue():
# If the micrograph doesn't come from Xmipp, we need to write
# a Xmipp ctfparam file to perform the phase flip on the micrograph
fnCTF = micrographToCTFParam(mic, self._getTmpPath("%s.ctfParam" % baseMicName))
# Insert step to flip micrograph
if self.doFlip:
localDeps = [self._insertFunctionStep('flipMicrographStep',
baseMicName, fnCTF, micrographToExtract,
prerequisites=localDeps)]
micrographToExtract = self._getTmpPath(baseMicName +"_flipped.xmp")
else:
fnCTF = None
# Actually extract
deps.append(self._insertFunctionStep('extractParticlesStep', mic.getObjId(), baseMicName,
fnCTF, micrographToExtract, prerequisites=localDeps))
# Insert step to create output objects
self._insertFunctionStep('createOutputStep', prerequisites=deps)
def _extractMicrograph(self, mic, doInvert, normalizeArgs, doBorders):
""" Extract particles from one micrograph """
fnLast = mic.getFileName()
baseMicName = pwutils.removeBaseExt(fnLast)
outputRoot = str(self._getExtraPath(baseMicName))
fnPosFile = self._getMicPos(mic)
# If it has coordinates extract the particles
particlesMd = 'particles@%s' % fnPosFile
boxSize = self.boxSize.get()
if exists(fnPosFile):
# Create a list with micrographs operations (programs in xmipp) and
# the required command line parameters (except input/ouput files)
micOps = []
# Check if it is required to downsample your micrographs
downFactor = self.downFactor.get()
def getMicTmp(suffix):
return self._getTmpPath(baseMicName + suffix)
if self.notOne(downFactor):
fnDownsampled = getMicTmp("_downsampled.xmp")
args = "-i %s -o %s --step %f --method fourier"
self.runJob('xmipp_transform_downsample',
args % (fnLast, fnDownsampled, downFactor))
fnLast = fnDownsampled
if self.doRemoveDust:
fnNoDust = getMicTmp("_noDust.xmp")
args = " -i %s -o %s --bad_pixels outliers %f"
self.runJob('xmipp_transform_filter',
args % (fnLast, fnNoDust, self.thresholdDust))
fnLast = fnNoDust
if self._useCTF():
# We need to write a Xmipp ctfparam file
# to perform the phase flip on the micrograph
fnCTF = self._getTmpPath("%s.ctfParam" % baseMicName)
micrographToCTFParam(mic, fnCTF)
# Insert step to flip micrograph
if self.doFlip:
fnFlipped = getMicTmp('_flipped.xmp')
args = " -i %s -o %s --ctf %s --sampling %f"
self.runJob(
'xmipp_ctf_phase_flip', args %
(fnLast, fnFlipped, fnCTF, self._getNewSampling()))
fnLast = fnFlipped
else:
fnCTF = None
args = " -i %s --pos %s" % (fnLast, particlesMd)
args += " -o %s --Xdim %d" % (outputRoot, boxSize)
if doInvert:
args += " --invert"
if fnCTF:
args += " --ctfparam " + fnCTF
if doBorders:
args += " --fillBorders"
self.runJob("xmipp_micrograph_scissor", args)
# Normalize
if normalizeArgs:
self.runJob('xmipp_transform_normalize',
'-i %s.stk %s' % (outputRoot, normalizeArgs))
else:
self.warning("The micrograph %s hasn't coordinate file! " %
baseMicName)
self.warning("Maybe you picked over a subset of micrographs")
# Let's clean the temporary mrc micrographs
if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
pwutils.cleanPattern(self._getTmpPath(baseMicName) + '*')
def _insertAllSteps(self):
"""for each micrograph insert the steps to preprocess it"""
self._setupBasicProperties()
# Write pos files for each micrograph
firstStepId = self._insertFunctionStep('writePosFilesStep')
# For each micrograph insert the steps, run in parallel
deps = []
for mic in self.inputMics:
localDeps = [firstStepId]
fnLast = mic.getFileName()
baseMicName = pwutils.removeBaseExt(mic.getFileName())
if self.ctfRelations.hasValue():
micKey = self.micKey(mic)
mic.setCTF(self.ctfDict[micKey])
def getMicTmp(suffix):
return self._getTmpPath(baseMicName + suffix)
# Create a list with micrographs operations (programs in xmipp) and
# the required command line parameters (except input/ouput files)
micOps = []
# Check if it is required to downsample your micrographs
downFactor = self.downFactor.get()
if self.notOne(downFactor):
fnDownsampled = getMicTmp("_downsampled.xmp")
args = "-i %s -o %s --step %f --method fourier"
micOps.append(('xmipp_transform_downsample',
args % (fnLast, fnDownsampled, downFactor)))
fnLast = fnDownsampled
if self.doRemoveDust:
fnNoDust = getMicTmp("_noDust.xmp")
args = " -i %s -o %s --bad_pixels outliers %f"
micOps.append(('xmipp_transform_filter',
args % (fnLast, fnNoDust, self.thresholdDust)))
fnLast = fnNoDust
if self.ctfRelations.hasValue():
# If the micrograph doesn't come from Xmipp, we need to write
# a Xmipp ctfparam file to perform the phase flip on the micrograph
fnCTF = self._getTmpPath("%s.ctfParam" % baseMicName)
# FIXME: Weird return of fnCTF, could be an internal xmipp one
fnCTF = micrographToCTFParam(mic, fnCTF)
# Insert step to flip micrograph
if self.doFlip:
fnFlipped = getMicTmp('_flipped.xmp')
args = " -i %s -o %s --ctf %s --sampling %f"
micOps.append(('xmipp_ctf_phase_flip',
args % (fnLast, fnFlipped, fnCTF,
self._getNewSampling())))
fnLast = fnFlipped
else:
fnCTF = None
# Actually extract
deps.append(self._insertFunctionStep('extractParticlesStep',
mic.getObjId(), baseMicName,
fnCTF, fnLast, micOps,
self.doInvert.get(),
self._getNormalizeArgs(),
self.doBorders.get(),
prerequisites=localDeps))
# Insert step to create output objects
metaDeps = self._insertFunctionStep('createMetadataImageStep',
prerequisites=deps)
if self.doSort:
screenDep = self._insertFunctionStep('screenParticlesStep',
prerequisites=[metaDeps])
finalDeps = [screenDep]
else:
finalDeps = [metaDeps]
self._insertFunctionStep('createOutputStep', prerequisites=finalDeps)
def _insertAllSteps(self):
"""for each micrograph insert the steps to preprocess it
"""
self._setupBasicProperties()
# Write pos files for each micrograph
firstStepId = self._insertFunctionStep('writePosFilesStep')
# For each micrograph insert the steps
#run in parallel
deps = []
for mic in self.inputMics:
localDeps = [firstStepId]
micrographToExtract = mic.getFileName()
baseMicName = removeBaseExt(mic.getFileName())
if self.ctfRelations.hasValue():
micKey = self.micKey(mic)
mic.setCTF(self.ctfDict[micKey])
# If downsample type is 'other' perform a downsample
downFactor = self.downFactor.get()
if self.downsampleType == OTHER and abs(downFactor - 1.) > 0.0001:
fnDownsampled = self._getTmpPath(baseMicName +
"_downsampled.xmp")
args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier"
localDeps = [
self._insertRunJobStep("xmipp_transform_downsample",
args % locals(),
prerequisites=localDeps)
]
micrographToExtract = fnDownsampled
# If remove dust
if self.doRemoveDust:
fnNoDust = self._getTmpPath(baseMicName + "_noDust.xmp")
thresholdDust = self.thresholdDust.get(
) #TODO: remove this extra variable
args = " -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f"
localDeps = [
self._insertRunJobStep("xmipp_transform_filter",
args % locals(),
prerequisites=localDeps)
]
micrographToExtract = fnNoDust
#self._insertFunctionStep('getCTF', micId, baseMicName, micrographToExtract)
#FIXME: Check only if mic has CTF when implemented ok
#if self.doFlip or mic.hasCTF():
fnCTF = None
if self.ctfRelations.hasValue():
# If the micrograph doesn't come from Xmipp, we need to write
# a Xmipp ctfparam file to perform the phase flip on the micrograph
fnCTF = micrographToCTFParam(
mic, self._getTmpPath("%s.ctfParam" % baseMicName))
# Insert step to flip micrograph
if self.doFlip:
localDeps = [
self._insertFunctionStep('flipMicrographStep',
baseMicName,
fnCTF,
micrographToExtract,
prerequisites=localDeps)
]
micrographToExtract = self._getTmpPath(baseMicName +
"_flipped.xmp")
else:
fnCTF = None
# Actually extract
deps.append(
self._insertFunctionStep('extractParticlesStep',
mic.getObjId(),
baseMicName,
fnCTF,
micrographToExtract,
prerequisites=localDeps))
# Insert step to create output objects
metaDeps = self._insertFunctionStep('createMetadataImageStep',
prerequisites=deps)
if self.doSort:
screenDep = self._insertFunctionStep('screenParticlesStep',
prerequisites=[metaDeps])
finalDeps = [screenDep]
else:
finalDeps = [metaDeps]
self._insertFunctionStep('createOutputStep', prerequisites=finalDeps)