def importCTFStep(self, importFrom):
""" Copy ctfs matching the filename pattern.
"""
ci = self.getImportClass()
inputMics = self.inputMicrographs.get()
ctfSet = self._createSetOfCTF()
ctfSet.setMicrographs(inputMics)
outputMics = self._createSetOfMicrographs()
SetOfMicrographs.copyInfo(outputMics, inputMics)
createOutputMics = False
for mic in inputMics:
for i, (fileName, fileId) in enumerate(self.iterFiles()):
#TODO: Define how to match mics and ctf files (temporary solution is to expect micname inside ctf file name)
if removeBaseExt(mic.getFileName()) in removeBaseExt(fileName):
ctf = ci.importCTF(mic, fileName)
ctfSet.append(ctf)
outputMics.append(mic)
break
else:
# If not CTF is found for a micrograph remove it from output mics
self.warning("CTF for micrograph id %d was not found. Removed from set of micrographs." % mic.getObjId())
createOutputMics = True
self._defineOutputs(outputCTF=ctfSet)
# If some of the micrographs had not ctf a subset of micrographs have been produced
if createOutputMics:
self._defineOutputs(outputMicrographs=outputMics)
self._defineCtfRelation(outputMics, ctfSet)
else:
self._defineCtfRelation(inputMics, ctfSet)
def _getNameExt(self, movieName, postFix, ext):
if movieName.endswith("bz2"):
# removeBaseExt function only eliminate the last extension,
# but if files are compressed, we need to eliminate two extensions:
# bz2 and its own image extension (e.g: mrcs, em, etc)
return removeBaseExt(removeBaseExt(movieName)) + postFix + '.' + ext
else:
return removeBaseExt(movieName) + postFix + '.' + ext
def getMatchingMic(self, coordFile, fileId):
""" Given a coordinates file check if there is a micrograph
that this files matches.
"""
micSet = self.inputMicrographs.get()
if fileId is None:
coordBase = removeBaseExt(coordFile)
for mic in micSet:
micBase = removeBaseExt(mic.getFileName())
if coordBase in micBase or micBase in coordBase: #temporal use of in
return mic
return None
else:
return micSet[fileId]
def __createTemporaryCtfs(self, obj, setOfMics):
""" Create a temporary .sqlite file to visualize CTF while the
protocol has not finished yet.
"""
cleanPath(obj._getPath("ctfs_temporary.sqlite"))
ctfSet = self.protocol._createSetOfCTF("_temporary")
for mic in setOfMics:
micFn = mic.getFileName()
micDir = obj._getExtraPath(removeBaseExt(mic.getFileName()))
samplingRate = mic.getSamplingRate() * self.protocol.ctfDownFactor.get()
mic.setSamplingRate(samplingRate)
out = self.protocol._getCtfOutPath(micDir)
psdFile = self.protocol._getPsdPath(micDir)
if exists(out) and exists(psdFile):
ctfModel = em.CTFModel()
readCtfModel(ctfModel, out,
ctf4=self.protocol.useCtffind4.get())
ctfModel.setPsdFile(psdFile)
ctfModel.setMicrograph(mic)
ctfSet.append(ctfModel)
if not ctfSet.isEmpty():
ctfSet.write()
ctfSet.close()
return ctfSet
def _getMicCTF(mic):
micName = mic.getMicName()
micBase = removeBaseExt(mic.getFileName())
# see if the base name of this mic is contained in other base names
micConflicts = [mc for mc in inputMicBases if micBase in mc and micBase != mc]
if micConflicts:
self.warning('WARNING: Micrograph base name "%s" conflicts with micrograph(s) "%s". '
'Will try to find a unique match...' % (micBase, '", "'.join(micConflicts)))
# check which matching file only matches with this mic and not its conflicts
goodFnMatches = [f for f in files if micBase in f and not any(c in f for c in micConflicts)]
for goodFnMatch in goodFnMatches:
try:
micCtf = ci.importCTF(mic, goodFnMatch)
self.warning("WARNING: Assigned file %s to micrograph %s." % (goodFnMatch, micBase))
return micCtf
except Exception as ex:
self.warning("WARNING: Can't import ctf for micrograph %s from file %s"
% (micBase, goodFnMatch))
continue
else:
return None
else:
for fileName, fileId in self.iterFiles():
if (fileId == mic.getObjId() or
micBase in fileName or micName in fileName):
return ci.importCTF(mic, fileName)
return None
def _insertAllSteps(self):
"""for each micrograph insert the steps to preprocess it
"""
self._defineBasicParams()
# Write pos files for each micrograph
firstStepId = self._insertFunctionStep('writePosFilesStep')
# For each micrograph insert the steps
#run in parallel
deps = []
for mic in self.inputMics:
localDeps = [firstStepId]
micrographToExtract = mic.getFileName()
baseMicName = removeBaseExt(mic.getFileName())
if self.ctfRelations.hasValue():
micName = mic.getMicName()
mic.setCTF(self.ctfDict[micName])
# If downsample type is 'other' perform a downsample
downFactor = self.downFactor.get()
if self.downsampleType == OTHER and abs(downFactor - 1.) > 0.0001:
fnDownsampled = self._getTmpPath(baseMicName+"_downsampled.xmp")
args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier"
localDeps = [self._insertRunJobStep("xmipp_transform_downsample", args % locals(),prerequisites=localDeps)]
micrographToExtract = fnDownsampled
# If remove dust
if self.doRemoveDust:
fnNoDust = self._getTmpPath(baseMicName+"_noDust.xmp")
thresholdDust = self.thresholdDust.get() #TODO: remove this extra variable
args=" -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f"
localDeps = [self._insertRunJobStep("xmipp_transform_filter", args % locals(),prerequisites=localDeps)]
micrographToExtract = fnNoDust
#self._insertFunctionStep('getCTF', micId, baseMicName, micrographToExtract)
#FIXME: Check only if mic has CTF when implemented ok
#if self.doFlip or mic.hasCTF():
fnCTF=None
if self.ctfRelations.hasValue():
# If the micrograph doesn't come from Xmipp, we need to write
# a Xmipp ctfparam file to perform the phase flip on the micrograph
fnCTF = micrographToCTFParam(mic, self._getTmpPath("%s.ctfParam" % baseMicName))
# Insert step to flip micrograph
if self.doFlip:
localDeps = [self._insertFunctionStep('flipMicrographStep',
baseMicName, fnCTF, micrographToExtract,
prerequisites=localDeps)]
micrographToExtract = self._getTmpPath(baseMicName +"_flipped.xmp")
else:
fnCTF = None
# Actually extract
deps.append(self._insertFunctionStep('extractParticlesStep', mic.getObjId(), baseMicName,
fnCTF, micrographToExtract, prerequisites=localDeps))
# Insert step to create output objects
self._insertFunctionStep('createOutputStep', prerequisites=deps)
def filterStep(self, filterType):
""" Apply the selected filter to particles.
Create the set of particles.
"""
particles = self.inputParticles.get()
n = particles.getSize()
OP = self._op
args = []
if not self.usePadding:
OP += ' NP'
if filterType <= FILTER_SPACE_REAL:
args.append(self.filterRadius.get())
else:
args.append('%f %f' % (self.lowFreq, self.highFreq))
if filterType == FILTER_FERMI:
args.append(self.temperature.get())
# Map to spected filter number in Spider for operation FQ
filterNumber = filterType * 2 + 1
# Consider low-pass or high-pass
filterNumber += self.filterMode.get()
imgSet = self._createSetOfParticles()
imgSet.copyInfo(self.inputParticles.get())
self._enterWorkingDir() # Do operations inside the run working dir
spi = SpiderShell(ext=self.getExt()) # Create the Spider process to send commands
particlesStk = removeBaseExt(self.particlesStk)
# Run a loop for filtering
locStr = particlesStk + '@******[part]'
cmds = ['do lb5 [part] = 1,%d' % n,
OP, locStr, locStr, filterNumber] + args + ['lb5']
for c in cmds:
spi.runCmd(c)
spi.close()
self._leaveWorkingDir() # Go back to project dir
for i, img in enumerate(particles):
img.setLocation(i+1, self.particlesStk)
imgSet.append(img)
self._defineOutputs(outputParticles=imgSet)
self._defineTransformRelation(particles, imgSet)
def importCoordinatesStep(self, importFrom, *args):
inputMics = self.inputMicrographs.get()
coordsSet = self._createSetOfCoordinates(inputMics)
coordsSet.setBoxSize(self.boxSize.get())
scaleFactor = self.scale.get()
invertX = self.invertX.get()
invertY = self.invertY.get()
ci = self.getImportClass()
for i, (fileName, fileId) in enumerate(self.iterFiles()):
for mic in inputMics:
if removeBaseExt(mic.getFileName()) in removeBaseExt(fileName):#temporal use of in
def addCoordinate(coord):
coord.setMicrograph(mic)
x = coord.getX()
y = coord.getY()
if scaleFactor != 1.:
x = coord.getX() * scaleFactor
y = coord.getY() * scaleFactor
if invertX:
width = mic.getDim()[0]
x = width - x
if invertY:
height = mic.getDim()[1]
y = height - y
coord.setX(x)
coord.setY(y)
coordsSet.append(coord)
ci.importCoordinates(fileName, addCoordinate=addCoordinate)
break
self._defineOutputs(outputCoordinates=coordsSet)
self._defineSourceRelation(self.inputMicrographs, coordsSet)
def writePosFilesStep(self):
""" Write the pos file for each micrograph on metadata format. """
#self.posFiles = writeSetOfCoordinates(self._getExtraPath(), self.inputCoords)
writeSetOfCoordinates(self._getExtraPath(), self.inputCoords)
# We need to find the mapping (either by micName or micId)
# between the micrographs in the SetOfCoordinates and
# the micrographs (if we are in a different case than 'same as picking'
if self.downsampleType != SAME_AS_PICKING:
micDict = {}
coordMics = self.inputCoords.getMicrographs()
for mic in coordMics:
micBase = removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
micDict[mic.getMicName()] = micPos
micDict[mic.getObjId()] = micPos
if any(mic.getMicName() in micDict for mic in self.inputMics):
micKey = lambda mic: mic.getMicName()
elif any(mic.getObjId() in micDict for mic in self.inputMics):
self.warning('Could not match input micrographs and coordinates '
'micrographs by micName, using micId.')
micKey = lambda mic: mic.getObjId()
else:
raise Exception('Could not match input micrographs and coordinates '
'neither by micName or micId.')
for mic in self.inputMics: # micrograph from input (other)
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' % (micPosCoord, micPos))
moveFile(micPosCoord, micPos)
def _insertAllSteps(self):
self._params = {}
# diameter must be passed in Armstrongs and therefore should be converted
self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate()
# self._params['num-slices'] = self.numberSizes.get()
# self._params['size-range'] = self.sizeRange.get()
self._params['apix'] = self.inputMicrographs.get().getSamplingRate()
self._params['thresh'] = self.threshold.get()
# self._params['max-thresh'] = self.maxThreshold.get()
# self._params['max-area'] = self.maxArea.get()
# self._params['max-peaks'] = self.maxPeaks.get()
args = ""
for par, val in self._params.iteritems():
args += " --%s=%s" % (par, str(val))
if self.invert:
args += " --invert"
args += " " + self.extraParams.get('')
deps = [] # Store all steps ids, final step createOutput depends on all of them
ih = ImageHandler()
for mic in self.inputMicrographs.get():
# Create micrograph folder
micName = mic.getFileName()
micDir = self._getTmpPath(removeBaseExt(micName))
makePath(micDir)
# If needed convert micrograph to mrc format, otherwise link it
if getExt(micName) != ".mrc":
fnMicBase = replaceBaseExt(micName, 'mrc')
inputMic = join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = join(micDir, basename(micName))
createLink(micName, inputMic)
# Insert step to execute program
stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args)
deps.append(stepId)
self._insertFinalSteps(deps)
def importCTFStep(self, importFrom):
""" Copy ctfs matching the filename pattern. """
ci = self.getImportClass()
inputMics = self.inputMicrographs.get()
ctfSet = self._createSetOfCTF()
ctfSet.setMicrographs(inputMics)
outputMics = self._createSetOfMicrographs()
SetOfMicrographs.copyInfo(outputMics, inputMics)
createOutputMics = False
files = [f for f, _ in self.iterFiles()]
n = len(files)
if n == 0:
raise Exception("No files where found in path: '%s'\n"
"matching the pattern: '%s'" %
(self.filesPath, self.filesPattern))
print "Matching files: ", len(files)
for mic in inputMics:
micName = mic.getMicName()
micBase = removeBaseExt(mic.getFileName())
for fileName, fileId in self.iterFiles():
if (fileId == mic.getObjId() or micBase in fileName
or micName in fileName):
ctf = ci.importCTF(mic, fileName)
ctfSet.append(ctf)
outputMics.append(mic)
break
else:
# If not CTF is found for a micrograph remove it from output mics
self.warning(
"CTF for micrograph id %d was not found. Removed from set of micrographs."
% mic.getObjId())
createOutputMics = True
self._defineOutputs(outputCTF=ctfSet)
# If some of the micrographs had not ctf a subset of micrographs have been produced
if createOutputMics:
self._defineOutputs(outputMicrographs=outputMics)
self._defineCtfRelation(outputMics, ctfSet)
else:
self._defineCtfRelation(inputMics, ctfSet)
def launchVesicleViewer(self, vesicle):
print("\n==> Running Vesicle Viewer:")
vesicleBaseName = removeBaseExt(vesicle.getFileName())
vtiName = join(self.vtiPath, vesicleBaseName + '.vti')
args = {'vti_file': vtiName}
if self.source == FROM_GRAPHS:
args['graph_file'] = glob.glob(
self.prot._getExtraPath(vesicleBaseName + '*_edges_2.vtp'))[0]
elif self.source == FROM_FILS:
args['net_file'] = glob.glob(
self.prot._getExtraPath(vesicleBaseName + '*_net.vtp'))[0]
elif self.source == FROM_PICKING:
args['peaks_file'] = glob.glob(
self.prot._getExtraPath(vesicleBaseName + '*_peak.vtp'))[0]
args['surf_file'] = glob.glob(
self.prot._getExtraPath(vesicleBaseName + '*_surf.vtp'))[0]
guiThread(VtkPlot, 'initializePlot', **args)
def launchMembAnnotatorForTomogram(self, tomoMask):
# Tomo files come from one dir, while tomoMask files comes from another, because they were generated in
# different protocols. MembraneAnnotator expects both to be in the same location, so a symbolic link is
# is generated in the extra dir of the segmentation protocol pointing to the selected tomogram
print("\n==> Running Membrane Annotator:")
tomoNameSrc = abspath(tomoMask.getVolName())
tomoName = abspath(
join(getParentFolder(tomoMask.getFileName()),
basename(tomoNameSrc)))
if not exists(tomoName):
symlink(tomoNameSrc, tomoName)
arguments = "inTomoFile '%s' " % tomoName
arguments += "outFilename '%s'" % abspath(
join(self.path, removeBaseExt(tomoName)))
Plugin.runMembraneAnnotator(self.prot,
arguments,
env=Plugin.getMembSegEnviron(),
cwd=self.path)
def readSetOfCoordinates(workDir, micSet, coordSet):
""" Read from Eman .json files.
It is expected a file named: base.json under the workDir.
Params:
workDir: where the Eman boxer output files are located.
micSet: the SetOfMicrographs to associate the .json, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
# Read the boxSize from the e2boxercache/base.json
jsonFnbase = join(workDir, 'e2boxercache', 'base.json')
jsonBoxDict = loadJson(jsonFnbase)
size = int(jsonBoxDict["box_size"])
jsonFninfo = join(workDir, 'info/')
for mic in micSet:
micPosFn = ''.join(glob.glob(jsonFninfo + '*' + removeBaseExt(mic.getFileName()) + '_info.json'))
readCoordinates(mic, micPosFn, coordSet)
coordSet.setBoxSize(size)
def readSetOfCoordinates(workDir, micSet, coordSet):
""" Read from Eman .json files.
It is expected a file named: base.json under the workDir.
Params:
workDir: where the Eman boxer output files are located.
micSet: the SetOfMicrographs to associate the .json, which
name should be the same of the micrographs.
coordSet: the SetOfCoordinates that will be populated.
"""
# Read the boxSize from the e2boxercache/base.json
jsonFnbase = join(workDir, 'e2boxercache', 'base.json')
jsonBoxDict = loadJson(jsonFnbase)
size = int(jsonBoxDict["box_size"])
jsonFninfo = join(workDir, 'info/')
for mic in micSet:
micPosFn = ''.join(glob.glob(jsonFninfo + '*' + removeBaseExt(mic.getFileName()) + '_info.json'))
readCoordinates(mic, micPosFn, coordSet)
coordSet.setBoxSize(size)
def filterStep(self, filterType):
""" Apply the selected filter to particles.
Create the set of particles.
"""
particles = self.inputParticles.get()
n = particles.getSize()
OP = self._op
args = []
if not self.usePadding:
OP += ' NP'
if filterType <= FILTER_FERMI:
args.append(self.filterRadius.get())
else:
args.append('%f %f' % (self.lowFreq, self.highFreq))
if filterType == FILTER_FERMI:
args.append(self.temperature.get())
# Map to spected filter number in Spider for operation FQ
filterNumber = filterType * 2 + 1
# Consider low-pass or high-pass
filterNumber += self.filterMode.get()
self._enterWorkingDir() # Do operations inside the run working dir
spi = SpiderShell(
ext=self.getExt()) # Create the Spider process to send commands
particlesStk = removeBaseExt(self.particlesStk)
# Run a loop for filtering
locStr = particlesStk + '@******[part]'
cmds = ['do lb5 [part] = 1,%d' % n, OP, locStr, locStr, filterNumber
] + args + ['lb5']
for c in cmds:
spi.runCmd(c)
spi.close()
self._leaveWorkingDir() # Go back to project dir
def importCTFStep(self, importFrom):
""" Copy ctfs matching the filename pattern. """
ci = self.getImportClass()
inputMics = self.inputMicrographs.get()
ctfSet = self._createSetOfCTF()
ctfSet.setMicrographs(inputMics)
outputMics = self._createSetOfMicrographs()
SetOfMicrographs.copyInfo(outputMics, inputMics)
createOutputMics = False
files = [f for f, _ in self.iterFiles()]
n = len(files)
if n == 0:
raise Exception("No files where found in path: '%s'\n"
"matching the pattern: '%s'" % (self.filesPath, self.filesPattern))
print "Matching files: ", len(files)
for mic in inputMics:
micName = mic.getMicName()
micBase = removeBaseExt(mic.getFileName())
for fileName, fileId in self.iterFiles():
if (fileId == mic.getObjId() or
micBase in fileName or micName in fileName):
ctf = ci.importCTF(mic, fileName)
ctfSet.append(ctf)
outputMics.append(mic)
break
else:
# If not CTF is found for a micrograph remove it from output mics
self.warning("CTF for micrograph id %d was not found. Removed from set of micrographs." % mic.getObjId())
createOutputMics = True
self._defineOutputs(outputCTF=ctfSet)
# If some of the micrographs had not ctf a subset of micrographs have been produced
if createOutputMics:
self._defineOutputs(outputMicrographs=outputMics)
self._defineCtfRelation(outputMics, ctfSet)
else:
self._defineCtfRelation(inputMics, ctfSet)
def _micBaseName(self, mic):
return removeBaseExt(mic.getFileName())
def _getMicrographDir(self, mic):
""" Return an unique dir name for results of the micrograph. """
return self._getExtraPath(removeBaseExt(mic.getFileName()))
def _getMovieRoot(self, movie):
return pwutils.removeBaseExt(movie.getFileName())
def createOutputStep(self):
# Create the SetOfImages objects on the database and the ImagesTiltPair
mdUntilted = xmipp.MetaData()
mdTilted = xmipp.MetaData()
#for objId in mdPairs:
for uMic, tMic in izip(self.uMics, self.tMics):
umicName = removeBaseExt(uMic.getFileName())
fnMicU = self._getExtraPath(umicName + ".xmd")
fnPosU = self._getExtraPath(umicName + ".pos")
# Check if there are picked particles in this micrographs
if exists(fnMicU):
mdMicU = xmipp.MetaData(fnMicU)
mdPosU = xmipp.MetaData('particles@%s' % fnPosU)
mdPosU.merge(mdMicU)
mdUntilted.unionAll(mdPosU)
tmicName = removeBaseExt(tMic.getFileName())
fnMicT = self._getExtraPath(tmicName + ".xmd")
fnPosT = self._getExtraPath(tmicName + ".pos")
mdMicT = xmipp.MetaData(fnMicT)
mdPosT = xmipp.MetaData('particles@%s' % fnPosT)
mdPosT.merge(mdMicT)
mdTilted.unionAll(mdPosT)
# Write image metadatas (check if it is really necessary)
fnTilted = self._getExtraPath("images_tilted.xmd")
fnUntilted = self._getExtraPath("images_untilted.xmd")
mdUntilted.write(fnUntilted)
mdTilted.write(fnTilted)
# Create outputs SetOfParticles both for tilted and untilted
imgSetU = self._createSetOfParticles(suffix="Untilted")
imgSetU.copyInfo(self.uMics)
imgSetT = self._createSetOfParticles(suffix="Tilted")
imgSetT.copyInfo(self.tMics)
if self.downsampleType == OTHER:
imgSetU.setSamplingRate(self.samplingFinal)
imgSetT.setSamplingRate(self.samplingFinal)
imgSetU.setCoordinates(self.inputCoordinatesTiltedPairs.get().getUntilted())
imgSetT.setCoordinates(self.inputCoordinatesTiltedPairs.get().getTilted())
#Read untilted and tilted particles on a temporary object (also disabled particles)
imgSetAuxU = self._createSetOfParticles('auxU')
imgSetAuxU.copyInfo(imgSetU)
readSetOfParticles(fnUntilted, imgSetAuxU, removeDisabled=False)
imgSetAuxU.write()
imgSetAuxT = self._createSetOfParticles('auxT')
imgSetAuxT.copyInfo(imgSetT)
readSetOfParticles(fnTilted, imgSetAuxT, removeDisabled=False)
imgSetAuxT.write()
coordsT = self.inputCoordinatesTiltedPairs.get().getTilted()
# For each untilted particle retrieve micId from SetOFCoordinates untilted
for imgU, coordU in izip(imgSetAuxU, self.inputCoordinatesTiltedPairs.get().getUntilted()):
#FIXME: REmove this check when sure that objIds are equal
id = imgU.getObjId()
if id != coordU.getObjId():
raise Exception('ObjId in untilted is not equal!!!!')
imgT = imgSetAuxT[id]
coordT = coordsT[id]
#If both particles are enabled append them
if imgU.isEnabled() and imgT.isEnabled():
imgU.setCoordinate(coordU)
imgSetU.append(imgU)
imgT.setCoordinate(coordT)
imgSetT.append(imgT)
# For each untilted particle retrieve micId from SetOFCoordinates tilted
#for img in imgSetAuxU:
# for img, coord in izip(imgSetAuxT, self.inputCoordinatesTiltedPairs.get().getTilted()):
# #FIXME: This can be slow to make a query to grab the coord, maybe use zip(imgSet, coordSet)???
# #FIXME: REmove this check when sure that objIds are equal
# if img.getObjId() != coord.getObjId():
# raise Exception('ObjId is not equal!!!!')
# #coord = self.inputCoordinatesTiltedPairs.get().getTilted()[img.getObjId()]
# img.setCoordinate(coord)
# #img.cleanObjId()
# imgSetT.append(img)
imgSetU.write()
imgSetT.write()
self._storeMethodsInfo(fnUntilted)
# Define output ParticlesTiltPair
outputset = ParticlesTiltPair(filename=self._getPath('particles_pairs.sqlite'))
outputset.setTilted(imgSetT)
outputset.setUntilted(imgSetU)
for imgU, imgT in izip(imgSetU, imgSetT):
outputset.append(TiltPair(imgU, imgT))
outputset.setCoordsPair(self.inputCoordinatesTiltedPairs.get())
self._defineOutputs(outputParticlesTiltPair=outputset)
self._defineSourceRelation(self.inputCoordinatesTiltedPairs, outputset)
def _getMicrographDir(self, mic):
""" Return an unique dir name for results of the micrograph. """
return self._getExtraPath(removeBaseExt(mic.getFileName()))
def getTSPath(self, tsFn):
tomoBaseDir = pwutils.removeBaseExt(tsFn)
pwutils.makePath(self._getExtraPath(tomoBaseDir))
return self._getExtraPath(tomoBaseDir)
def importCTFStep(self, importFrom):
""" Copy ctfs matching the filename pattern. """
ci = self.getImportClass()
inputMics = self.inputMicrographs.get()
ctfSet = self._createSetOfCTF()
ctfSet.setMicrographs(inputMics)
outputMics = self._createSetOfMicrographs()
SetOfMicrographs.copyInfo(outputMics, inputMics)
createOutputMics = False
files = [f for f, _ in self.iterFiles()]
n = len(files)
if n == 0:
raise Exception("No files where found in path: '%s'\n"
"matching the pattern: '%s'" %
(self.filesPath, self.filesPattern))
print("Matching files: %s" % n)
inputMicBases = [removeBaseExt(m.getFileName()) for m in inputMics]
if len(files) > len(inputMicBases):
self.warning("WARNING: The number of files matched by your pattern (%d) is larger than "
"the number of available micrographs (%d). It is advised to carefully "
"review the output of this run or to re-run with a more restrictive pattern."
% (n, len(inputMicBases)))
def _getMicCTF(mic):
micName = mic.getMicName()
micBase = removeBaseExt(mic.getFileName())
# see if the base name of this mic is contained in other base names
micConflicts = [mc for mc in inputMicBases if micBase in mc and micBase != mc]
if micConflicts:
self.warning('WARNING: Micrograph base name "%s" conflicts with micrograph(s) "%s". '
'Will try to find a unique match...' % (micBase, '", "'.join(micConflicts)))
# check which matching file only matches with this mic and not its conflicts
goodFnMatches = [f for f in files if micBase in f and not any(c in f for c in micConflicts)]
for goodFnMatch in goodFnMatches:
try:
micCtf = ci.importCTF(mic, goodFnMatch)
self.warning("WARNING: Assigned file %s to micrograph %s." % (goodFnMatch, micBase))
return micCtf
except Exception as ex:
self.warning("WARNING: Can't import ctf for micrograph %s from file %s"
% (micBase, goodFnMatch))
continue
else:
return None
else:
for fileName, fileId in self.iterFiles():
if (fileId == mic.getObjId() or
micBase in fileName or micName in fileName):
return ci.importCTF(mic, fileName)
return None
# Check if the CTF import class has a method to retrieve the CTF
# from a given micrograph. If not, we will try to associated based
# on matching the filename or id
getMicCTF = getattr(ci, 'getMicCTF', None) or _getMicCTF
for mic in inputMics:
ctf = getMicCTF(mic)
if ctf is not None:
ctfSet.append(ctf)
outputMics.append(mic)
else:
# If CTF is not found for a micrograph remove it from output mics
self.warning("CTF for micrograph id %d was not found. Removed "
"from set of micrographs." % mic.getObjId())
createOutputMics = True
self._defineOutputs(outputCTF=ctfSet)
# If some of the micrographs had not ctf a subset of micrographs
# have been produced
if createOutputMics:
self._defineOutputs(outputMicrographs=outputMics)
self._defineCtfRelation(outputMics, ctfSet)
else:
self._defineCtfRelation(inputMics, ctfSet)
def _getMicrographDir(mic):
""" Return an unique dir name for results of the micrograph. """
from pyworkflow.utils.path import removeBaseExt
return obj._getExtraPath(removeBaseExt(mic.getFileName()))
def _micBaseName(self, mic):
return removeBaseExt(mic.getFileName())
def _insertAllSteps(self):
"""for each micrograph insert the steps to preprocess it
"""
self.uMics = self.inputCoordinatesTiltedPairs.get().getUntilted().getMicrographs()
self.tMics = self.inputCoordinatesTiltedPairs.get().getTilted().getMicrographs()
self.inputMics = self._createSetOfParticles('auxMics')
self.inputMics.copyInfo(self.uMics)
self.inputMics.setStore(False)
for micU, micT in izip(self.uMics, self.tMics):
micU.cleanObjId()
micT.cleanObjId()
self.inputMics.append(micU)
self.inputMics.append(micT)
self.samplingInput = self.uMics.getSamplingRate()
if self.downsampleType.get() != OTHER:
# If 'same as picking' or 'original' get sampling rate from input micrographs
#TODO: Review this when downsampling before picking is possible
self.samplingFinal = self.samplingInput
else:
# If 'other' multiply the input sampling rate by the factor provided
self.samplingFinal = self.samplingInput*self.downFactor.get()
# Write pos files for each micrograph
firstStepId = self._insertFunctionStep('writePosFilesStep')
# For each micrograph insert the steps
#run in parallel
deps = []
for mic in self.inputMics:
localDeps = [firstStepId]
micrographToExtract = mic.getFileName()
micName = removeBaseExt(mic.getFileName())
micId = mic.getObjId()
# If downsample type is 'other' perform a downsample
if self.downsampleType == OTHER:
fnDownsampled = self._getTmpPath(micName+"_downsampled.xmp")
downFactor = self.downFactor.get()
args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier"
localDeps=[self._insertRunJobStep("xmipp_transform_downsample", args % locals(),prerequisites=localDeps)]
micrographToExtract = fnDownsampled
# If remove dust
if self.doRemoveDust:
fnNoDust = self._getTmpPath(micName+"_noDust.xmp")
thresholdDust = self.thresholdDust.get() #TODO: remove this extra variable
args=" -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f"
localDeps=[self._insertRunJobStep("xmipp_transform_filter", args % locals(),prerequisites=localDeps)]
micrographToExtract = fnNoDust
#self._insertFunctionStep('getCTF', micId, micName, micrographToExtract)
micName = removeBaseExt(mic.getFileName())
# Actually extract
deps.append(self._insertFunctionStep('extractParticlesStep', micId, micName,
None, micrographToExtract, prerequisites=localDeps))
# TODO: Delete temporary files
# Insert step to create output objects
self._insertFunctionStep('createOutputStep', prerequisites=deps)
def createOutputStep(self):
# Create the SetOfImages objects on the database and the ImagesTiltPair
mdUntilted = xmipp.MetaData()
mdTilted = xmipp.MetaData()
#for objId in mdPairs:
for uMic, tMic in izip(self.uMics, self.tMics):
umicName = removeBaseExt(uMic.getFileName())
fnMicU = self._getExtraPath(umicName + ".xmd")
fnPosU = self._getExtraPath(umicName + ".pos")
# Check if there are picked particles in this micrographs
if exists(fnMicU):
mdMicU = xmipp.MetaData(fnMicU)
mdPosU = xmipp.MetaData('particles@%s' % fnPosU)
mdPosU.merge(mdMicU)
mdUntilted.unionAll(mdPosU)
tmicName = removeBaseExt(tMic.getFileName())
fnMicT = self._getExtraPath(tmicName + ".xmd")
fnPosT = self._getExtraPath(tmicName + ".pos")
mdMicT = xmipp.MetaData(fnMicT)
mdPosT = xmipp.MetaData('particles@%s' % fnPosT)
mdPosT.merge(mdMicT)
mdTilted.unionAll(mdPosT)
# Write image metadatas (check if it is really necessary)
fnTilted = self._getExtraPath("images_tilted.xmd")
fnUntilted = self._getExtraPath("images_untilted.xmd")
mdUntilted.write(fnUntilted)
mdTilted.write(fnTilted)
# Create outputs SetOfParticles both for tilted and untilted
imgSetU = self._createSetOfParticles(suffix="Untilted")
imgSetU.copyInfo(self.uMics)
imgSetT = self._createSetOfParticles(suffix="Tilted")
imgSetT.copyInfo(self.tMics)
if self.downsampleType == OTHER:
imgSetU.setSamplingRate(self.samplingFinal)
imgSetT.setSamplingRate(self.samplingFinal)
imgSetU.setCoordinates(self.inputCoordinatesTiltedPairs.get().getUntilted())
imgSetT.setCoordinates(self.inputCoordinatesTiltedPairs.get().getTilted())
#Read untilted and tilted particles on a temporary object (also disabled particles)
imgSetAuxU = self._createSetOfParticles('auxU')
imgSetAuxU.copyInfo(imgSetU)
readSetOfParticles(fnUntilted, imgSetAuxU, removeDisabled=False)
imgSetAuxU.write()
imgSetAuxT = self._createSetOfParticles('auxT')
imgSetAuxT.copyInfo(imgSetT)
readSetOfParticles(fnTilted, imgSetAuxT, removeDisabled=False)
imgSetAuxT.write()
coordsT = self.inputCoordinatesTiltedPairs.get().getTilted()
# For each untilted particle retrieve micId from SetOFCoordinates untilted
for imgU, coordU in izip(imgSetAuxU, self.inputCoordinatesTiltedPairs.get().getUntilted()):
#FIXME: REmove this check when sure that objIds are equal
id = imgU.getObjId()
if id != coordU.getObjId():
raise Exception('ObjId in untilted is not equal!!!!')
imgT = imgSetAuxT[id]
coordT = coordsT[id]
#If both particles are enabled append them
if imgU.isEnabled() and imgT.isEnabled():
imgU.setCoordinate(coordU)
imgSetU.append(imgU)
imgT.setCoordinate(coordT)
imgSetT.append(imgT)
# For each untilted particle retrieve micId from SetOFCoordinates tilted
#for img in imgSetAuxU:
# for img, coord in izip(imgSetAuxT, self.inputCoordinatesTiltedPairs.get().getTilted()):
# #FIXME: This can be slow to make a query to grab the coord, maybe use zip(imgSet, coordSet)???
# #FIXME: REmove this check when sure that objIds are equal
# if img.getObjId() != coord.getObjId():
# raise Exception('ObjId is not equal!!!!')
# #coord = self.inputCoordinatesTiltedPairs.get().getTilted()[img.getObjId()]
# img.setCoordinate(coord)
# #img.cleanObjId()
# imgSetT.append(img)
imgSetU.write()
imgSetT.write()
self._storeMethodsInfo(fnUntilted)
# Define output ParticlesTiltPair
outputset = ParticlesTiltPair(filename=self._getPath('particles_pairs.sqlite'))
outputset.setTilted(imgSetT)
outputset.setUntilted(imgSetU)
for imgU, imgT in izip(imgSetU, imgSetT):
outputset.append(TiltPair(imgU, imgT))
outputset.setCoordsPair(self.inputCoordinatesTiltedPairs.get())
self._defineOutputs(outputParticlesTiltPair=outputset)
self._defineSourceRelation(self.inputCoordinatesTiltedPairs, outputset)
def _insertAllSteps(self):
"""for each micrograph insert the steps to preprocess it
"""
self._setupBasicProperties()
# Write pos files for each micrograph
firstStepId = self._insertFunctionStep('writePosFilesStep')
# For each micrograph insert the steps
#run in parallel
deps = []
for mic in self.inputMics:
localDeps = [firstStepId]
micrographToExtract = mic.getFileName()
baseMicName = removeBaseExt(mic.getFileName())
if self.ctfRelations.hasValue():
micKey = self.micKey(mic)
mic.setCTF(self.ctfDict[micKey])
# If downsample type is 'other' perform a downsample
downFactor = self.downFactor.get()
if self.downsampleType == OTHER and abs(downFactor - 1.) > 0.0001:
fnDownsampled = self._getTmpPath(baseMicName +
"_downsampled.xmp")
args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier"
localDeps = [
self._insertRunJobStep("xmipp_transform_downsample",
args % locals(),
prerequisites=localDeps)
]
micrographToExtract = fnDownsampled
# If remove dust
if self.doRemoveDust:
fnNoDust = self._getTmpPath(baseMicName + "_noDust.xmp")
thresholdDust = self.thresholdDust.get(
) #TODO: remove this extra variable
args = " -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f"
localDeps = [
self._insertRunJobStep("xmipp_transform_filter",
args % locals(),
prerequisites=localDeps)
]
micrographToExtract = fnNoDust
#self._insertFunctionStep('getCTF', micId, baseMicName, micrographToExtract)
#FIXME: Check only if mic has CTF when implemented ok
#if self.doFlip or mic.hasCTF():
fnCTF = None
if self.ctfRelations.hasValue():
# If the micrograph doesn't come from Xmipp, we need to write
# a Xmipp ctfparam file to perform the phase flip on the micrograph
fnCTF = micrographToCTFParam(
mic, self._getTmpPath("%s.ctfParam" % baseMicName))
# Insert step to flip micrograph
if self.doFlip:
localDeps = [
self._insertFunctionStep('flipMicrographStep',
baseMicName,
fnCTF,
micrographToExtract,
prerequisites=localDeps)
]
micrographToExtract = self._getTmpPath(baseMicName +
"_flipped.xmp")
else:
fnCTF = None
# Actually extract
deps.append(
self._insertFunctionStep('extractParticlesStep',
mic.getObjId(),
baseMicName,
fnCTF,
micrographToExtract,
prerequisites=localDeps))
# Insert step to create output objects
metaDeps = self._insertFunctionStep('createMetadataImageStep',
prerequisites=deps)
if self.doSort:
screenDep = self._insertFunctionStep('screenParticlesStep',
prerequisites=[metaDeps])
finalDeps = [screenDep]
else:
finalDeps = [metaDeps]
self._insertFunctionStep('createOutputStep', prerequisites=finalDeps)
def runScript(inputScript, ext, log=None, cwd=None):
scriptName = removeBaseExt(inputScript)
args = " %s @%s" % (ext, scriptName)
runJob(log, SPIDER, args, env=dict(environment), cwd=cwd)
def runScript(inputScript, ext, program, nummpis, log=None, cwd=None):
scriptName = removeBaseExt(inputScript)
args = " %s @%s" % (ext, scriptName)
runJob(log, program, args, numberOfMpi=nummpis,
env=Plugin.getEnviron(), cwd=cwd)
def _getMicrographDir(mic):
""" Return an unique dir name for results of the micrograph. """
from pyworkflow.utils.path import removeBaseExt
return obj._getExtraPath(removeBaseExt(mic.getFileName()))
def runScript(inputScript, ext, log=None, cwd=None):
scriptName = removeBaseExt(inputScript)
args = " %s @%s" % (ext, scriptName)
runJob(log, SPIDER, args, env=dict(environment), cwd=cwd)
def _getMicName(self, ctf):
""" Get the micName to be used as Key from
a given ctf object.
"""
return removeBaseExt(ctf.getMicrograph().getFileName())
def _getMicName(self, ctf):
""" Get the micName to be used as Key from
a given ctf object.
"""
return removeBaseExt(ctf.getMicrograph().getFileName())
def _getPlotName(self, movieName, plotType):
if plotType == PLOT_CART:
return removeBaseExt(movieName) + '_plot_cart.png'
else:
return removeBaseExt(movieName) + '_plot_polar.png'
def _insertAllSteps(self):
"""for each micrograph insert the steps to preprocess it
"""
self.uMics = self.inputCoordinatesTiltedPairs.get().getUntilted().getMicrographs()
self.tMics = self.inputCoordinatesTiltedPairs.get().getTilted().getMicrographs()
self.inputMics = self._createSetOfParticles('auxMics')
self.inputMics.copyInfo(self.uMics)
self.inputMics.setStore(False)
for micU, micT in izip(self.uMics, self.tMics):
micU.cleanObjId()
micT.cleanObjId()
self.inputMics.append(micU)
self.inputMics.append(micT)
self.samplingInput = self.uMics.getSamplingRate()
if self.downsampleType.get() != OTHER:
# If 'same as picking' or 'original' get sampling rate from input micrographs
#TODO: Review this when downsampling before picking is possible
self.samplingFinal = self.samplingInput
else:
# If 'other' multiply the input sampling rate by the factor provided
self.samplingFinal = self.samplingInput*self.downFactor.get()
# Write pos files for each micrograph
firstStepId = self._insertFunctionStep('writePosFilesStep')
# For each micrograph insert the steps
#run in parallel
deps = []
for mic in self.inputMics:
localDeps = [firstStepId]
micrographToExtract = mic.getFileName()
micName = removeBaseExt(mic.getFileName())
micId = mic.getObjId()
# If downsample type is 'other' perform a downsample
if self.downsampleType == OTHER:
fnDownsampled = self._getTmpPath(micName+"_downsampled.xmp")
downFactor = self.downFactor.get()
args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier"
localDeps=[self._insertRunJobStep("xmipp_transform_downsample", args % locals(),prerequisites=localDeps)]
micrographToExtract = fnDownsampled
# If remove dust
if self.doRemoveDust:
fnNoDust = self._getTmpPath(micName+"_noDust.xmp")
thresholdDust = self.thresholdDust.get() #TODO: remove this extra variable
args=" -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f"
localDeps=[self._insertRunJobStep("xmipp_transform_filter", args % locals(),prerequisites=localDeps)]
micrographToExtract = fnNoDust
#self._insertFunctionStep('getCTF', micId, micName, micrographToExtract)
micName = removeBaseExt(mic.getFileName())
# Actually extract
deps.append(self._insertFunctionStep('extractParticlesStep', micId, micName,
None, micrographToExtract, prerequisites=localDeps))
# TODO: Delete temporary files
# Insert step to create output objects
self._insertFunctionStep('createOutputStep', prerequisites=deps)
