def importCTFStep(self, importFrom): """ Copy ctfs matching the filename pattern. """ ci = self.getImportClass() inputMics = self.inputMicrographs.get() ctfSet = self._createSetOfCTF() ctfSet.setMicrographs(inputMics) outputMics = self._createSetOfMicrographs() SetOfMicrographs.copyInfo(outputMics, inputMics) createOutputMics = False for mic in inputMics: for i, (fileName, fileId) in enumerate(self.iterFiles()): #TODO: Define how to match mics and ctf files (temporary solution is to expect micname inside ctf file name) if removeBaseExt(mic.getFileName()) in removeBaseExt(fileName): ctf = ci.importCTF(mic, fileName) ctfSet.append(ctf) outputMics.append(mic) break else: # If not CTF is found for a micrograph remove it from output mics self.warning("CTF for micrograph id %d was not found. Removed from set of micrographs." % mic.getObjId()) createOutputMics = True self._defineOutputs(outputCTF=ctfSet) # If some of the micrographs had not ctf a subset of micrographs have been produced if createOutputMics: self._defineOutputs(outputMicrographs=outputMics) self._defineCtfRelation(outputMics, ctfSet) else: self._defineCtfRelation(inputMics, ctfSet)
def _getNameExt(self, movieName, postFix, ext): if movieName.endswith("bz2"): # removeBaseExt function only eliminate the last extension, # but if files are compressed, we need to eliminate two extensions: # bz2 and its own image extension (e.g: mrcs, em, etc) return removeBaseExt(removeBaseExt(movieName)) + postFix + '.' + ext else: return removeBaseExt(movieName) + postFix + '.' + ext
def getMatchingMic(self, coordFile, fileId): """ Given a coordinates file check if there is a micrograph that this files matches. """ micSet = self.inputMicrographs.get() if fileId is None: coordBase = removeBaseExt(coordFile) for mic in micSet: micBase = removeBaseExt(mic.getFileName()) if coordBase in micBase or micBase in coordBase: #temporal use of in return mic return None else: return micSet[fileId]
def __createTemporaryCtfs(self, obj, setOfMics): """ Create a temporary .sqlite file to visualize CTF while the protocol has not finished yet. """ cleanPath(obj._getPath("ctfs_temporary.sqlite")) ctfSet = self.protocol._createSetOfCTF("_temporary") for mic in setOfMics: micFn = mic.getFileName() micDir = obj._getExtraPath(removeBaseExt(mic.getFileName())) samplingRate = mic.getSamplingRate() * self.protocol.ctfDownFactor.get() mic.setSamplingRate(samplingRate) out = self.protocol._getCtfOutPath(micDir) psdFile = self.protocol._getPsdPath(micDir) if exists(out) and exists(psdFile): ctfModel = em.CTFModel() readCtfModel(ctfModel, out, ctf4=self.protocol.useCtffind4.get()) ctfModel.setPsdFile(psdFile) ctfModel.setMicrograph(mic) ctfSet.append(ctfModel) if not ctfSet.isEmpty(): ctfSet.write() ctfSet.close() return ctfSet
def _getMicCTF(mic): micName = mic.getMicName() micBase = removeBaseExt(mic.getFileName()) # see if the base name of this mic is contained in other base names micConflicts = [mc for mc in inputMicBases if micBase in mc and micBase != mc] if micConflicts: self.warning('WARNING: Micrograph base name "%s" conflicts with micrograph(s) "%s". ' 'Will try to find a unique match...' % (micBase, '", "'.join(micConflicts))) # check which matching file only matches with this mic and not its conflicts goodFnMatches = [f for f in files if micBase in f and not any(c in f for c in micConflicts)] for goodFnMatch in goodFnMatches: try: micCtf = ci.importCTF(mic, goodFnMatch) self.warning("WARNING: Assigned file %s to micrograph %s." % (goodFnMatch, micBase)) return micCtf except Exception as ex: self.warning("WARNING: Can't import ctf for micrograph %s from file %s" % (micBase, goodFnMatch)) continue else: return None else: for fileName, fileId in self.iterFiles(): if (fileId == mic.getObjId() or micBase in fileName or micName in fileName): return ci.importCTF(mic, fileName) return None
def _insertAllSteps(self): """for each micrograph insert the steps to preprocess it """ self._defineBasicParams() # Write pos files for each micrograph firstStepId = self._insertFunctionStep('writePosFilesStep') # For each micrograph insert the steps #run in parallel deps = [] for mic in self.inputMics: localDeps = [firstStepId] micrographToExtract = mic.getFileName() baseMicName = removeBaseExt(mic.getFileName()) if self.ctfRelations.hasValue(): micName = mic.getMicName() mic.setCTF(self.ctfDict[micName]) # If downsample type is 'other' perform a downsample downFactor = self.downFactor.get() if self.downsampleType == OTHER and abs(downFactor - 1.) > 0.0001: fnDownsampled = self._getTmpPath(baseMicName+"_downsampled.xmp") args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier" localDeps = [self._insertRunJobStep("xmipp_transform_downsample", args % locals(),prerequisites=localDeps)] micrographToExtract = fnDownsampled # If remove dust if self.doRemoveDust: fnNoDust = self._getTmpPath(baseMicName+"_noDust.xmp") thresholdDust = self.thresholdDust.get() #TODO: remove this extra variable args=" -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f" localDeps = [self._insertRunJobStep("xmipp_transform_filter", args % locals(),prerequisites=localDeps)] micrographToExtract = fnNoDust #self._insertFunctionStep('getCTF', micId, baseMicName, micrographToExtract) #FIXME: Check only if mic has CTF when implemented ok #if self.doFlip or mic.hasCTF(): fnCTF=None if self.ctfRelations.hasValue(): # If the micrograph doesn't come from Xmipp, we need to write # a Xmipp ctfparam file to perform the phase flip on the micrograph fnCTF = micrographToCTFParam(mic, self._getTmpPath("%s.ctfParam" % baseMicName)) # Insert step to flip micrograph if self.doFlip: localDeps = [self._insertFunctionStep('flipMicrographStep', baseMicName, fnCTF, micrographToExtract, prerequisites=localDeps)] micrographToExtract = self._getTmpPath(baseMicName +"_flipped.xmp") else: fnCTF = None # Actually extract deps.append(self._insertFunctionStep('extractParticlesStep', mic.getObjId(), baseMicName, fnCTF, micrographToExtract, prerequisites=localDeps)) # Insert step to create output objects self._insertFunctionStep('createOutputStep', prerequisites=deps)
def filterStep(self, filterType): """ Apply the selected filter to particles. Create the set of particles. """ particles = self.inputParticles.get() n = particles.getSize() OP = self._op args = [] if not self.usePadding: OP += ' NP' if filterType <= FILTER_SPACE_REAL: args.append(self.filterRadius.get()) else: args.append('%f %f' % (self.lowFreq, self.highFreq)) if filterType == FILTER_FERMI: args.append(self.temperature.get()) # Map to spected filter number in Spider for operation FQ filterNumber = filterType * 2 + 1 # Consider low-pass or high-pass filterNumber += self.filterMode.get() imgSet = self._createSetOfParticles() imgSet.copyInfo(self.inputParticles.get()) self._enterWorkingDir() # Do operations inside the run working dir spi = SpiderShell(ext=self.getExt()) # Create the Spider process to send commands particlesStk = removeBaseExt(self.particlesStk) # Run a loop for filtering locStr = particlesStk + '@******[part]' cmds = ['do lb5 [part] = 1,%d' % n, OP, locStr, locStr, filterNumber] + args + ['lb5'] for c in cmds: spi.runCmd(c) spi.close() self._leaveWorkingDir() # Go back to project dir for i, img in enumerate(particles): img.setLocation(i+1, self.particlesStk) imgSet.append(img) self._defineOutputs(outputParticles=imgSet) self._defineTransformRelation(particles, imgSet)
def importCoordinatesStep(self, importFrom, *args): inputMics = self.inputMicrographs.get() coordsSet = self._createSetOfCoordinates(inputMics) coordsSet.setBoxSize(self.boxSize.get()) scaleFactor = self.scale.get() invertX = self.invertX.get() invertY = self.invertY.get() ci = self.getImportClass() for i, (fileName, fileId) in enumerate(self.iterFiles()): for mic in inputMics: if removeBaseExt(mic.getFileName()) in removeBaseExt(fileName):#temporal use of in def addCoordinate(coord): coord.setMicrograph(mic) x = coord.getX() y = coord.getY() if scaleFactor != 1.: x = coord.getX() * scaleFactor y = coord.getY() * scaleFactor if invertX: width = mic.getDim()[0] x = width - x if invertY: height = mic.getDim()[1] y = height - y coord.setX(x) coord.setY(y) coordsSet.append(coord) ci.importCoordinates(fileName, addCoordinate=addCoordinate) break self._defineOutputs(outputCoordinates=coordsSet) self._defineSourceRelation(self.inputMicrographs, coordsSet)
def writePosFilesStep(self): """ Write the pos file for each micrograph on metadata format. """ #self.posFiles = writeSetOfCoordinates(self._getExtraPath(), self.inputCoords) writeSetOfCoordinates(self._getExtraPath(), self.inputCoords) # We need to find the mapping (either by micName or micId) # between the micrographs in the SetOfCoordinates and # the micrographs (if we are in a different case than 'same as picking' if self.downsampleType != SAME_AS_PICKING: micDict = {} coordMics = self.inputCoords.getMicrographs() for mic in coordMics: micBase = removeBaseExt(mic.getFileName()) micPos = self._getExtraPath(micBase + ".pos") micDict[mic.getMicName()] = micPos micDict[mic.getObjId()] = micPos if any(mic.getMicName() in micDict for mic in self.inputMics): micKey = lambda mic: mic.getMicName() elif any(mic.getObjId() in micDict for mic in self.inputMics): self.warning('Could not match input micrographs and coordinates ' 'micrographs by micName, using micId.') micKey = lambda mic: mic.getObjId() else: raise Exception('Could not match input micrographs and coordinates ' 'neither by micName or micId.') for mic in self.inputMics: # micrograph from input (other) mk = micKey(mic) if mk in micDict: micPosCoord = micDict[mk] if exists(micPosCoord): micBase = removeBaseExt(mic.getFileName()) micPos = self._getExtraPath(micBase + ".pos") if micPos != micPosCoord: self.info('Moving %s -> %s' % (micPosCoord, micPos)) moveFile(micPosCoord, micPos)
def _insertAllSteps(self): self._params = {} # diameter must be passed in Armstrongs and therefore should be converted self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate() # self._params['num-slices'] = self.numberSizes.get() # self._params['size-range'] = self.sizeRange.get() self._params['apix'] = self.inputMicrographs.get().getSamplingRate() self._params['thresh'] = self.threshold.get() # self._params['max-thresh'] = self.maxThreshold.get() # self._params['max-area'] = self.maxArea.get() # self._params['max-peaks'] = self.maxPeaks.get() args = "" for par, val in self._params.iteritems(): args += " --%s=%s" % (par, str(val)) if self.invert: args += " --invert" args += " " + self.extraParams.get('') deps = [] # Store all steps ids, final step createOutput depends on all of them ih = ImageHandler() for mic in self.inputMicrographs.get(): # Create micrograph folder micName = mic.getFileName() micDir = self._getTmpPath(removeBaseExt(micName)) makePath(micDir) # If needed convert micrograph to mrc format, otherwise link it if getExt(micName) != ".mrc": fnMicBase = replaceBaseExt(micName, 'mrc') inputMic = join(micDir, fnMicBase) ih.convert(mic.getLocation(), inputMic) else: inputMic = join(micDir, basename(micName)) createLink(micName, inputMic) # Insert step to execute program stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args) deps.append(stepId) self._insertFinalSteps(deps)
def importCTFStep(self, importFrom): """ Copy ctfs matching the filename pattern. """ ci = self.getImportClass() inputMics = self.inputMicrographs.get() ctfSet = self._createSetOfCTF() ctfSet.setMicrographs(inputMics) outputMics = self._createSetOfMicrographs() SetOfMicrographs.copyInfo(outputMics, inputMics) createOutputMics = False files = [f for f, _ in self.iterFiles()] n = len(files) if n == 0: raise Exception("No files where found in path: '%s'\n" "matching the pattern: '%s'" % (self.filesPath, self.filesPattern)) print "Matching files: ", len(files) for mic in inputMics: micName = mic.getMicName() micBase = removeBaseExt(mic.getFileName()) for fileName, fileId in self.iterFiles(): if (fileId == mic.getObjId() or micBase in fileName or micName in fileName): ctf = ci.importCTF(mic, fileName) ctfSet.append(ctf) outputMics.append(mic) break else: # If not CTF is found for a micrograph remove it from output mics self.warning( "CTF for micrograph id %d was not found. Removed from set of micrographs." % mic.getObjId()) createOutputMics = True self._defineOutputs(outputCTF=ctfSet) # If some of the micrographs had not ctf a subset of micrographs have been produced if createOutputMics: self._defineOutputs(outputMicrographs=outputMics) self._defineCtfRelation(outputMics, ctfSet) else: self._defineCtfRelation(inputMics, ctfSet)
def launchVesicleViewer(self, vesicle): print("\n==> Running Vesicle Viewer:") vesicleBaseName = removeBaseExt(vesicle.getFileName()) vtiName = join(self.vtiPath, vesicleBaseName + '.vti') args = {'vti_file': vtiName} if self.source == FROM_GRAPHS: args['graph_file'] = glob.glob( self.prot._getExtraPath(vesicleBaseName + '*_edges_2.vtp'))[0] elif self.source == FROM_FILS: args['net_file'] = glob.glob( self.prot._getExtraPath(vesicleBaseName + '*_net.vtp'))[0] elif self.source == FROM_PICKING: args['peaks_file'] = glob.glob( self.prot._getExtraPath(vesicleBaseName + '*_peak.vtp'))[0] args['surf_file'] = glob.glob( self.prot._getExtraPath(vesicleBaseName + '*_surf.vtp'))[0] guiThread(VtkPlot, 'initializePlot', **args)
def launchMembAnnotatorForTomogram(self, tomoMask): # Tomo files come from one dir, while tomoMask files comes from another, because they were generated in # different protocols. MembraneAnnotator expects both to be in the same location, so a symbolic link is # is generated in the extra dir of the segmentation protocol pointing to the selected tomogram print("\n==> Running Membrane Annotator:") tomoNameSrc = abspath(tomoMask.getVolName()) tomoName = abspath( join(getParentFolder(tomoMask.getFileName()), basename(tomoNameSrc))) if not exists(tomoName): symlink(tomoNameSrc, tomoName) arguments = "inTomoFile '%s' " % tomoName arguments += "outFilename '%s'" % abspath( join(self.path, removeBaseExt(tomoName))) Plugin.runMembraneAnnotator(self.prot, arguments, env=Plugin.getMembSegEnviron(), cwd=self.path)
def readSetOfCoordinates(workDir, micSet, coordSet): """ Read from Eman .json files. It is expected a file named: base.json under the workDir. Params: workDir: where the Eman boxer output files are located. micSet: the SetOfMicrographs to associate the .json, which name should be the same of the micrographs. coordSet: the SetOfCoordinates that will be populated. """ # Read the boxSize from the e2boxercache/base.json jsonFnbase = join(workDir, 'e2boxercache', 'base.json') jsonBoxDict = loadJson(jsonFnbase) size = int(jsonBoxDict["box_size"]) jsonFninfo = join(workDir, 'info/') for mic in micSet: micPosFn = ''.join(glob.glob(jsonFninfo + '*' + removeBaseExt(mic.getFileName()) + '_info.json')) readCoordinates(mic, micPosFn, coordSet) coordSet.setBoxSize(size)
def filterStep(self, filterType): """ Apply the selected filter to particles. Create the set of particles. """ particles = self.inputParticles.get() n = particles.getSize() OP = self._op args = [] if not self.usePadding: OP += ' NP' if filterType <= FILTER_FERMI: args.append(self.filterRadius.get()) else: args.append('%f %f' % (self.lowFreq, self.highFreq)) if filterType == FILTER_FERMI: args.append(self.temperature.get()) # Map to spected filter number in Spider for operation FQ filterNumber = filterType * 2 + 1 # Consider low-pass or high-pass filterNumber += self.filterMode.get() self._enterWorkingDir() # Do operations inside the run working dir spi = SpiderShell( ext=self.getExt()) # Create the Spider process to send commands particlesStk = removeBaseExt(self.particlesStk) # Run a loop for filtering locStr = particlesStk + '@******[part]' cmds = ['do lb5 [part] = 1,%d' % n, OP, locStr, locStr, filterNumber ] + args + ['lb5'] for c in cmds: spi.runCmd(c) spi.close() self._leaveWorkingDir() # Go back to project dir
def importCTFStep(self, importFrom): """ Copy ctfs matching the filename pattern. """ ci = self.getImportClass() inputMics = self.inputMicrographs.get() ctfSet = self._createSetOfCTF() ctfSet.setMicrographs(inputMics) outputMics = self._createSetOfMicrographs() SetOfMicrographs.copyInfo(outputMics, inputMics) createOutputMics = False files = [f for f, _ in self.iterFiles()] n = len(files) if n == 0: raise Exception("No files where found in path: '%s'\n" "matching the pattern: '%s'" % (self.filesPath, self.filesPattern)) print "Matching files: ", len(files) for mic in inputMics: micName = mic.getMicName() micBase = removeBaseExt(mic.getFileName()) for fileName, fileId in self.iterFiles(): if (fileId == mic.getObjId() or micBase in fileName or micName in fileName): ctf = ci.importCTF(mic, fileName) ctfSet.append(ctf) outputMics.append(mic) break else: # If not CTF is found for a micrograph remove it from output mics self.warning("CTF for micrograph id %d was not found. Removed from set of micrographs." % mic.getObjId()) createOutputMics = True self._defineOutputs(outputCTF=ctfSet) # If some of the micrographs had not ctf a subset of micrographs have been produced if createOutputMics: self._defineOutputs(outputMicrographs=outputMics) self._defineCtfRelation(outputMics, ctfSet) else: self._defineCtfRelation(inputMics, ctfSet)
def _micBaseName(self, mic): return removeBaseExt(mic.getFileName())
def _getMicrographDir(self, mic): """ Return an unique dir name for results of the micrograph. """ return self._getExtraPath(removeBaseExt(mic.getFileName()))
def _getMovieRoot(self, movie): return pwutils.removeBaseExt(movie.getFileName())
def createOutputStep(self): # Create the SetOfImages objects on the database and the ImagesTiltPair mdUntilted = xmipp.MetaData() mdTilted = xmipp.MetaData() #for objId in mdPairs: for uMic, tMic in izip(self.uMics, self.tMics): umicName = removeBaseExt(uMic.getFileName()) fnMicU = self._getExtraPath(umicName + ".xmd") fnPosU = self._getExtraPath(umicName + ".pos") # Check if there are picked particles in this micrographs if exists(fnMicU): mdMicU = xmipp.MetaData(fnMicU) mdPosU = xmipp.MetaData('particles@%s' % fnPosU) mdPosU.merge(mdMicU) mdUntilted.unionAll(mdPosU) tmicName = removeBaseExt(tMic.getFileName()) fnMicT = self._getExtraPath(tmicName + ".xmd") fnPosT = self._getExtraPath(tmicName + ".pos") mdMicT = xmipp.MetaData(fnMicT) mdPosT = xmipp.MetaData('particles@%s' % fnPosT) mdPosT.merge(mdMicT) mdTilted.unionAll(mdPosT) # Write image metadatas (check if it is really necessary) fnTilted = self._getExtraPath("images_tilted.xmd") fnUntilted = self._getExtraPath("images_untilted.xmd") mdUntilted.write(fnUntilted) mdTilted.write(fnTilted) # Create outputs SetOfParticles both for tilted and untilted imgSetU = self._createSetOfParticles(suffix="Untilted") imgSetU.copyInfo(self.uMics) imgSetT = self._createSetOfParticles(suffix="Tilted") imgSetT.copyInfo(self.tMics) if self.downsampleType == OTHER: imgSetU.setSamplingRate(self.samplingFinal) imgSetT.setSamplingRate(self.samplingFinal) imgSetU.setCoordinates(self.inputCoordinatesTiltedPairs.get().getUntilted()) imgSetT.setCoordinates(self.inputCoordinatesTiltedPairs.get().getTilted()) #Read untilted and tilted particles on a temporary object (also disabled particles) imgSetAuxU = self._createSetOfParticles('auxU') imgSetAuxU.copyInfo(imgSetU) readSetOfParticles(fnUntilted, imgSetAuxU, removeDisabled=False) imgSetAuxU.write() imgSetAuxT = self._createSetOfParticles('auxT') imgSetAuxT.copyInfo(imgSetT) readSetOfParticles(fnTilted, imgSetAuxT, removeDisabled=False) imgSetAuxT.write() coordsT = self.inputCoordinatesTiltedPairs.get().getTilted() # For each untilted particle retrieve micId from SetOFCoordinates untilted for imgU, coordU in izip(imgSetAuxU, self.inputCoordinatesTiltedPairs.get().getUntilted()): #FIXME: REmove this check when sure that objIds are equal id = imgU.getObjId() if id != coordU.getObjId(): raise Exception('ObjId in untilted is not equal!!!!') imgT = imgSetAuxT[id] coordT = coordsT[id] #If both particles are enabled append them if imgU.isEnabled() and imgT.isEnabled(): imgU.setCoordinate(coordU) imgSetU.append(imgU) imgT.setCoordinate(coordT) imgSetT.append(imgT) # For each untilted particle retrieve micId from SetOFCoordinates tilted #for img in imgSetAuxU: # for img, coord in izip(imgSetAuxT, self.inputCoordinatesTiltedPairs.get().getTilted()): # #FIXME: This can be slow to make a query to grab the coord, maybe use zip(imgSet, coordSet)??? # #FIXME: REmove this check when sure that objIds are equal # if img.getObjId() != coord.getObjId(): # raise Exception('ObjId is not equal!!!!') # #coord = self.inputCoordinatesTiltedPairs.get().getTilted()[img.getObjId()] # img.setCoordinate(coord) # #img.cleanObjId() # imgSetT.append(img) imgSetU.write() imgSetT.write() self._storeMethodsInfo(fnUntilted) # Define output ParticlesTiltPair outputset = ParticlesTiltPair(filename=self._getPath('particles_pairs.sqlite')) outputset.setTilted(imgSetT) outputset.setUntilted(imgSetU) for imgU, imgT in izip(imgSetU, imgSetT): outputset.append(TiltPair(imgU, imgT)) outputset.setCoordsPair(self.inputCoordinatesTiltedPairs.get()) self._defineOutputs(outputParticlesTiltPair=outputset) self._defineSourceRelation(self.inputCoordinatesTiltedPairs, outputset)
def getTSPath(self, tsFn): tomoBaseDir = pwutils.removeBaseExt(tsFn) pwutils.makePath(self._getExtraPath(tomoBaseDir)) return self._getExtraPath(tomoBaseDir)
def importCTFStep(self, importFrom): """ Copy ctfs matching the filename pattern. """ ci = self.getImportClass() inputMics = self.inputMicrographs.get() ctfSet = self._createSetOfCTF() ctfSet.setMicrographs(inputMics) outputMics = self._createSetOfMicrographs() SetOfMicrographs.copyInfo(outputMics, inputMics) createOutputMics = False files = [f for f, _ in self.iterFiles()] n = len(files) if n == 0: raise Exception("No files where found in path: '%s'\n" "matching the pattern: '%s'" % (self.filesPath, self.filesPattern)) print("Matching files: %s" % n) inputMicBases = [removeBaseExt(m.getFileName()) for m in inputMics] if len(files) > len(inputMicBases): self.warning("WARNING: The number of files matched by your pattern (%d) is larger than " "the number of available micrographs (%d). It is advised to carefully " "review the output of this run or to re-run with a more restrictive pattern." % (n, len(inputMicBases))) def _getMicCTF(mic): micName = mic.getMicName() micBase = removeBaseExt(mic.getFileName()) # see if the base name of this mic is contained in other base names micConflicts = [mc for mc in inputMicBases if micBase in mc and micBase != mc] if micConflicts: self.warning('WARNING: Micrograph base name "%s" conflicts with micrograph(s) "%s". ' 'Will try to find a unique match...' % (micBase, '", "'.join(micConflicts))) # check which matching file only matches with this mic and not its conflicts goodFnMatches = [f for f in files if micBase in f and not any(c in f for c in micConflicts)] for goodFnMatch in goodFnMatches: try: micCtf = ci.importCTF(mic, goodFnMatch) self.warning("WARNING: Assigned file %s to micrograph %s." % (goodFnMatch, micBase)) return micCtf except Exception as ex: self.warning("WARNING: Can't import ctf for micrograph %s from file %s" % (micBase, goodFnMatch)) continue else: return None else: for fileName, fileId in self.iterFiles(): if (fileId == mic.getObjId() or micBase in fileName or micName in fileName): return ci.importCTF(mic, fileName) return None # Check if the CTF import class has a method to retrieve the CTF # from a given micrograph. If not, we will try to associated based # on matching the filename or id getMicCTF = getattr(ci, 'getMicCTF', None) or _getMicCTF for mic in inputMics: ctf = getMicCTF(mic) if ctf is not None: ctfSet.append(ctf) outputMics.append(mic) else: # If CTF is not found for a micrograph remove it from output mics self.warning("CTF for micrograph id %d was not found. Removed " "from set of micrographs." % mic.getObjId()) createOutputMics = True self._defineOutputs(outputCTF=ctfSet) # If some of the micrographs had not ctf a subset of micrographs # have been produced if createOutputMics: self._defineOutputs(outputMicrographs=outputMics) self._defineCtfRelation(outputMics, ctfSet) else: self._defineCtfRelation(inputMics, ctfSet)
def _getMicrographDir(mic): """ Return an unique dir name for results of the micrograph. """ from pyworkflow.utils.path import removeBaseExt return obj._getExtraPath(removeBaseExt(mic.getFileName()))
def _insertAllSteps(self): """for each micrograph insert the steps to preprocess it """ self.uMics = self.inputCoordinatesTiltedPairs.get().getUntilted().getMicrographs() self.tMics = self.inputCoordinatesTiltedPairs.get().getTilted().getMicrographs() self.inputMics = self._createSetOfParticles('auxMics') self.inputMics.copyInfo(self.uMics) self.inputMics.setStore(False) for micU, micT in izip(self.uMics, self.tMics): micU.cleanObjId() micT.cleanObjId() self.inputMics.append(micU) self.inputMics.append(micT) self.samplingInput = self.uMics.getSamplingRate() if self.downsampleType.get() != OTHER: # If 'same as picking' or 'original' get sampling rate from input micrographs #TODO: Review this when downsampling before picking is possible self.samplingFinal = self.samplingInput else: # If 'other' multiply the input sampling rate by the factor provided self.samplingFinal = self.samplingInput*self.downFactor.get() # Write pos files for each micrograph firstStepId = self._insertFunctionStep('writePosFilesStep') # For each micrograph insert the steps #run in parallel deps = [] for mic in self.inputMics: localDeps = [firstStepId] micrographToExtract = mic.getFileName() micName = removeBaseExt(mic.getFileName()) micId = mic.getObjId() # If downsample type is 'other' perform a downsample if self.downsampleType == OTHER: fnDownsampled = self._getTmpPath(micName+"_downsampled.xmp") downFactor = self.downFactor.get() args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier" localDeps=[self._insertRunJobStep("xmipp_transform_downsample", args % locals(),prerequisites=localDeps)] micrographToExtract = fnDownsampled # If remove dust if self.doRemoveDust: fnNoDust = self._getTmpPath(micName+"_noDust.xmp") thresholdDust = self.thresholdDust.get() #TODO: remove this extra variable args=" -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f" localDeps=[self._insertRunJobStep("xmipp_transform_filter", args % locals(),prerequisites=localDeps)] micrographToExtract = fnNoDust #self._insertFunctionStep('getCTF', micId, micName, micrographToExtract) micName = removeBaseExt(mic.getFileName()) # Actually extract deps.append(self._insertFunctionStep('extractParticlesStep', micId, micName, None, micrographToExtract, prerequisites=localDeps)) # TODO: Delete temporary files # Insert step to create output objects self._insertFunctionStep('createOutputStep', prerequisites=deps)
def _insertAllSteps(self): """for each micrograph insert the steps to preprocess it """ self._setupBasicProperties() # Write pos files for each micrograph firstStepId = self._insertFunctionStep('writePosFilesStep') # For each micrograph insert the steps #run in parallel deps = [] for mic in self.inputMics: localDeps = [firstStepId] micrographToExtract = mic.getFileName() baseMicName = removeBaseExt(mic.getFileName()) if self.ctfRelations.hasValue(): micKey = self.micKey(mic) mic.setCTF(self.ctfDict[micKey]) # If downsample type is 'other' perform a downsample downFactor = self.downFactor.get() if self.downsampleType == OTHER and abs(downFactor - 1.) > 0.0001: fnDownsampled = self._getTmpPath(baseMicName + "_downsampled.xmp") args = "-i %(micrographToExtract)s -o %(fnDownsampled)s --step %(downFactor)f --method fourier" localDeps = [ self._insertRunJobStep("xmipp_transform_downsample", args % locals(), prerequisites=localDeps) ] micrographToExtract = fnDownsampled # If remove dust if self.doRemoveDust: fnNoDust = self._getTmpPath(baseMicName + "_noDust.xmp") thresholdDust = self.thresholdDust.get( ) #TODO: remove this extra variable args = " -i %(micrographToExtract)s -o %(fnNoDust)s --bad_pixels outliers %(thresholdDust)f" localDeps = [ self._insertRunJobStep("xmipp_transform_filter", args % locals(), prerequisites=localDeps) ] micrographToExtract = fnNoDust #self._insertFunctionStep('getCTF', micId, baseMicName, micrographToExtract) #FIXME: Check only if mic has CTF when implemented ok #if self.doFlip or mic.hasCTF(): fnCTF = None if self.ctfRelations.hasValue(): # If the micrograph doesn't come from Xmipp, we need to write # a Xmipp ctfparam file to perform the phase flip on the micrograph fnCTF = micrographToCTFParam( mic, self._getTmpPath("%s.ctfParam" % baseMicName)) # Insert step to flip micrograph if self.doFlip: localDeps = [ self._insertFunctionStep('flipMicrographStep', baseMicName, fnCTF, micrographToExtract, prerequisites=localDeps) ] micrographToExtract = self._getTmpPath(baseMicName + "_flipped.xmp") else: fnCTF = None # Actually extract deps.append( self._insertFunctionStep('extractParticlesStep', mic.getObjId(), baseMicName, fnCTF, micrographToExtract, prerequisites=localDeps)) # Insert step to create output objects metaDeps = self._insertFunctionStep('createMetadataImageStep', prerequisites=deps) if self.doSort: screenDep = self._insertFunctionStep('screenParticlesStep', prerequisites=[metaDeps]) finalDeps = [screenDep] else: finalDeps = [metaDeps] self._insertFunctionStep('createOutputStep', prerequisites=finalDeps)
def runScript(inputScript, ext, log=None, cwd=None): scriptName = removeBaseExt(inputScript) args = " %s @%s" % (ext, scriptName) runJob(log, SPIDER, args, env=dict(environment), cwd=cwd)
def runScript(inputScript, ext, program, nummpis, log=None, cwd=None): scriptName = removeBaseExt(inputScript) args = " %s @%s" % (ext, scriptName) runJob(log, program, args, numberOfMpi=nummpis, env=Plugin.getEnviron(), cwd=cwd)
def _getMicName(self, ctf): """ Get the micName to be used as Key from a given ctf object. """ return removeBaseExt(ctf.getMicrograph().getFileName())
def _getPlotName(self, movieName, plotType): if plotType == PLOT_CART: return removeBaseExt(movieName) + '_plot_cart.png' else: return removeBaseExt(movieName) + '_plot_polar.png'