def getSubparticles(self, localrec, partItem, symMatrices, params, subpartVectorList, filters): if getVersion() == '1.1.0': subparticles, _ = localrec.create_subparticles(partItem, symMatrices, subpartVectorList, params["dim"], self.relaxSym, self.randomize, "subparticles", params["unique"], 0, self.alignSubparticles, "", True, filters) else: subparticles, _ = localrec.create_subparticles(partItem, symMatrices, subpartVectorList, params["dim"], self.randomize, "subparticles", params["unique"], 0, self.alignSubparticles, "", False, filters) return subparticles
def convertInputStep(self, particlesId): """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. Params: particlesId: use this parameter just to force redo of convert if the input particles are changed. """ imgSet = self._getInputParticles() imgStar = self._getFileName('movie_particles') imgStarTmp = self._getTmpPath('movie_particles.star') self.info("Converting set from '%s' into '%s'" % (imgSet.getFileName(), imgStarTmp)) writeSetOfParticles(imgSet, imgStarTmp, self._getExtraPath(), alignType=imgSet.getAlignment(), extraLabels=MOVIE_EXTRA_LABELS) mdImg = md.MetaData(imgStarTmp) # replace mdColumn from *.stk to *.mrcs as Relion2 requires if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME from convert import relionToLocation, locationToRelion for objId in mdImg: index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId)) if not imgPath.endswith('mrcs'): newName = pwutils.replaceBaseExt(os.path.basename(imgPath), 'mrcs') newPath = self._getTmpPath(newName) newLoc = locationToRelion(index, newPath) if not exists(newPath): pwutils.createLink(imgPath, newPath) mdImg.setValue(mdColumn, newLoc, objId) mdImg.write(imgStar)
def organizeDataStep(self): from convert import relionToLocation, locationToRelion if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME shinyStar = self._getFileName('shiny') newDir = self._getExtraPath('polished_particles') pwutils.makePath(newDir) if not isVersion2(): pwutils.makePath(self._getExtraPath('shiny')) shinyOld = "shiny.star" inputFit = "movie_particles_shiny.star" try: pwutils.moveFile(shinyOld, shinyStar) pwutils.moveFile( self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star")) for half in self.PREFIXES: pwutils.moveFile( self._getPath( 'movie_particles_shiny_%sclass001_unfil.mrc' % half), self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half)) self._renameFiles('movie_particles_shiny_post*', 'movie_particles_') self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_') except: raise Exception('ERROR: some file(s) were not found!') # move polished particles from Tmp to Extra path # and restore previous mdColumn mdShiny = md.MetaData(shinyStar) oldPath = "" for objId in mdShiny: index, imgPath = relionToLocation( mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = pwutils.join(newDir, str(imgPath).split('/')[-1]) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldPath != imgPath and exists(imgPath): pwutils.moveFile(imgPath, newPath) oldPath = imgPath index2, imgPath2 = relionToLocation( mdShiny.getValue(mdColumn, objId)) absPath = os.path.realpath(imgPath2) newPath2 = 'Runs' + str(absPath).split('Runs')[1] newLoc2 = locationToRelion(index2, newPath2) mdShiny.setValue(mdColumn, newLoc2, objId) mdShiny.write(shinyStar, md.MD_OVERWRITE) pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def getSubparticles(self, localrec, partItem, symMatrices, params, subpartVectorList, filters, pxSize): if getVersion() == '1.2.1': subparticles, _ = localrec.create_subparticles( partItem, symMatrices, subpartVectorList, params["dim"], self.randomize, "subparticles", params["unique"], 0, self.alignSubparticles, "", False, filters, pxSize) elif getVersion() == '1.1.0': subparticles, _ = localrec.create_subparticles( partItem, symMatrices, subpartVectorList, params["dim"], self.relaxSym, self.randomize, "subparticles", params["unique"], 0, self.alignSubparticles, "", True, filters) else: subparticles, _ = localrec.create_subparticles( partItem, symMatrices, subpartVectorList, params["dim"], self.randomize, "subparticles", params["unique"], 0, self.alignSubparticles, "", False, filters) return subparticles
def _validate(self): errors = [] # Check that the program exists if not exists(self._getProgram()): errors.append("Binary '%s' does not exits.\n" "Check configuration file: \n" "~/.config/scipion/scipion.conf\n" "and set GCTF variables properly." % self._getProgram()) if self.doPhShEst and getVersion() == '0.50': errors.append('This version of Gctf (0.50) does not support phase shift estimation!' ' Please update to a newer version.') return errors
def convertInputStep(self, particlesId): """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. Params: particlesId: use this parameters just to force redo of convert if the input particles are changed. """ imgSet = self._getInputParticles() imgStar = self._getFileName('input_star') self.info("Converting set from '%s' into '%s'" % (imgSet.getFileName(), imgStar)) # Pass stack file as None to avoid write the images files writeSetOfParticles(imgSet, imgStar, self._getExtraPath()) if self.doCtfManualGroups: self._splitInCTFGroups(imgStar) if not self.IS_CLASSIFY: if self.realignMovieFrames: movieParticleSet = self.inputMovieParticles.get() auxMovieParticles = self._createSetOfMovieParticles(suffix='tmp') auxMovieParticles.copyInfo(movieParticleSet) # Discard the movie particles that are not present in the refinement set for movieParticle in movieParticleSet: particle = imgSet[movieParticle.getParticleId()] if particle is not None: auxMovieParticles.append(movieParticle) writeSetOfParticles(auxMovieParticles, self._getFileName('movie_particles'), None, originalSet=imgSet, postprocessImageRow=self._postprocessImageRow) mdMovies = md.MetaData(self._getFileName('movie_particles')) mdParts = md.MetaData(self._getFileName('input_star')) if getVersion() == "1.4": mdParts.renameColumn(md.RLN_IMAGE_NAME, md.RLN_PARTICLE_ORI_NAME) else: mdParts.renameColumn(md.RLN_IMAGE_NAME, md.RLN_PARTICLE_NAME) mdParts.removeLabel(md.RLN_MICROGRAPH_NAME) detectorPxSize = movieParticleSet.getAcquisition().getMagnification() * movieParticleSet.getSamplingRate() / 10000 mdAux = md.MetaData() mdMovies.fillConstant(md.RLN_CTF_DETECTOR_PIXEL_SIZE, detectorPxSize) mdAux.join2(mdMovies, mdParts, md.RLN_PARTICLE_ID, md.RLN_IMAGE_ID, md.INNER_JOIN) mdAux.write(self._getFileName('movie_particles'), md.MD_OVERWRITE) cleanPath(auxMovieParticles.getFileName())
def _argsGctf(self): self._args = " --apix %f " % self._params['sampling'] self._args += "--kV %f " % self._params['voltage'] self._args += "--cs %f " % self._params['sphericalAberration'] self._args += "--ac %f " % self._params['ampContrast'] self._args += "--dstep %f " % self._params['scannedPixelSize'] self._args += "--defL %f " % self._params['minDefocus'] self._args += "--defH %f " % self._params['maxDefocus'] self._args += "--defS %f " % self._params['step_focus'] self._args += "--astm %f " % self.astigmatism.get() self._args += "--resL %f " % self._params['lowRes'] self._args += "--resH %f " % self._params['highRes'] self._args += "--do_EPA %d " % (1 if self.doEPA else 0) self._args += "--boxsize %d " % self._params['windowSize'] self._args += "--plot_res_ring %d " % (1 if self.plotResRing else 0) self._args += "--gid %d " % self.GPUCore.get() self._args += "--bfac %d " % self.bfactor.get() self._args += "--B_resH %f " % (2 * self._params['sampling']) self._args += "--overlap %f " % self.overlap.get() self._args += "--convsize %d " % self.convsize.get() self._args += "--do_Hres_ref %d " % (1 if self.doHighRes else 0) # local refine options self._args += "--do_local_refine 1 --boxsuffix _coords.star " self._args += "--local_radius %d " % self.locRad.get() self._args += "--local_avetype %d " % self.locAveType.get() self._args += "--local_boxsize %d " % self.locBoxSize.get() self._args += "--local_overlap % 0.2f " % self.locOverlap.get() self._args += "--local_resL %d " % self.locResL.get() self._args += "--local_resH %d " % self.locResH.get() self._args += "--refine_local_astm %d " % (1 if self.locAstm else 0) if getVersion() == '0.50': self._args += "--do_basic_rotave %d " % (1 if self.doBasicRotave else 0) else: self._args += "--EPA_oversmp %d " % self.EPAsmp.get() if self.doPhShEst: self._args += "--phase_shift_L %f " % self.phaseShiftL.get() self._args += "--phase_shift_H %f " % self.phaseShiftH.get() self._args += "--phase_shift_S %f " % self.phaseShiftS.get() self._args += "--phase_shift_T %d " % (1 + self.phaseShiftT.get()) if self.doHighRes: self._args += "--Href_resL %d " % self.HighResL.get() self._args += "--Href_resH %d " % self.HighResH.get() self._args += "--Href_bfac %d " % self.HighResBf.get()
def organizeDataStep(self): from convert import relionToLocation, locationToRelion if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME shinyStar = self._getFileName('shiny') newDir = self._getExtraPath('polished_particles') pwutils.makePath(newDir) if not isVersion2(): pwutils.makePath(self._getExtraPath('shiny')) shinyOld = "shiny.star" inputFit = "movie_particles_shiny.star" try: pwutils.moveFile(shinyOld, shinyStar) pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star")) for half in self.PREFIXES: pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half), self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half)) self._renameFiles('movie_particles_shiny_post*', 'movie_particles_') self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_') except: raise Exception('ERROR: some file(s) were not found!') # move polished particles from Tmp to Extra path # and restore previous mdColumn mdShiny = md.MetaData(shinyStar) oldPath = "" for objId in mdShiny: index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = pwutils.join(newDir, str(imgPath).split('/')[-1]) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldPath != imgPath and exists(imgPath): pwutils.moveFile(imgPath, newPath) oldPath = imgPath index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId)) absPath = os.path.realpath(imgPath2) newPath2 = 'Runs' + str(absPath).split('Runs')[1] newLoc2 = locationToRelion(index2, newPath2) mdShiny.setValue(mdColumn, newLoc2, objId) mdShiny.write(shinyStar, md.MD_OVERWRITE) pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def versionGE(self, version): """ Return True if current version of motioncor2 is greater or equal than the input argument. Params: version: string version (semantic version, e.g 1.0.1) """ if not self.isSemVersion(): return False v1 = map(int, getVersion('MOTIONCOR2').split('.')) v2 = map(int, version.split('.')) for x1, x2 in zip(v1, v2): if x1 < x2: return False return True
def _argsGctf(self): self._args = " --apix %f " % self._params['sampling'] self._args += "--kV %f " % self._params['voltage'] self._args += "--cs %f " % self._params['sphericalAberration'] self._args += "--ac %f " % self._params['ampContrast'] self._args += "--dstep %f " % self._params['scannedPixelSize'] self._args += "--defL %f " % self._params['minDefocus'] self._args += "--defH %f " % self._params['maxDefocus'] self._args += "--defS %f " % self._params['step_focus'] self._args += "--astm %f " % self.astigmatism.get() self._args += "--resL %f " % self._params['lowRes'] self._args += "--resH %f " % self._params['highRes'] self._args += "--do_EPA %d " % (1 if self.doEPA else 0) self._args += "--boxsize %d " % self._params['windowSize'] self._args += "--plot_res_ring %d " % (1 if self.plotResRing else 0) self._args += "--gid %%(GPU)s " # Use %% to escape when formatting self._args += "--bfac %d " % self.bfactor.get() self._args += "--B_resH %f " % (2 * self._params['sampling']) self._args += "--overlap %f " % self.overlap.get() self._args += "--convsize %d " % self.convsize.get() self._args += "--do_Hres_ref %d " % (1 if self.doHighRes else 0) if getVersion() == '0.50': self._args += "--do_basic_rotave %d " % (1 if self.doBasicRotave else 0) else: self._args += "--EPA_oversmp %d " % self.EPAsmp.get() if self.doPhShEst: self._args += "--phase_shift_L %f " % self.phaseShiftL.get() self._args += "--phase_shift_H %f " % self.phaseShiftH.get() self._args += "--phase_shift_S %f " % self.phaseShiftS.get() self._args += "--phase_shift_T %d " % (1 + self.phaseShiftT.get()) if self.doHighRes: self._args += "--Href_resL %d " % self.HighResL.get() self._args += "--Href_resH %d " % self.HighResH.get() self._args += "--Href_bfac %d " % self.HighResBf.get() self._args += "--do_validation %d " % (1 if self.doValidate else 0) self._args += "%(micFn)s " self._args += "> %(gctfOut)s"
def _validate(self): errors = [] # Check that the program exists if not exists(self._getProgram()): errors.append("Binary '%s' does not exits.\n" "Check configuration file: \n" "~/.config/scipion/scipion.conf\n" "and set GCTF variables properly." % self._getProgram()) if self.doPhShEst and getVersion() == '0.50': errors.append('This version of Gctf (0.50) does not support phase ' 'shift estimation! Please update to a newer version.') nprocs = max(self.numberOfMpi.get(), self.numberOfThreads.get()) if nprocs < len(self.getGpuList()): errors.append("Multiple GPUs can not be used by a single process. " "Make sure you specify more processors than GPUs. ") if self._getStreamingBatchSize() > 1: errors.append("Batch steps are not implemented yet for Gctf. ") return errors
def isDisabled(cls): return getVersion() in [V1_3, V1_4, V2_0]
def _supportsMagCorrection(self): return getVersion('MOTIONCOR2') not in ['03162016', '10192016']
def isSemVersion(self): """ Return True if it is a semantic version of motioncor2. It started with release 1.0.0. """ return getVersion('MOTIONCOR2').startswith('1.0.')
def _isOutStackSupport(self): # checks if output aligned movies can be saved by motioncor2 return True if getVersion('MOTIONCOR2') != '03162016' else False
def _processMovie(self, movie): inputMovies = self.inputMovies.get() movieFolder = self._getOutputMovieFolder(movie) outputMicFn = self._getRelPath(self._getOutputMicName(movie), movieFolder) outputMovieFn = self._getRelPath(self._getOutputMovieName(movie), movieFolder) movieBaseName = pwutils.removeExt(movie.getFileName()) aveMicFn = movieBaseName + '_uncorrected_avg.mrc' logFile = self._getRelPath(self._getMovieLogFile(movie), movieFolder) a0, aN = self._getRange(movie, 'align') if not self.useMotioncor2: # Get the number of frames and the range to be used # for alignment and sum s0, sN = self._getRange(movie, 'sum') argsDict = { '-crx': self.cropOffsetX.get(), '-cry': self.cropOffsetY.get(), '-cdx': self.cropDimX.get(), '-cdy': self.cropDimY.get(), '-bin': self.binFactor.get(), '-nst': '%d' % a0, '-ned': '%d' % aN, '-nss': '%d' % s0, '-nes': '%d' % sN, '-gpu': self.GPUIDs.get(), '-flg': logFile, } args = '"%s" ' % movie.getBaseName() args += ' '.join( ['%s %s' % (k, v) for k, v in argsDict.iteritems()]) if inputMovies.getGain(): args += ' -fgr "%s"' % inputMovies.getGain() if inputMovies.getDark(): args += ' -fdr "%s"' % inputMovies.getDark() if self.doSaveAveMic: args += ' -fcs "%s" ' % outputMicFn if self.doSaveMovie: args += ' -fct "%s" -ssc 1' % outputMovieFn args += ' ' + self.extraParams.get() program = MOTIONCORR_PATH else: logFileBase = (logFile.replace('0-Full.log', '').replace('0-Patch-Full.log', '')) # default values for motioncor2 are (1, 1) cropDimX = self.cropDimX.get() or 1 cropDimY = self.cropDimY.get() or 1 numbOfFrames = self._getNumberOfFrames(movie) if self.doApplyDoseFilter: preExp, dose = self._getCorrectedDose(inputMovies) else: preExp, dose = 0.0, 0.0 # reset values = 1 to 0 (motioncor2 does it automatically, # but we need to keep this for consistency) if self.patchX.get() == 1: self.patchX.set(0) if self.patchY.get() == 1: self.patchY.set(0) argsDict = { '-OutMrc': '"%s"' % outputMicFn, '-Patch': '%d %d' % (self.patchX, self.patchY), '-MaskCent': '%d %d' % (self.cropOffsetX, self.cropOffsetY), '-MaskSize': '%d %d' % (cropDimX, cropDimY), '-FtBin': self.binFactor.get(), '-Tol': self.tol.get(), '-Group': self.group.get(), '-FmDose': dose, '-Throw': '%d' % a0, '-Trunc': '%d' % (abs(aN - numbOfFrames + 1)), '-PixSize': inputMovies.getSamplingRate(), '-kV': inputMovies.getAcquisition().getVoltage(), '-Gpu': self.GPUIDs.get(), '-LogFile': logFileBase, } if getVersion('MOTIONCOR2') != '03162016': argsDict['-InitDose'] = preExp argsDict['-OutStack'] = 1 if self.doSaveMovie else 0 if self.isSemVersion(): if self.defectFile.get(): argsDict['-DefectFile'] = self.defectFile.get() if self.versionGE( '1.0.1'): # Patch overlap was introduced in 1.0.1 patchOverlap = self.getAttributeValue('patchOverlap', None) if patchOverlap: # 0 or None is False argsDict['-Patch'] += " %d" % patchOverlap if self._supportsMagCorrection() and self.doMagCor: if self.useEst: inputEst = self.inputEst.get().getOutputLog() if getVersion('MOTIONCOR2') == '01302017': input_params = parseMagCorrInput(inputEst) # this version uses stretch parameters as following: # 1/maj, 1/min, -angle argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % ( 1.0 / input_params[1], 1.0 / input_params[2], -1 * input_params[0]) else: # While motioncor2 >=1.0.0 uses estimation params AS IS input_params = parseMagEstOutput(inputEst) argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % ( input_params[1], input_params[2], input_params[0]) else: argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % ( self.scaleMaj, self.scaleMin, self.angDist) ext = pwutils.getExt(movie.getFileName()).lower() if ext in ['.mrc', '.mrcs']: args = ' -InMrc "%s" ' % movie.getBaseName() elif ext in ['.tif', '.tiff']: args = ' -InTiff "%s" ' % movie.getBaseName() else: raise Exception("Unsupported format: %s" % ext) args += ' '.join( ['%s %s' % (k, v) for k, v in argsDict.iteritems()]) if inputMovies.getGain(): args += ' -Gain "%s" ' % inputMovies.getGain() if inputMovies.getDark(): args += ' -Dark "%s"' % inputMovies.getDark() args += ' ' + self.extraParams2.get() program = MOTIONCOR2_PATH try: self.runJob(program, args, cwd=movieFolder, env=getEnviron(self.useMotioncor2)) self._fixMovie(movie) # Compute PSDs outMicFn = self._getExtraPath(self._getOutputMicName(movie)) if not os.path.exists(outMicFn): # if only DW mic is saved outMicFn = self._getExtraPath(self._getOutputMicWtName(movie)) if self.doComputePSD: # Compute uncorrected avg mic roi = [ self.cropOffsetX.get(), self.cropOffsetY.get(), self.cropDimX.get(), self.cropDimY.get() ] fakeShiftsFn = self.writeZeroShifts(movie) self.averageMovie(movie, fakeShiftsFn, aveMicFn, binFactor=self.binFactor.get(), roi=roi, dark=inputMovies.getDark(), gain=inputMovies.getGain()) self.computePSDs(movie, aveMicFn, outMicFn, outputFnCorrected=self._getPsdJpeg(movie)) self._saveAlignmentPlots(movie) if self._doComputeMicThumbnail(): self.computeThumbnail( outMicFn, outputFn=self._getOutputMicThumbnail(movie)) except: print("ERROR: Movie %s failed\n" % movie.getName())
def isDisabled(cls): return getVersion() in [V1_3, V1_4]
def createOutputStep(self): setEnviron() # Set the environment to access localrec modules import localrec import pyrelion # return inputSet = self._getInputParticles() outputSet = self._createSetOfCoordinates(inputSet) params = { "symmetryGroup": self.symmetryGroup.get(), "vector": self.vector.get(), "vectorFile": self.vectorFile.get(), "length": self.length.get(), "unique": self.unique.get(), "mindist": self.mindist.get(), "side": self.side.get(), "top": self.top.get(), "pxSize": self.inputParticles.get().getSamplingRate(), "dim": self.inputParticles.get().getXDim() } symMatrices = localrec.matrix_from_symmetry(self.symmetryGroup.get()) if self.defineVector == CMM: cmmFn = params["vectorFile"] vector = " " else: cmmFn = "" vector = params["vector"] subpartVectorList = localrec.load_vectors(cmmFn, vector, params["length"], params["pxSize"]) # Define some conditions to filter subparticles filters = localrec.load_filters(math.radians(params["side"]), math.radians(params["top"]), params["mindist"]) coord = Coordinate() for part in inputSet: partItem = pyrelion.Item() particleToRow(part, partItem) if getVersion() == '1.2.1': subparticles = self.getSubparticles(localrec, partItem, symMatrices, params, subpartVectorList, filters, params["pxSize"]) else: subparticles = self.getSubparticles(localrec, partItem, symMatrices, params, subpartVectorList, filters) for subpart in subparticles: rowToSubcoordinate(subpart, coord, part) coord.setObjId(None) # Force to insert as a new item outputSet.append(coord) if part.hasAttribute('_rlnRandomSubset'): coord._subparticle.copyAttributes(part, '_rlnRandomSubset') restituteRelionHome() self._defineOutputs(outputCoordinates=outputSet) self._defineSourceRelation(self.inputParticles, outputSet)
def _processMovie(self, movie): inputMovies = self.inputMovies.get() movieFolder = self._getOutputMovieFolder(movie) outputMicFn = self._getRelPath(self._getOutputMicName(movie), movieFolder) outputMovieFn = self._getRelPath(self._getOutputMovieName(movie), movieFolder) movieBaseName = pwutils.removeExt(movie.getFileName()) aveMicFn = movieBaseName + '_uncorrected_avg.mrc' logFile = self._getRelPath(self._getMovieLogFile(movie), movieFolder) a0, aN = self._getRange(movie, 'align') if not self.useMotioncor2: # Get the number of frames and the range to be used # for alignment and sum s0, sN = self._getRange(movie, 'sum') argsDict = { '-crx': self.cropOffsetX.get(), '-cry': self.cropOffsetY.get(), '-cdx': self.cropDimX.get(), '-cdy': self.cropDimY.get(), '-bin': self.binFactor.get(), '-nst': '%d' % a0, '-ned': '%d' % aN, '-nss': '%d' % s0, '-nes': '%d' % sN, '-gpu': self.GPUIDs.get(), '-flg': logFile, } args = '"%s" ' % movie.getBaseName() args += ' '.join( ['%s %s' % (k, v) for k, v in argsDict.iteritems()]) if inputMovies.getGain(): args += ' -fgr "%s"' % inputMovies.getGain() if inputMovies.getDark(): args += ' -fdr "%s"' % inputMovies.getDark() if self.doSaveAveMic: args += ' -fcs "%s" ' % outputMicFn if self.doSaveMovie: args += ' -fct "%s" -ssc 1' % outputMovieFn args += ' ' + self.extraParams.get() program = MOTIONCORR_PATH else: logFileBase = (logFile.replace('0-Full.log', '').replace('0-Patch-Full.log', '')) # default values for motioncor2 are (1, 1) cropDimX = self.cropDimX.get() or 1 cropDimY = self.cropDimY.get() or 1 numbOfFrames = self._getNumberOfFrames(movie) if self.doApplyDoseFilter: preExp, dose = self._getCorrectedDose(inputMovies) else: preExp, dose = 0.0, 0.0 argsDict = { '-OutMrc': '"%s"' % outputMicFn, '-Patch': '%d %d' % (self.patchX, self.patchY), '-MaskCent': '%d %d' % (self.cropOffsetX, self.cropOffsetY), '-MaskSize': '%d %d' % (cropDimX, cropDimY), '-FtBin': self.binFactor.get(), '-Tol': self.tol.get(), '-Group': self.group.get(), '-FmDose': dose, '-Throw': '%d' % a0, '-Trunc': '%d' % (abs(aN - numbOfFrames + 1)), '-PixSize': inputMovies.getSamplingRate(), '-kV': inputMovies.getAcquisition().getVoltage(), '-Gpu': self.GPUIDs.get(), '-LogFile': logFileBase, } if getVersion('MOTIONCOR2') != '03162016': argsDict['-InitDose'] = preExp argsDict['-OutStack'] = 1 if self.doSaveMovie else 0 if self._supportsMagCorrection() and self.doMagCor: if self.useEst: inputEst = self.inputEst.get().getOutputLog() input_params = parseMagCorrInput(inputEst) # mag dist angle is inverted due to a different convention argsDict['-Mag'] = '%0.2f %0.2f %0.2f' % ( input_params[1], input_params[2], -1 * input_params[0]) else: argsDict['-Mag'] = '%0.2f %0.2f %0.2f' % ( self.scaleMaj.get(), self.scaleMin.get(), self.angDist.get()) args = ' -InMrc "%s" ' % movie.getBaseName() args += ' '.join( ['%s %s' % (k, v) for k, v in argsDict.iteritems()]) if inputMovies.getGain(): args += ' -Gain "%s" ' % inputMovies.getGain() args += ' ' + self.extraParams2.get() program = MOTIONCOR2_PATH try: self.runJob(program, args, cwd=movieFolder, env=getEnviron(self.useMotioncor2)) self._fixMovie(movie) # Compute PSDs outMicFn = self._getExtraPath(self._getOutputMicName(movie)) if not os.path.exists(outMicFn): # if only DW mic is saved outMicFn = self._getExtraPath(self._getOutputMicWtName(movie)) if self.doComputePSD: uncorrectedPSD = movieBaseName + '_uncorrected' correctedPSD = movieBaseName + '_corrected' # Compute uncorrected avg mic roi = [ self.cropOffsetX.get(), self.cropOffsetY.get(), self.cropDimX.get(), self.cropDimY.get() ] fakeShiftsFn = self.writeZeroShifts(movie) self.averageMovie(movie, fakeShiftsFn, aveMicFn, binFactor=self.binFactor.get(), roi=roi, dark=None, gain=inputMovies.getGain()) self.computePSD(aveMicFn, uncorrectedPSD) self.computePSD(outMicFn, correctedPSD) self.composePSD(uncorrectedPSD + ".psd", correctedPSD + ".psd", self._getPsdCorr(movie)) self._saveAlignmentPlots(movie) if self._doComputeMicThumbnail(): self.computeThumbnail( outMicFn, outputFn=self._getOutputMicThumbnail(movie)) except: print("ERROR: Movie %s failed\n" % movie.getName())
def convertInputStep(self, particlesId): """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. Params: particlesId: use this parameters just to force redo of convert if the input particles are changed. """ imgSet = self._getInputParticles() imgStar = self._getFileName('input_star') self.info("Converting set from '%s' into '%s'" % (imgSet.getFileName(), imgStar)) # Pass stack file as None to avoid write the images files writeSetOfParticles(imgSet, imgStar, self._getExtraPath()) if self.doCtfManualGroups: self._splitInCTFGroups(imgStar) if not self.IS_CLASSIFY: if self.realignMovieFrames: movieParticleSet = self.inputMovieParticles.get() auxMovieParticles = self._createSetOfMovieParticles( suffix='tmp') auxMovieParticles.copyInfo(movieParticleSet) # Discard the movie particles that are not present in the refinement set for movieParticle in movieParticleSet: particle = imgSet[movieParticle.getParticleId()] if particle is not None: auxMovieParticles.append(movieParticle) writeSetOfParticles( auxMovieParticles, self._getFileName('movie_particles'), None, originalSet=imgSet, postprocessImageRow=self._postprocessImageRow) mdMovies = md.MetaData(self._getFileName('movie_particles')) mdParts = md.MetaData(self._getFileName('input_star')) if getVersion() == "1.4": mdParts.renameColumn(md.RLN_IMAGE_NAME, md.RLN_PARTICLE_ORI_NAME) else: mdParts.renameColumn(md.RLN_IMAGE_NAME, md.RLN_PARTICLE_NAME) mdParts.removeLabel(md.RLN_MICROGRAPH_NAME) detectorPxSize = movieParticleSet.getAcquisition( ).getMagnification() * movieParticleSet.getSamplingRate( ) / 10000 mdAux = md.MetaData() mdMovies.fillConstant(md.RLN_CTF_DETECTOR_PIXEL_SIZE, detectorPxSize) mdAux.join2(mdMovies, mdParts, md.RLN_PARTICLE_ID, md.RLN_IMAGE_ID, md.INNER_JOIN) mdAux.write(self._getFileName('movie_particles'), md.MD_OVERWRITE) cleanPath(auxMovieParticles.getFileName())
def _oldVersion(self): return True if getVersion() == '0.50' else False
def _processMovie(self, movie): inputMovies = self.inputMovies.get() movieFolder = self._getOutputMovieFolder(movie) outputMicFn = self._getRelPath(self._getOutputMicName(movie), movieFolder) outputMovieFn = self._getRelPath(self._getOutputMovieName(movie), movieFolder) movieBaseName = pwutils.removeExt(movie.getFileName()) aveMicFn = movieBaseName + '_uncorrected_avg.mrc' logFile = self._getRelPath(self._getMovieLogFile(movie), movieFolder) a0, aN = self._getRange(movie, 'align') if not self.useMotioncor2: # Get the number of frames and the range to be used # for alignment and sum s0, sN = self._getRange(movie, 'sum') argsDict = {'-crx': self.cropOffsetX.get(), '-cry': self.cropOffsetY.get(), '-cdx': self.cropDimX.get(), '-cdy': self.cropDimY.get(), '-bin': self.binFactor.get(), '-nst': '%d' % a0, '-ned': '%d' % aN, '-nss': '%d' % s0, '-nes': '%d' % sN, '-flg': logFile, } args = '"%s" ' % movie.getBaseName() args += ' '.join( ['%s %s' % (k, v) for k, v in argsDict.iteritems()]) if inputMovies.getGain(): args += ' -fgr "%s"' % inputMovies.getGain() if inputMovies.getDark(): args += ' -fdr "%s"' % inputMovies.getDark() if self.doSaveAveMic: args += ' -fcs "%s" ' % outputMicFn if self.doSaveMovie: args += ' -fct "%s" -ssc 1' % outputMovieFn args += ' -gpu %(GPU)s' args += ' ' + self.extraParams.get() program = MOTIONCORR_PATH else: logFileBase = (logFile.replace('0-Full.log', '').replace( '0-Patch-Full.log', '')) # default values for motioncor2 are (1, 1) cropDimX = self.cropDimX.get() or 1 cropDimY = self.cropDimY.get() or 1 numbOfFrames = self._getNumberOfFrames(movie) if self.doApplyDoseFilter: preExp, dose = self._getCorrectedDose(inputMovies) else: preExp, dose = 0.0, 0.0 # reset values = 1 to 0 (motioncor2 does it automatically, # but we need to keep this for consistency) if self.patchX.get() == 1: self.patchX.set(0) if self.patchY.get() == 1: self.patchY.set(0) argsDict = {'-OutMrc': '"%s"' % outputMicFn, '-Patch': '%d %d' % (self.patchX, self.patchY), '-MaskCent': '%d %d' % (self.cropOffsetX, self.cropOffsetY), '-MaskSize': '%d %d' % (cropDimX, cropDimY), '-FtBin': self.binFactor.get(), '-Tol': self.tol.get(), '-Group': self.group.get(), '-FmDose': dose, '-Throw': '%d' % a0, '-Trunc': '%d' % (abs(aN - numbOfFrames + 1)), '-PixSize': inputMovies.getSamplingRate(), '-kV': inputMovies.getAcquisition().getVoltage(), '-LogFile': logFileBase, } if getVersion('MOTIONCOR2') != '03162016': argsDict['-InitDose'] = preExp argsDict['-OutStack'] = 1 if self.doSaveMovie else 0 if self.isSemVersion(): if self.defectFile.get(): argsDict['-DefectFile'] = self.defectFile.get() if self.versionGE('1.0.1'): # Patch overlap was introduced in 1.0.1 patchOverlap = self.getAttributeValue('patchOverlap', None) if patchOverlap: # 0 or None is False argsDict['-Patch'] += " %d" % patchOverlap if self._supportsMagCorrection() and self.doMagCor: if self.useEst: inputEst = self.inputEst.get().getOutputLog() if getVersion('MOTIONCOR2') == '01302017': input_params = parseMagCorrInput(inputEst) # this version uses stretch parameters as following: # 1/maj, 1/min, -angle argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % ( 1.0 / input_params[1], 1.0 / input_params[2], -1 * input_params[0]) else: # While motioncor2 >=1.0.0 uses estimation params AS IS input_params = parseMagEstOutput(inputEst) argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % ( input_params[1], input_params[2], input_params[0]) else: argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % (self.scaleMaj, self.scaleMin, self.angDist) ext = pwutils.getExt(movie.getFileName()).lower() if ext in ['.mrc', '.mrcs']: args = ' -InMrc "%s" ' % movie.getBaseName() elif ext in ['.tif', '.tiff']: args = ' -InTiff "%s" ' % movie.getBaseName() else: raise Exception("Unsupported format: %s" % ext) args += ' '.join(['%s %s' % (k, v) for k, v in argsDict.iteritems()]) if inputMovies.getGain(): args += ' -Gain "%s" ' % inputMovies.getGain() if inputMovies.getDark(): args += ' -Dark "%s"' % inputMovies.getDark() args += ' -Gpu %(GPU)s' args += ' ' + self.extraParams2.get() program = MOTIONCOR2_PATH try: self.runJob(program, args, cwd=movieFolder, env=getEnviron(self.useMotioncor2)) self._fixMovie(movie) # Compute PSDs outMicFn = self._getExtraPath(self._getOutputMicName(movie)) if not os.path.exists(outMicFn): # if only DW mic is saved outMicFn = self._getExtraPath(self._getOutputMicWtName(movie)) def _extraWork(): if self.doComputePSD: # Compute uncorrected avg mic roi = [self.cropOffsetX.get(), self.cropOffsetY.get(), self.cropDimX.get(), self.cropDimY.get()] fakeShiftsFn = self.writeZeroShifts(movie) self.averageMovie(movie, fakeShiftsFn, aveMicFn, binFactor=self.binFactor.get(), roi=roi, dark=inputMovies.getDark(), gain=inputMovies.getGain()) self.computePSDs(movie, aveMicFn, outMicFn, outputFnCorrected=self._getPsdJpeg(movie)) self._saveAlignmentPlots(movie) if self._doComputeMicThumbnail(): self.computeThumbnail(outMicFn, outputFn=self._getOutputMicThumbnail( movie)) # This protocols takes control of clean up the temporary movie folder # which is required mainly when using a thread for this extra work self._cleanMovieFolder(movieFolder) if self._useWorkerThread(): thread = Thread(target=_extraWork) thread.start() else: _extraWork() except: print("ERROR: Movie %s failed\n" % movie.getName())
def _defineParams(self, form): form.addSection(label='Input') form.addParam( 'protRefine', PointerParam, pointerClass="ProtRefine3D", label='Select a previous refinement protocol', help='Select any previous refinement protocol to get the ' '3D half maps. Note that is recomended that the ' 'refinement protocol uses a gold-standard method.') form.addSection(label='Masking') form.addParam('doAutoMask', BooleanParam, default=True, label='Perform automated masking?', help='Perform automated masking, based on a density ' 'threshold') form.addParam('initMaskThreshold', FloatParam, default=0.02, condition='doAutoMask', label='Initial binarisation threshold', help='This threshold is used to make an initial binary ' 'mask from the average of the two unfiltered ' 'half-reconstructions. If you do not know what ' 'value to use, display one of the unfiltered ' 'half-maps in a 3D surface rendering viewer, that ' 'gives no noise peaks outside the reconstruction.') form.addParam('extendInitMask', IntParam, default=3, label='Mask pixels extension (px)', condition='doAutoMask', help='The initial binary mask is extended this number ' 'of pixels in all directions.') form.addParam('addMaskEdge', IntParam, default=6, label='add soft-edge width (px)', condition='doAutoMask', help='The extended binary mask is further extended with ' 'a raised-cosine soft edge of the specified width.') form.addParam( 'mask', PointerParam, pointerClass='VolumeMask', label='Provide a mask', allowsNull=True, condition='not doAutoMask', help='Provide a soft mask where the protein is white (1) ' 'and the solvent is black (0). Often, the softer ' 'the mask the higher resolution estimates you will ' 'get. A soft edge of 5-10 pixels is often a good ' 'edge width.') form.addSection(label='Sharpening') form.addParam('mtf', FileParam, label='MTF-curve file', help='User-provided STAR-file with the MTF-curve ' 'of the detector.' 'Relion param: <--mtf>') form.addParam('doAutoBfactor', BooleanParam, default=True, label='Estimate B-factor automatically?', help='If set to Yes, then the program will use the ' 'automated procedure described by Rosenthal and ' 'Henderson (2003, JMB) to estimate an overall ' 'B-factor for your map, and sharpen it accordingly.') line = form.addLine('B-factor resolution (A): ', condition='doAutoBfactor', help='There are the frequency (in Angstroms), ' 'lowest and highest, that will be included in ' 'the linear fit of the Guinier plot as ' 'described in Rosenthal and Henderson ' '(2003, JMB).') line.addParam('bfactorLowRes', FloatParam, default='10.0', label='low') line.addParam('bfactorHighRes', FloatParam, default='0.0', label='high') form.addParam('bfactor', FloatParam, default=-350, condition='not doAutoBfactor', label='Provide B-factor:', help='User-provided B-factor (in A^2) for map ' 'sharpening, e.g. -400. Use negative values for ' 'sharpening. Be careful: if you over-sharpen\n' 'your map, you may end up interpreting noise for ' 'signal!\n' 'Relion param: *--adhoc_bfac*') form.addSection(label='Filtering') form.addParam('skipFscWeighting', BooleanParam, default=False, label='Skip FSC-weighting for sharpening?', help='If set to No (the default), then the output map ' 'will be low-pass filtered according to the ' 'mask-corrected, gold-standard FSC-curve. ' 'Sometimes, it is also useful to provide an ad-hoc ' 'low-pass filter (option below), as due to local ' 'resolution variations some parts of the map may ' 'be better and other parts may be worse than the ' 'overall resolution as measured by the FSC. In ' 'such cases, set this option to Yes and provide ' 'an ad-hoc filter as described below.') form.addParam('lowRes', FloatParam, default=5, condition='skipFscWeighting', label='Low-pass filter (A):', help='This option allows one to low-pass filter the map ' 'at a user-provided frequency (in Angstroms). When ' 'using a resolution that is higher than the ' 'gold-standard FSC-reported resolution, take care ' 'not to interpret noise in the map for signal...') if getVersion() == "2.0": pass form.addParam('filterEdgeWidth', IntParam, default=2, expertLevel=LEVEL_ADVANCED, label='Low-pass filter edge width:', help='Width of the raised cosine on the low-pass filter ' 'edge (in resolution shells)\n' 'Relion param: *--filter_edge_width*') form.addParam('randomizeAtFsc', FloatParam, default=0.8, expertLevel=LEVEL_ADVANCED, label='Randomize phases threshold', help='Randomize phases from the resolution where FSC ' 'drops below this value\n' 'Relion param: *--randomize_at_fsc*') form.addParallelSection(threads=0, mpi=0)