def _pickMicrograph(self, mic, args): # Prepare mic folder and convert if needed micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) ih = ImageHandler() # If needed convert micrograph to mrc format, otherwise link it if pwutils.getExt(micName) != ".mrc": fnMicBase = pwutils.replaceBaseExt(micName, 'mrc') inputMic = os.path.join(micDir, fnMicBase) ih.convert(mic.getLocation(), inputMic) else: inputMic = os.path.join(micDir, os.path.basename(micName)) pwutils.createLink(micName, inputMic) # Prepare environment from appion import Plugin Plugin.getEnviron() # Program to execute and it arguments program = "python2" outputFile = self._getExtraPath(pwutils.replaceBaseExt( inputMic, "txt")) args += " --image=%s --outfile=%s" % (inputMic, outputFile) dogpicker = Plugin.getHome("ApDogPicker.py") args = dogpicker + " " + args self.runJob(program, args)
def convertInputStep(self, particlesId): """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. Params: particlesId: use this parameter just to force redo of convert if the input particles are changed. """ imgSet = self._getInputParticles() imgStar = self._getFileName('movie_particles') imgStarTmp = self._getTmpPath('movie_particles.star') self.info("Converting set from '%s' into '%s'" % (imgSet.getFileName(), imgStarTmp)) writeSetOfParticles(imgSet, imgStarTmp, self._getExtraPath(), alignType=imgSet.getAlignment(), extraLabels=MOVIE_EXTRA_LABELS) mdImg = md.MetaData(imgStarTmp) # replace mdColumn from *.stk to *.mrcs as Relion2 requires if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME from convert import relionToLocation, locationToRelion for objId in mdImg: index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId)) if not imgPath.endswith('mrcs'): newName = pwutils.replaceBaseExt(os.path.basename(imgPath), 'mrcs') newPath = self._getTmpPath(newName) newLoc = locationToRelion(index, newPath) if not exists(newPath): pwutils.createLink(imgPath, newPath) mdImg.setValue(mdColumn, newLoc, objId) mdImg.write(imgStar)
def createBinaryLink(fn): """ Just create a link named .mrcs to Relion understand that it is a binary stack file and not a volume. """ newFn = getUniqueFileName(fn, extension) pwutils.createLink(fn, newFn) return newFn
def _inputVol2Mrc(self, inputFname, outputFname): if inputFname.endswith(".mrc") or inputFname.endswith(".map"): if not os.path.exists(outputFname): createLink(inputFname, outputFname) else: self.runJob('xmipp_image_convert', " -i %s -o %s:mrc -t vol" % (inputFname, outputFname))
def convertInputStep(self, micsId, refsId): """ This step will take of the convertions from the inputs. Micrographs: they will be linked if are in '.mrc' format, converted otherwise. References: will always be converted to '.mrc' format Mask: either converted ('.tif' format) or generated a circular one """ ih = em.ImageHandler() micDir = self._getTmpPath('micrographs') pwutils.makePath(micDir) for mic in self.getInputMicrographs(): # Create micrograph folder micName = mic.getFileName() # If micrographs are in .mrc format just link it # otherwise convert them outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc')) if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: ih.convert(mic, outMic) refDir = self._getTmpPath('templates') pwutils.makePath(refDir) # We will always convert the templates, since # they can be in an stack and link will not be possible sometimes inputRefs = self.inputReferences.get() for i, ref in enumerate(inputRefs): outRef = join(refDir, 'ref%02d.mrc' % (i + 1)) ih.convert(ref, outRef) self.createInputMasks(inputRefs)
def _mergeDataStar(self, outStar, mdInput, iter, rLev): imgStar = self._getFileName('data', iter=iter, lev=self._level, rLev=rLev) mdData = md.MetaData(imgStar) print("reading %s and begin the loop" % imgStar) for row in md.iterRows(mdData, sortByLabel=md.RLN_PARTICLE_CLASS): clsPart = row.getValue(md.RLN_PARTICLE_CLASS) if clsPart != self._lastCls: self._newClass += 1 self._clsDict['%s.%s' % (rLev, clsPart)] = self._newClass self._lastCls = clsPart # write symlink to new Maps relionFn = self._getFileName('relionMap', lev=self._level, iter=self._getnumberOfIters(), ref3d=clsPart, rLev=rLev) newFn = self._getFileName('map', lev=self._level, rLev=self._newClass) print(('link from %s to %s' % (relionFn, newFn))) createLink(relionFn, newFn) row.setValue(md.RLN_PARTICLE_CLASS, self._newClass) row.addToMd(mdInput) print("writing %s and ending the loop" % outStar)
def classifyStep(self): """ Run MSA-CL and MSA-SUM from IMAGIC. """ inputFile = self.inputMSA.get().getParticlesStack() inputFileBase = pwutils.removeExt(inputFile) inputFileImg = inputFileBase + '.img' inputFileHed = inputFileBase + '.hed' pwutils.createLink(inputFileImg, self._getTmpPath("particles.img")) pwutils.createLink(inputFileHed, self._getTmpPath("particles.hed")) inputFn = "tmp/particles" if self.doDownweight.get(): downweight = 'YES' else: downweight = 'NO' self._params.update({ 'particles': inputFn, 'eigs_num': self.numberOfFactors.get(), 'cls_num': self.numberOfClasses.get(), 'perc_ign': self.percentIgnore.get(), 'downweight': downweight, 'perc_ign_bad': self.percentIgnoreBad.get() }) classDir = self._getPath(self.CLASS_DIR) pwutils.cleanPath(classDir) pwutils.makePath(classDir) self.runTemplate('msa/msa-cls.b', self._params)
def importParticles(self): """ Import particles from Frealign. Params: parFile: the filename of the parameter file with the alignment in Frealign. stackFile: single stack file with the images. """ partSet = self.protocol._createSetOfParticles() partSet.setObjComment('Particles imported from Frealign parfile:\n%s' % self.parFile) # Create a local link to the input stack file localStack = self.protocol._getExtraPath( os.path.basename(self.stackFile)) pwutils.createLink(self.stackFile, localStack) # Create a temporary set only with location tmpSet = SetOfParticles(filename=':memory:') tmpSet.readStack(localStack) self._setupSet(tmpSet) # Update both samplingRate and acquisition with parameters # selected in the protocol form self._setupSet(partSet) # Now read the alignment parameters from par file readSetOfParticles(tmpSet, partSet, self.parFile) partSet.setHasCTF(True) # Register the output set of particles self.protocol._defineOutputs(outputParticles=partSet)
def retrieveTrainSets(self): """ Retrieve, link and return a setOfParticles corresponding to the NegativeTrain DeepConsensus trainning set with certain extraction conditions (phaseFlip/invContrast) """ prefixYES = '' prefixNO = 'no' modelType = "negativeTrain_%sPhaseFlip_%sInvert.mrcs" % ( prefixYES if self.doInvert.get() else prefixNO, prefixYES if self.ignoreCTF.get() else prefixNO) modelPath = xmipp3.Plugin.getModel("deepConsensus", modelType) print("Precompiled negative particles found at %s" % (modelPath)) modelFn = self._getTmpPath(modelType) pwutils.createLink(modelPath, modelFn) tmpSqliteSuff = "AddTrain" partSet = self._createSetOfParticles(tmpSqliteSuff) img = SetOfParticles.ITEM_TYPE() imgh = ImageHandler() _, _, _, n = imgh.getDimensions(modelFn) if n > 1: for index in range(1, n + 1): img.cleanObjId() img.setMicId(9999) img.setFileName(modelFn) img.setIndex(index) partSet.append(img) partSet.setAlignment(ALIGN_NONE) cleanPath(self._getPath("particles%s.sqlite" % tmpSqliteSuff)) return partSet
def convertInputStep(self, micsId, refsId): """ This step will take of the convertions from the inputs. Micrographs: they will be linked if are in '.mrc' format, converted otherwise. References: will always be converted to '.mrc' format Mask: either converted ('.tif' format) or generated a circular one """ ih = em.ImageHandler() micDir = self._getTmpPath('micrographs') pwutils.makePath(micDir) for mic in self.getInputMicrographs(): # Create micrograph folder micName = mic.getFileName() # If micrographs are in .mrc format just link it # otherwise convert them outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc')) if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: ih.convert(mic, outMic) refDir = self._getTmpPath('templates') pwutils.makePath(refDir) # We will always convert the templates, since # they can be in an stack and link will not be possible sometimes inputRefs = self.inputReferences.get() for i, ref in enumerate(inputRefs): outRef = join(refDir, 'ref%02d.mrc' % (i+1)) ih.convert(ref, outRef) self.createInputMasks(inputRefs)
def importParticles(self): """ Import particles from Frealign. Params: parFile: the filename of the parameter file with the alignment in Frealing. stackFile: single stack file with the images. """ partSet = self.protocol._createSetOfParticles() partSet.setObjComment("Particles imported from Frealign parfile:\n%s" % self.parFile) # Create a local link to the input stack file localStack = self.protocol._getExtraPath(os.path.basename(self.stackFile)) pwutils.createLink(self.stackFile, localStack) # Create a temporarly set only with location tmpSet = SetOfParticles(filename=":memory:") tmpSet.readStack(localStack) self._setupSet(tmpSet) # Update both samplingRate and acquisition with parameters # selected in the protocol form self._setupSet(partSet) # Now read the alignment parameters from par file readSetOfParticles(tmpSet, partSet, self.parFile) partSet.setHasCTF(True) # Register the output set of particles self.protocol._defineOutputs(outputParticles=partSet)
def exportWorkflow(self): project = self.getProject() workflowProts = [p for p in project.getRuns() ] # workflow prots are all prots if no json provided workflowJsonPath = os.path.join( project.path, self.getTopLevelPath(self.OUTPUT_WORKFLOW)) protDicts = project.getProtocolDicts(workflowProts) for inputSetPointer in self.inputSets: inputSet = inputSetPointer.get() setName = inputSet.getObjName() setParentId = inputSet.getObjParentId() setParentObj = project.getObject(setParentId) protDicts[setParentId]['filesPath'] = os.path.join('.', setName) pwutils.createLink(setParentObj._getExtraPath(), self.getTopLevelPath(setName)) with open(workflowJsonPath, 'w') as f: f.write( json.dumps(list(protDicts.values()), indent=4, separators=(',', ': '))) self.workflowPath.set(workflowJsonPath) return workflowJsonPath
def _initialize(self): # The prediction is expecting the training and validation datasets to be in the same place as the training # model, but they are located in the training data generation extra directory. Hence, a symbolic link will # be created makeDatasetSymLinks(self, self.trainDataDir.get()) createLink(join('..', self.basedir.get()), self._getExtraPath(CRYOCARE_MODEL))
def writeCtfStarStep(self): inputCTF = self.inputCTF.get() if self.micrographSource == 0: # same as CTF estimation ctfMicSet = inputCTF.getMicrographs() else: ctfMicSet = self.inputMicrographs.get() micSet = SetOfMicrographs(filename=':memory:') psd = inputCTF.getFirstItem().getPsdFile() hasPsd = psd and os.path.exists(psd) if hasPsd: psdPath = self._getPath('PSD') pwutils.makePath(psdPath) print "Writing PSD files to %s" % psdPath for ctf in inputCTF: # Get the corresponding micrograph mic = ctfMicSet[ctf.getObjId()] if mic is None: print("Skipping CTF id: %s, it is missing from input " "micrographs. " % ctf.getObjId()) continue micFn = mic.getFileName() if not os.path.exists(micFn): print "Skipping micrograph %s, it does not exists. " % micFn continue mic2 = mic.clone() mic2.setCTF(ctf) if hasPsd: psdFile = ctf.getPsdFile() newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName()) if not os.path.exists(psdFile): print "PSD file %s does not exits" % psdFile print "Skipping micrograph %s" % micFn continue pwutils.copyFile(psdFile, newPsdFile) ctf.setPsdFile(newPsdFile) micSet.append(mic2) starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId()) print "Writing set: %s" % inputCTF print " to: %s" % starFile acq = ctfMicSet.getAcquisition() self.samplingRate = ctfMicSet.getSamplingRate() mag = acq.getMagnification() self.detectorPixelSize = 1e-4 * self.samplingRate * mag writeSetOfMicrographs(micSet, starFile, preprocessImageRow=self.preprocessMicrograph) # Let's create a link from the project roots to facilitate the import # of the star file into a Relion project pwutils.createLink(starFile, os.path.basename(starFile))
def createBinaryLink(fn): """ Just create a link named .mrcs to Relion understand that it is a binary stack file and not a volume. """ newFn = getUniqueFileName(fn, extension) if not os.path.exists(newFn): pwutils.createLink(fn, newFn) print(" %s -> %s" % (newFn, fn)) return newFn
def createLinksStep(self): prot = self._inputProt() prevPartDir = prot._getExtraPath("particles") currPartDir = self._getExtraPath("particles") prevSetsDir = prot._getExtraPath("sets") currSetsDir = self._getExtraPath("sets") createLink(prevPartDir, currPartDir) createLink(prevSetsDir, currSetsDir)
def importMicrographs(self, pattern, suffix, voltage, sphericalAberration, amplitudeContrast): """ Copy images matching the filename pattern Register other parameters. """ filePaths = glob(pwutils.expandPattern(pattern)) # imgSet = SetOfMicrographs(filename=self.micsPairsSqlite, prefix=suffix) imgSet = self._createSetOfMicrographs(suffix=suffix) acquisition = imgSet.getAcquisition() # Setting Acquisition properties acquisition.setVoltage(voltage) acquisition.setSphericalAberration(sphericalAberration) acquisition.setAmplitudeContrast(amplitudeContrast) # Call a function that should be implemented by each subclass self._setOtherPars(imgSet) outFiles = [imgSet.getFileName()] imgh = emlib.image.ImageHandler() img = imgSet.ITEM_TYPE() n = 1 size = len(filePaths) filePaths.sort() for i, fn in enumerate(filePaths): # ext = os.path.splitext(basename(f))[1] dst = self._getExtraPath(basename(fn)) if self.copyToProj: pwutils.copyFile(fn, dst) else: pwutils.createLink(fn, dst) if n > 1: for index in range(1, n + 1): img.cleanObjId() img.setFileName(dst) img.setIndex(index) imgSet.append(img) else: img.cleanObjId() img.setFileName(dst) # Fill the micName if img is a Micrograph. self._fillMicName(img, fn, pattern) imgSet.append(img) outFiles.append(dst) sys.stdout.write("\rImported %d/%d" % (i + 1, size)) sys.stdout.flush() print("\n") imgSet.write() return imgSet
def writeCtfStarStep(self): inputCTF = self.inputCTF.get() if self.micrographSource == 0: # same as CTF estimation ctfMicSet = inputCTF.getMicrographs() else: ctfMicSet = self.inputMicrographs.get() micSet = SetOfMicrographs(filename=':memory:') psd = inputCTF.getFirstItem().getPsdFile() hasPsd = psd and os.path.exists(psd) if hasPsd: psdPath = self._getPath('PSD') pwutils.makePath(psdPath) print "Writing PSD files to %s" % psdPath for ctf in inputCTF: # Get the corresponding micrograph mic = ctfMicSet[ctf.getObjId()] if mic is None: print( "Skipping CTF id: %s, it is missing from input " "micrographs. " % ctf.getObjId()) continue micFn = mic.getFileName() if not os.path.exists(micFn): print "Skipping micrograph %s, it does not exists. " % micFn continue mic2 = mic.clone() mic2.setCTF(ctf) if hasPsd: psdFile = ctf.getPsdFile() newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName()) if not os.path.exists(psdFile): print "PSD file %s does not exits" % psdFile print "Skipping micrograph %s" % micFn continue pwutils.copyFile(psdFile, newPsdFile) ctf.setPsdFile(newPsdFile) micSet.append(mic2) starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId()) print "Writing set: %s" % inputCTF print " to: %s" % starFile writeSetOfMicrographs(micSet, starFile, preprocessImageRow=self.preprocessMicrograph) # Let's create a link from the project roots to facilitate the import # of the star file into a Relion project pwutils.createLink(starFile, os.path.basename(starFile))
def convertImagesStep(self): partSet = self._getInputParticles() partAlign = partSet.getAlignment() storePath = self._getExtraPath("particles") makePath(storePath) writeSetOfParticles(partSet, storePath, alignType=partAlign) if self.useInputBispec and self.inputBispec is not None: print( "Skipping CTF estimation since input bispectra were provided") self.skipctf.set(True) if not self.skipctf: program = eman2.Plugin.getProgram('e2ctf.py') acq = partSet.getAcquisition() args = " --voltage %d" % acq.getVoltage() args += " --cs %f" % acq.getSphericalAberration() args += " --ac %f" % (100 * acq.getAmplitudeContrast()) args += " --threads=%d" % self.numberOfThreads.get() if not partSet.isPhaseFlipped(): args += " --phaseflip" args += " --computesf --apix %f" % partSet.getSamplingRate() args += " --allparticles --autofit --curdefocusfix --storeparm -v 8" self.runJob(program, args, cwd=self._getExtraPath(), numberOfMpi=1, numberOfThreads=1) program = eman2.Plugin.getProgram('e2buildsets.py') args = " --setname=inputSet --allparticles --minhisnr=-1" self.runJob(program, args, cwd=self._getExtraPath(), numberOfMpi=1, numberOfThreads=1) if self.useInputBispec: prot = self.inputBispec.get() prot._createFilenameTemplates() bispec = prot.outputParticles_flip_bispec if not bispec.getSize() == partSet.getSize(): raise Exception( 'Input particles and bispectra sets have different size!') # link bispec hdf files and lst file pattern = prot._getExtraPath('particles/*__ctf_flip_bispec.hdf') print("\nLinking bispectra input files...") for fn in sorted(glob(pattern)): newFn = join(self._getExtraPath('particles'), basename(fn)) createLink(fn, newFn) print(" %s -> %s" % (fn, newFn)) lstFn = prot._getFileName('partSetFlipBispec') newLstFn = self._getFileName('partBispecSet') createLink(lstFn, newLstFn)
def writeSetOfVolumes(volSet, volXml, volDir): """ Convert a SetOfVolumes to a xml file used by PyTom. The volumes will be converted to .mrc format if not are '.em' or '.mrc' Params: volSet: input SetOfVolumes. volXml: filename where to write the xml file. volDir: where to create links or copies (converted to mrc). """ # Add to the path the root to pytom backupPath = list(sys.path) addPyTomPaths() from pytom.basic.structures import Particle, ParticleList, Wedge, SingleTiltWedge from pytom.score.score import Score, PeakPrior, xcfScore from pytom.frm.FRMAlignment import FRMScore w = SingleTiltWedge() #s = PeakPrior() pl = ParticleList() ih = em.convert.ImageHandler() for vol in volSet: index, fn = vol.getLocation() convert = True # by default, convert, which is the save way if index == em.NO_INDEX: # means single volumes volName = os.path.basename(fn) if fn.endswith('.em') or fn.endswith('.mrc'): convert = False # we can just create a link in this case else: volName = 'volume_%03d.mrc' % vol.getObjId() volFn = os.path.join(volDir, volName) if convert: ih.convert(vol, volFn) else: pwutils.createLink(fn, volFn) # Make the volumes names relative to the xml file # where the programs will be executed volRel = os.path.relpath(volFn, os.path.dirname(volXml)) p = Particle() s = xcfScore() s.setValue(1.0) pytomInfo = getattr(vol, 'pytomInfo', None) if pytomInfo is None: p.setWedge(w) else: p.fromXML(pytomInfo.get()) # Get stored XML format from PyTom p.setFilename(volRel) p.setScore(s) pl.append(p) pl.toXMLFile(volXml) #pl.setWedgeAllParticles(w) sys.path = backupPath
def _visualize(self, obj, **args): cls = type(obj) if issubclass(cls, Volume) or issubclass(cls, Image): fn = obj.getFileName() if pwutils.getExt(fn) in ['.stk', '.vol', '.xmp']: # eman2 expects spider files with .spi extension only pwutils.createLink(fn, pwutils.replaceExt(fn, 'spi')) fn = pwutils.replaceExt(fn, 'spi') view = CommandView("%s %s &" % (Plugin.getProgram('e2display.py'), fn), env=Plugin.getEnviron()) return [view] elif isinstance(obj, EmanProtBoxing): coords = obj.getCoords() if coords: return DataViewer._visualize(self, obj.outputCoordinates) elif isinstance(obj, EmanProtInitModel): obj = obj.outputVolumes fn = obj.getFileName() labels = 'id enabled comment _filename ' objCommands = "'%s' '%s' '%s'" % (OBJCMD_CLASSAVG_PROJS, OBJCMD_PROJS, OBJCMD_INITVOL) self._views.append( ObjectView(self._project, obj.strId(), fn, viewParams={ showj.MODE: showj.MODE_MD, showj.VISIBLE: labels, showj.RENDER: '_filename', showj.OBJCMDS: objCommands })) return self._views elif isinstance(obj, EmanProtInitModelSGD): obj = obj.outputVolumes fn = obj.getFileName() labels = 'id enabled comment _filename ' objCommands = "'%s'" % OBJCMD_CLASSAVG_PROJS self._views.append( ObjectView(self._project, obj.strId(), fn, viewParams={ showj.MODE: showj.MODE_MD, showj.VISIBLE: labels, showj.RENDER: '_filename', showj.OBJCMDS: objCommands })) return self._views
def createBsoftInputParticles(imgSet, starFile, stackFile): """ Ensure that 'starFile' is a valid STAR files with particles. If the imgSet comes from Bsoft, just create a link. If not, then write the proper file. """ imgsStar = getattr(imgSet, '_bsoftStar', None) if imgsStar is None: writeSetOfParticles(imgSet, starFile, stackFile) else: imgsFn = imgsStar.get() createLink(imgsFn, imgsStar.get())
def copyOrLinkFileName(imgRow, prefixDir, outputDir, copyFiles=False): index, imgPath = relionToLocation(imgRow.getValue(md.RLN_IMAGE_NAME)) baseName = os.path.basename(imgPath) newName = os.path.join(outputDir, baseName) if not os.path.exists(newName): if copyFiles: pwutils.copyFile(os.path.join(prefixDir, imgPath), newName) else: pwutils.createLink(os.path.join(prefixDir, imgPath), newName) imgRow.setValue(md.RLN_IMAGE_NAME, locationToRelion(index, newName))
def create_resmap_project(request): if request.is_ajax(): import os from pyworkflow.em.protocol import ProtImportVolumes from pyworkflow.em.packages.resmap.protocol_resmap import ProtResMap # Create a new project projectName = request.GET.get(PROJECT_NAME) # Filename to use as test data testDataKey = request.GET.get('testData') manager = getServiceManager(MYRESMAP_SERVICE) writeCustomMenu(manager.protocols) project = manager.createProject(projectName, runsView=1, hostsConf=manager.hosts, protocolsConf=manager.protocols, chdir=False) project.getSettings().setLifeTime(336) # 14 days * 24 hours project.saveSettings() projectPath = manager.getProjectPath(projectName) # 1. Import maps if testDataKey: attr = getAttrTestFile(testDataKey) source = attr['file'] dest = os.path.join(projectPath, 'Uploads', basename(source)) pwutils.createLink(source, dest) label_import = "import volumes (" + testDataKey + ")" protImport = project.newProtocol(ProtImportVolumes, objLabel=label_import) protImport.filesPath.set(dest) protImport.samplingRate.set(attr['samplingRate']) project.launchProtocol(protImport, wait=True, chdir=False) else: protImport = project.newProtocol(ProtImportVolumes, objLabel='import volumes') project.saveProtocol(protImport) # 2. ResMap protResMap = project.newProtocol(ProtResMap) protResMap.setObjLabel('resmap - local resolution') protResMap.inputVolume.set(protImport) protResMap.inputVolume.setExtended('outputVolume') loadProtocolConf(protResMap) project.saveProtocol(protResMap) return HttpResponse(content_type='application/javascript')
def __init__(self, srcPath, dstPath, copyFiles=False): self._src = srcPath self._dst = dstPath self._processed = set() if copyFiles: pwutils.makePath(dstPath) self.transform = self._copyFile else: pwutils.makePath(os.path.dirname(srcPath)) pwutils.createLink(srcPath, dstPath) self.transform = self._addPrefix
def create_resmap_project(request): if request.is_ajax(): import os from pyworkflow.object import Pointer from pyworkflow.em.protocol import ProtImportVolumes from pyworkflow.em.packages.resmap.protocol_resmap import ProtResMap # Create a new project projectName = request.GET.get('projectName') # Filename to use as test data testDataKey = request.GET.get('testData') manager = getServiceManager('myresmap') writeCustomMenu(manager.protocols) project = manager.createProject(projectName, runsView=1, hostsConf=manager.hosts, protocolsConf=manager.protocols ) project.getSettings().setLifeTime(14) project.saveSettings() projectPath = manager.getProjectPath(projectName) # 1. Import movies if testDataKey : attr = getAttrTestFile(testDataKey) source = attr['path'] + attr['file'] dest = os.path.join(projectPath,'Uploads', attr['file']) pwutils.createLink(source, dest) label_import = "import volumes ("+ testDataKey +")" protImport = project.newProtocol(ProtImportVolumes, objLabel=label_import) protImport.filesPath.set(dest) protImport.samplingRate.set(attr['samplingRate']) project.launchProtocol(protImport, wait=True) else: protImport = project.newProtocol(ProtImportVolumes, objLabel='import volumes') project.saveProtocol(protImport) # 2. ResMap protResMap = project.newProtocol(ProtResMap) protResMap.setObjLabel('resmap - local resolution') protResMap.inputVolume.set(protImport) protResMap.inputVolume.setExtendedAttribute('outputVolume') project.saveProtocol(protResMap) return HttpResponse(mimetype='application/javascript')
def _convertInputStep(self): self.info("Relion version:") self.runJob("relion_convert_to_tiff --version", "", numberOfMpi=1) self.info("Detected version from config: %s" % relion.Plugin.getActiveVersion()) # Create a local link to the input gain file if necessary inputGain = self.getInputGain() if inputGain: tmpGain = self._getTmpPath('gain_estimate.bin') pwutils.createLink(inputGain, tmpGain) pwutils.createLink(inputGain.replace('.bin', '_reliablity.bin'), tmpGain.replace('.bin', '_reliablity.bin')) ProtAlignMovies._convertInputStep(self)
def createOutputStep(self): imgSet = self.inputParticles.get() # Let us use extension "vol" for the output vol # use stk creates visualization probrems. vol = Volume() volNameStk = self._getPath('volume.stk') volName = volNameStk.replace(".stk", ".vol") pwutils.createLink(volNameStk, volName) vol.setFileName(volName) vol.setSamplingRate(imgSet.getSamplingRate()) self._defineOutputs(outputVolume=vol) self._defineSourceRelation(self.inputParticles, vol)
def _convertCoords(self, micSet, tmpDir, coordsType): """ Link specified coord set to tmpDir folder and convert it to .pos files""" coordTypes = { 'autopick': 'coordinates.sqlite', 'rejected': 'coordinates_rejected.sqlite' } coordsFnIn = self.protocol._getPath(coordTypes[coordsType]) coordsFnOut = pwutils.join(tmpDir, 'coordinates.sqlite') pwutils.createLink(coordsFnIn, coordsFnOut) coordSet = SetOfCoordinates(filename=coordsFnOut) coordSet.setMicrographs(micSet) from .convert import writeSetOfCoordinatesXmipp writeSetOfCoordinatesXmipp(tmpDir, coordSet, ismanual=False)
def _processMovie(self, movie): fn = movie.getAttributeValue('_originalFileName', movie.getFileName()) baseName = os.path.basename(fn) compression = self.getEnumText('compression') pwutils.createLink(fn, self._getTmpPath(baseName)) args = "--i %s --o ../extra/ " % baseName args += "--compression %s " % compression # TODO: Check if deflateLevel is only valid for zip (deflate) if compression == 'zip': # deflate args += "--deflate_level %d" % self.deflateLevel if self.getInputGain(): args += "--gain gain_estimate.bin " self.runJob('relion_convert_to_tiff', args, cwd=self._getTmpPath())
def runGautomatch(micName, refStack, workDir, args, env=None): # We convert the input micrograph on demand if not in .mrc outMic = os.path.join(workDir, pwutils.replaceBaseExt(micName, 'mrc')) if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: em.ImageHandler().convert(micName, outMic) if refStack is not None: args = ' %s --T %s %s' % (outMic, refStack, args) else: args = ' %s %s' % (outMic, args) pwutils.runJob(None, getProgram(), args, env=env) # After picking we can remove the temporary file. pwutils.cleanPath(outMic)
def _preprocessMicrographRow(self, img, imgRow): # Temporarly convert the few micrographs to tmp and make sure # they are in 'mrc' format # Get basename and replace extension by 'mrc' newName = pwutils.replaceBaseExt(img.getFileName(), 'mrc') newPath = self._getExtraPath(newName) # If the micrographs are in 'mrc' format just create a link # if not, convert to 'mrc' if img.getFileName().endswith('mrc'): pwutils.createLink(img.getFileName(), newPath) else: self._ih.convert(img, newPath) # The command will be launched from the working dir # so, let's make the micrograph path relative to that img.setFileName(os.path.join('extra', newName))
def runGempicker(micName, workingDir, useGPU, args): # We convert the input micrograph on demand if not in .mrc outMic = os.path.join(workingDir, pwutils.replaceBaseExt(micName, 'mrc')) if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: em.ImageHandler().convert(micName, outMic) refDir = join(workingDir, 'templates') maskSchDir = join(workingDir, 'maskSch') args += ' --dirTgt=%s --dirSch=%s --dirMskSch=%s ' % (workingDir, refDir, maskSchDir) # Run Gempicker: for mode in [0, 1]: pwutils.runJob(None, getProgram(useGPU), args + ' --mode=%d' % mode) # After picking we can remove the temporary file. pwutils.cleanPath(outMic)
def denoisingStep(self): input_mics = self.inputMicrographs.get() # Create links to the movies desired to denoise in tmp folder (subset case, janni only accepts directories # and work with all the files contained in them) for mic in input_mics: micName = mic.getFileName() createLink(mic.getFileName(), self._getTmpPath(basename(micName))) args = "denoise {}/ {}/ {}".format(self._getTmpPath(), self._getTmpPath(), self.getInputModel()) Plugin.runCryolo(self, 'janni_denoise.py', args) # Move the resulting denoised files to the extra folder [ moveFile(file, self._getExtraPath(basename(file))) for file in glob.glob(self._getTmpPath(join('tmp', '*.mrc'))) ]
def runGempicker(micName, workingDir, useGPU, args, log=None): # We convert the input micrograph on demand if not in .mrc outMic = os.path.join(workingDir, pwutils.replaceBaseExt(micName, 'mrc')) if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: em.ImageHandler().convert(micName, outMic) refDir = join(workingDir, 'templates') maskSchDir = join(workingDir, 'maskSch') args += ' --dirTgt=%s --dirSch=%s --dirMskSch=%s ' % (workingDir, refDir, maskSchDir) # Run Gempicker: for mode in [0, 1]: pwutils.runJob(log, getProgram(useGPU), args + ' --mode=%d' % mode, env=getEnviron()) # After picking we can remove the temporary file. pwutils.cleanPath(outMic)
def copyOrLinkBinary(self, imgRow, label, basePath, destBasePath, copyFiles=False): index, imgPath = relionToLocation(imgRow.get(label)) baseName = os.path.join(os.path.dirname(imgPath), os.path.basename(imgPath)) os.makedirs(os.path.join(destBasePath, os.path.dirname(imgPath)), exist_ok=True) newName = os.path.join(destBasePath, baseName) if not os.path.exists(newName): if copyFiles: pwutils.copyFile(os.path.join(basePath, imgPath), newName) else: pwutils.createLink(os.path.join(basePath, imgPath), newName) imgRow.set(label, locationToRelion(index, newName))
def _getConvertFunc(self, img): """ Return a function that will link or convert the input file depending if the input img has a format valid for Relion. """ srcFile = img.getFileName() if srcFile.endswith('mrcs'): return lambda s, d: pwutils.createLink(s, d) else: return lambda s, d: self._ih.convert(s, d)
def _pickMicrograph(self, mic, args): # Prepare mic folder and convert if needed micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) ih = ImageHandler() # If needed convert micrograph to mrc format, otherwise link it if pwutils.getExt(micName) != ".mrc": fnMicBase = pwutils.replaceBaseExt(micName, 'mrc') inputMic = os.path.join(micDir, fnMicBase) ih.convert(mic.getLocation(), inputMic) else: inputMic = os.path.join(micDir, os.path.basename(micName)) pwutils.createLink(micName, inputMic) # Program to execute and it arguments program = "ApDogPicker.py" outputFile = self._getExtraPath(pwutils.replaceBaseExt(inputMic, "txt")) args += " --image=%s --outfile=%s" % (inputMic, outputFile) self.runJob(program, args)
def _createLink(self, movie): movieFn = movie.getFileName() if movieFn.endswith("mrcs"): pwutils.createLink(movieFn, self._getMovieFn(movie))
def create_movies_project(request): if request.is_ajax(): import os from pyworkflow.object import Pointer from pyworkflow.em.protocol import ProtImportMovies from pyworkflow.em.packages.xmipp3 import ProtMovieAlignment # Create a new project projectName = request.GET.get('projectName') # Filename to use as test data testDataKey = request.GET.get('testData') manager = getServiceManager('movies') writeCustomMenu(manager.protocols) project = manager.createProject(projectName, runsView=1, hostsConf=manager.hosts, protocolsConf=manager.protocols ) project.getSettings().setLifeTime(14) project.saveSettings() # copyFile(customMenu, project.getPath('.config', 'protocols.conf')) # Create symbolic link for uploads projectPath = manager.getProjectPath(projectName) dest = os.path.join(projectPath,'Uploads') # @todo: this path to uploads dir should be configurable outside the code... source = "/services/scipion/data/uploads/"+ projectName pwutils.path.makePath(source) pwutils.createLink(source, dest) # 1. Import movies if testDataKey : attr = getAttrTestFile(testDataKey) path_test = attr['path'] for f in os.listdir(path_test): # Create a symbolic link for each file file_path = os.path.join(path_test, f) source_file = os.path.join(source, f) pwutils.createAbsLink(file_path, source_file) label_import = "import movies ("+ testDataKey +")" protImport = project.newProtocol(ProtImportMovies, objLabel=label_import) protImport.filesPath.set(attr["filesPath"]) protImport.voltage.set(attr['voltage']) protImport.sphericalAberration.set(attr['sphericalAberration']) protImport.amplitudeContrast.set(attr['amplitudeContrast']) protImport.magnification.set(attr['magnification']) protImport.samplingRate.set(attr['samplingRate']) project.launchProtocol(protImport, wait=True) else: protImport = project.newProtocol(ProtImportMovies, objLabel='import movies') project.saveProtocol(protImport) # 2. Movie Alignment protMovAlign = project.newProtocol(ProtMovieAlignment) protMovAlign.setObjLabel('xmipp - movie alignment') protMovAlign.inputMovies.set(protImport) protMovAlign.inputMovies.setExtended('outputMovies') project.saveProtocol(protMovAlign) return HttpResponse(mimetype='application/javascript')
def convertInputStep(self): inputParticles = self.inputParticles.get() firstCoord = inputParticles.getFirstItem().getCoordinate() self.hasMicName = firstCoord.getMicName() is not None inputMics = self._getMicrographs() self.alignType = inputParticles.getAlignment() self.downFactor = self.ctfDownFactor.get() # create a tmp set for matching mics self.matchingMics = self._createSetOfMicrographs(suffix='_tmp') self.matchingMics.copyInfo(inputMics) if self.downFactor != 1.: self.matchingMics.setDownsample(self.downFactor) # create a tmp set for coords coords = self._createSetOfCoordinates(inputMics, suffix='_tmp') newCoord = Coordinate() self.scale = inputParticles.getSamplingRate() / inputMics.getSamplingRate() if self.scale != 1.0: print "Scaling coordinates by a factor *%0.2f*" % self.scale # Create the micrograph dicts micDict = {} # dict with micName or micId micBaseDict = {} # dict with micName (just basename) micKey2 = None insertedMics = {} for mic in inputMics: if self.hasMicName: micKey = mic.getMicName() micKey2 = pwutils.removeBaseExt(micKey) else: micKey = mic.getObjId() if micKey in micDict: print ">>> ERROR: micrograph key %s is duplicated!" % micKey print " Used in micrographs:" print " - %s" % micDict[micKey].getLocation() print " - %s" % mic.getLocation() raise Exception("Micrograph key %s is duplicated!" % micKey) micDict[micKey] = mic.clone() if self.hasMicName: micBaseDict[micKey2] = mic.clone() # match the mic from coord with micDict for particle in inputParticles: coord = particle.getCoordinate() or None if coord is None: print "Skipping particle, coordinates not found" continue if self.hasMicName: micKey = coord.getMicName() micKey2 = pwutils.removeBaseExt(micKey) else: micKey = coord.getMicId() # find the mapping by micName (with or without ext) or micId mic = micDict.get(micKey, None) or micBaseDict.get(micKey2, None) if mic is None: print "Skipping particle, key %s not found" % micKey else: newCoord.copyObjId(particle) x, y = coord.getPosition() if self.applyShifts: shifts = getShifts(particle.getTransform(), self.alignType) xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1]) newCoord.setPosition(xCoor * self.scale, yCoor * self.scale) else: newCoord.setPosition(x * self.scale, y * self.scale) newCoord.setMicrograph(mic) coords.append(newCoord) if mic.getObjId() not in insertedMics: insertedMics[mic.getObjId()] = mic self.matchingMics.append(mic) ih = em.ImageHandler() # We convert matching micrographs if they are not *.mrc for mic in self.matchingMics: # Create micrograph dir micName = mic.getFileName() micDir = self._getTmpPath(pwutils.removeBaseExt(micName)) pwutils.makePath(micDir) outMic = pwutils.join(micDir, pwutils.replaceBaseExt(micName, 'mrc')) if self.downFactor != 1.: ih.scaleFourier(micName, outMic, self.downFactor) sps = inputMics.getScannedPixelSize() * self.downFactor self._params['scannedPixelSize'] = sps else: if micName.endswith('.mrc'): pwutils.createLink(micName, outMic) else: ih.convert(micName, outMic) # Write out coordinate files and sets writeSetOfCoordinates(self._getTmpPath(), coords, self.matchingMics) coords.clear() pwutils.cleanPath(coords.getFileName()) self.matchingMics.write() self.matchingMics.close()