def getBoxSize(self, coordFile): """ Try to infer the box size from the given coordinate file. In the case of .box files, the size is the 3rd column In the case of .json files, we will look for file e2boxercache/base.json """ if coordFile.endswith('.box'): md = MetaData() md.readPlain(coordFile, "xcoor ycoor particleSize") return md.getValue(MDL_PICKING_PARTICLE_SIZE, md.firstObject()) elif coordFile.endswith('.json'): infoDir = os.path.dirname(coordFile) # Still go one level up of info dir jsonBase = pwutils.join(os.path.dirname(infoDir), 'e2boxercache', 'base.json') jsonBase2 = pwutils.join(infoDir, 'project.json') if pwutils.exists(jsonBase): jsonDict = loadJson(jsonBase) if 'box_size' in jsonDict: return int(jsonDict["box_size"]) elif pwutils.exists(jsonBase2): jsonDict = loadJson(jsonBase2) if 'global.boxsize' in jsonDict: return int(jsonDict["global.boxsize"]) return None
def readSetOfCoordinates(outputDir, micSet, coordSet): """ Read from Bsoft .star files. Params: outputDir: the directory where the .star files are. micSet: the SetOfMicrographs to associate the .star, which name should be the same of the micrographs. coordSet: the SetOfCoordinates that will be populated. """ boxSize = 100 for mic in micSet: outputFile = join(outputDir, replaceBaseExt(mic.getFileName(), 'star')) if exists(outputFile): posMd = md.MetaData(outputFile) for objId in posMd: coordRow = rowFromMd(posMd, objId) coord = rowToCoordinate(coordRow) boxSize = 2 * float(coordRow.getValue("particle.x_origin", 50)) coord.setMicrograph(mic) coord.setX(coord.getX()) coord.setY(coord.getY()) coordSet.append(coord) # Add an unique ID that will be propagated to particles # posMd.setValue(md.MDL_PARTICLE_ID, long(coord.getObjId()), objId) # reading origin.x value and converting to particle # size, can change, we take last value coordSet.setBoxSize(boxSize)
def createOutputStep(self): # Check that the indexing created proper output assert (pwutils.exists(self.getOutputReflFile())) assert (pwutils.exists(self.getOutputModelFile())) outputSet = self._createSetOfIndexedSpots() outputSet.setDialsModel(self.getOutputModelFile()) outputSet.setDialsRefl(self.getOutputReflFile()) try: outputSet.setDialsHtml(self.getOutputHtmlFile()) except FileNotFoundError: self.info(self.getError()) outputSet.write() self._defineOutputs(outputScaledSpots=outputSet)
def _openHTML(self, e=None): reportHtml = self.protocol.createHtmlReport() reportPath = reportHtml.reportPath if pwutils.exists(reportPath): text._open_cmd(reportPath) else: self.showInfo('Your html file is not ready yet. Please try again in a minute.')
def createMetadataImageStep(self): mdUntilted = md.MetaData() mdTilted = md.MetaData() # for objId in mdPairs: for uMic, tMic in izip(self.uMics, self.tMics): umicName = pwutils.removeBaseExt(uMic.getFileName()) fnMicU = self._getExtraPath(umicName + ".xmd") fnPosU = self._getExtraPath(umicName + ".pos") # Check if there are picked particles in these micrographs if pwutils.exists(fnMicU): mdMicU = md.MetaData(fnMicU) mdPosU = md.MetaData('particles@%s' % fnPosU) mdPosU.merge(mdMicU) mdUntilted.unionAll(mdPosU) tmicName = pwutils.removeBaseExt(tMic.getFileName()) fnMicT = self._getExtraPath(tmicName + ".xmd") fnPosT = self._getExtraPath(tmicName + ".pos") mdMicT = md.MetaData(fnMicT) mdPosT = md.MetaData('particles@%s' % fnPosT) mdPosT.merge(mdMicT) mdTilted.unionAll(mdPosT) # Write image metadata (check if it is really necessary) fnTilted = self._getExtraPath("images_tilted.xmd") fnUntilted = self._getExtraPath("images_untilted.xmd") mdUntilted.write(fnUntilted) mdTilted.write(fnTilted)
def readCTFModel(ctfModel, filename): jsonDict = loadJson(filename) keyPos = None ctfPhaseShift = 0.0 if jsonDict.has_key('ctf_frame'): keyPos = jsonDict['ctf_frame'][1] elif jsonDict.has_key('ctf'): keyPos = jsonDict['ctf'][0] else: setWrongDefocus(ctfModel) if keyPos: defocus = float(keyPos['defocus']) defocusAngle = float(keyPos['dfang']) dfdiff = float(keyPos['dfdiff']) ampcont = float(keyPos['ampcont']) defocusU = 10000.0 * defocus + 5000.0 * dfdiff defocusV = 20000.0 * defocus - defocusU ctfPhaseShift = calculatePhaseShift(ampcont) ctfModel.setStandardDefocus(defocusU, defocusV, defocusAngle) if jsonDict.has_key('ctf_im2d'): # psdFile = jsonDict['ctf_im2d']['__image__'][0] fnBase = pwutils.removeExt(filename) + '_jsonimg' psdFile = "1@%s.hdf" % fnBase if pwutils.exists(psdFile): ctfModel.setPsdFile(psdFile) ctfModel.setPhaseShift(float(ctfPhaseShift))
def __getRemoteFile(self, sourceFilePath, targetFilePath, gatewayHosts, sftp): """ Send local file to remote machine sourceFilePath -- Source file path (/file path/...). targetFilePath -- Target file path (/file path/...). sftp -- sftp connection. """ log.info("Getting " + sourceFilePath + " to " + targetFilePath) # Check if file already existsFilePath and it is up to date existsFilePath = False if exists(targetFilePath): if self.__getRemoteSHA1( sourceFilePath, self.ssh) == self.__getLocalSHA1(targetFilePath): existsFilePath = True log.info(targetFilePath + " already existed") if not existsFilePath: makeFilePath(targetFilePath) try: sftp.get(sourceFilePath, targetFilePath) except IOError as err: log.error("Fail getting remote file " + sourceFilePath + " to local file " + targetFilePath + " - " + str(err)) raise
def _getValues(self): fileName = self.protocol._getFileName('outputAngles') planethres = self.planethres.get() r = [] theta = [] datap = [] zaxis = [] if pwutils.exists(fileName): jsonPosDict = loadJson(fileName) if jsonPosDict.has_key("particletilt_list"): tiltpairs = jsonPosDict["particletilt_list"] maxcolorval = max(tiltpairs, key=lambda x: x[3])[3] for tp in tiltpairs: if tp[3] > planethres: continue datap.append(tp[0]) r.append(tp[1]) theta.append(math.radians(tp[2])) # Color the Z axis out of planeness zaxis.append(self._computeRGBcolor(tp[3], 0, maxcolorval)) return datap, r, theta, zaxis
def _visualize(self, obj, **args): if hasattr(obj, 'outputVolumes'): XmippViewer._visualize(self,self.protocol.outputVolumes) fnBasis=self.protocol._getExtraPath('split_pc1.vol') if exists(fnBasis): self._views.append(DataView(fnBasis)) return self._views
def _openHTML(self, e=None): reportHtml = self.protocol.createHtmlReport() reportPath = reportHtml.reportPath if pwutils.exists(reportPath): text._open_cmd(reportPath) else: self.showInfo( 'Your html file is not ready yet. Please try again in a minute.' )
def _summary(self): summary = [] fname = os.path.abspath( join(self._getPath('emxData'), self.outputPrefix.get() + '.emx')) if exists(fname): summary.append('Exported %s to %s' % (self._exportTypes[self.inputType.get()], fname)) return summary
def _showHtmlReport(self, paramName=None): reportPath = self.protocol._getFileName('reportHtml', run=self.protocol._getRun()) if pwutils.exists(reportPath): text._open_cmd(reportPath, self.getTkRoot()) else: self.showInfo( 'Your html report is not ready yet. Please try again in a minute.' )
def _getHtml(self): try: reportPath = self.protocol.getOutputHtmlFile() except AttributeError: return None if pwutils.exists(reportPath): return reportPath else: return None
def _summary(self): summary = [] if exists(self._getExtraPath('histogram.png')): results = self._parseOutput() summary.append('Mean resolution: %0.2f A' % results[0]) summary.append('Median resolution: %0.2f A' % results[1]) else: summary.append("Output is not ready yet.") return summary
def _summary(self): """ Should be overwritten in subclasses to return summary message for NORMAL EXECUTION. """ summary = [] postStarFn = self._getExtraPath("postprocess.star") if exists(postStarFn): mdResol = md.RowMetaData(postStarFn) resol = mdResol.getValue(md.RLN_POSTPROCESS_FINAL_RESOLUTION) summary.append("Final resolution: *%0.2f A*" % resol) return summary
def readSetOfParticles(lstFile, partSet, copyOrLink, direc): for index, fn in iterLstFile(lstFile): item = Particle() # set full path to particles stack file abspath = os.path.abspath(lstFile) fn = abspath.replace('sets/%s' % os.path.basename(lstFile), '') + fn newFn = pwutils.join(direc, os.path.basename(fn)) if not pwutils.exists(newFn): copyOrLink(fn, newFn) item.setLocation(index, newFn) partSet.append(item)
def readCoordinates(mic, fileName, coordsSet): if exists(fileName): md = emlib.MetaData() md.readPlain(fileName, 'xcoor ycoor') for objId in md: x = md.getValue(emlib.MDL_XCOOR, objId) y = md.getValue(emlib.MDL_YCOOR, objId) coord = Coordinate() coord.setPosition(x, y) coord.setMicrograph(mic) coordsSet.append(coord)
def _summary(self): summary = [] self._createFilenameTemplates() if exists(self._getFileName('outResmapVol')): results = self._parseOutput() summary.append('Mean resolution: %0.2f A' % results[0]) summary.append('Median resolution: %0.2f A' % results[1]) else: summary.append("Output is not ready yet.") return summary
def testStepsNoGPU(self): xmipp3Protocols = Domain.importFromPlugin('xmipp3.protocols', doRaise=True) protXmippPreproc = self.newProtocol(xmipp3Protocols.XmippProtPreprocessParticles, doNormalize=True, doRemoveDust=True) protXmippPreproc.inputParticles.set(self.protImport.outputParticles) protXmippPreproc.setObjLabel("Xmipp preprocess steps") protXmippPreproc._useQueue.set(True) QUEUE_PARAMS[1]['QUEUE_FOR_JOBS'] = 'Y' protXmippPreproc._queueParams.set(json.dumps(QUEUE_PARAMS)) # Launch protocol but wait until it finishes self.launchProtocol(protXmippPreproc, wait=True) # Check that job files have been created jobFilesPath = join(pwutils.getParentFolder(protXmippPreproc.getLogPaths()[0]), str(protXmippPreproc.getObjId())) self.assertTrue(pwutils.exists(jobFilesPath + "-0-1.out") and pwutils.exists(jobFilesPath + "-0-1.err") and pwutils.exists(jobFilesPath + "-0-1.job") and pwutils.exists(jobFilesPath + "-0-2.out") and pwutils.exists(jobFilesPath + "-0-2.err") and pwutils.exists(jobFilesPath + "-0-2.job"), "Job queue files not found on log folder, job did " "not make it to the queue.") # Check that results have been produced self.assertIsNotNone(protXmippPreproc.outputParticles, "There was a problem with Xmipp preprocess particles.")
def _validate(self): errors = [] # Check that the program exists if not pwutils.exists(self._getProgram()): errors.append("Binary '%s' does not exits.\n" "Check configuration file: \n" "~/.config/scipion/scipion.conf\n" "and set GCTF variables properly." % self._getProgram()) if self.useInputCtf and not self.ctfRelations.get(): errors.append("Please provide input CTFs for refinement.") return errors
def getProgram(cls, program): """ Return the program binary that will be used. """ if program == CTFFIND4_BIN: # if CTFFIND4_HOME is found, use it path = cls.getVar(CTFFIND4_HOME) if pwutils.exists(path): binary = os.path.join(path, 'bin', program) else: binary = os.path.join(cls.getHome(), program) else: binary = os.path.join(cls.getHome(), program) return binary
def _showPlot(self, paramName=None): views = [] color = self.colozaxis rmax = self.radcut.get() if self.displayPlot == TILT_SCATTER: views.append(self._createScatterPlot(rmax, colorzaxis=color)) elif self.displayPlot == TILT_CONTOUR: plotFn = self.protocol._getFileName('outputContourPlot') if pwutils.exists(plotFn): views.append(DataView(plotFn)) else: raise Exception("Contour plot file not found: %s" % plotFn) return views
def __copyLocalFile(self, sourceFilePath, targetFilePath): """ Send local file to remote machine sourceFilePath -- Source file path (/file path/...). targetFilePath -- Target file path (/file path/...). """ log.info("Copying " + sourceFilePath + " to " + targetFilePath) # Check if file already existsFilePath and it is up to date existsFilePath = False if exists(targetFilePath): if self.__getLocalSHA1(sourceFilePath) == self.__getLocalSHA1( targetFilePath): existsFilePath = True log.info(targetFilePath + " already existed") if not existsFilePath: makeFilePath(targetFilePath) shutil.copy2(sourceFilePath, targetFilePath)
def readCoordinates(mic, fn, coordsSet): """ Parse coords file and populate coordsSet. :param mic: input micrograph object :param fn: input file to parse :param coordsSet: output set of coords """ if exists(fn): with open(fn, 'r') as f: for line in f: values = line.strip().split() # plt coords are in Imagic style x = float(values[1]) y = float(mic.getYDim() - float(values[0])) coord = Coordinate() coord.setPosition(x, y) coord.setMicrograph(mic) coordsSet.append(coord)
def readCoordinates(mic, fileName, coordsSet, invertY=False): if pwutils.exists(fileName): jsonPosDict = loadJson(fileName) if jsonPosDict.has_key("boxes"): boxes = jsonPosDict["boxes"] for box in boxes: x, y = box[:2] if invertY: y = mic.getYDim() - y coord = Coordinate() coord.setPosition(x, y) coord.setMicrograph(mic) coordsSet.append(coord)
def importCoordinates3D(self, fileName, addCoordinate): from tomo.objects import Coordinate3D if pwutils.exists(fileName): ext = pwutils.getExt(fileName) if ext == ".txt": md = MetaData() md.readPlain(fileName, "xcoor ycoor zcoor") for objId in md: x = md.getValue(MDL_XCOOR, objId) y = md.getValue(MDL_YCOOR, objId) z = md.getValue(MDL_ZCOOR, objId) coord = Coordinate3D() addCoordinate(coord, x, y, z) else: raise Exception( 'Unknown extension "%s" to import Eman coordinates' % ext)
def _getIterClasses(self, it, clean=False): """ Return a classes .sqlite file for this iteration. If the file doesn't exists, it will be created by converting from this iteration mltomo_it??????_img.xmd file. """ data_classes = self._getFileName('classes_scipion', iter=it) if clean: pwutils.cleanPath(data_classes) if not pwutils.exists(data_classes): clsSet = SetOfClassesVol(filename=data_classes) clsSet.setImages(self.inputVols.get()) self._fillClassesFromIter(clsSet, it) clsSet.write() clsSet.close() return data_classes
def _validate(self): """ Should be overwritten in subclasses to return summary message for NORMAL EXECUTION. """ errors = [] mtfFile = self.mtf.get() if mtfFile and not exists(mtfFile): errors.append("Missing MTF-file '%s'" % mtfFile) protClassName = self.protRefine.get().getClassName() if protClassName.startswith('SpiderProtRefinement'): errors.append("Relion local resolution protocol is not " "implemented for Spider - refinement.") if protClassName.startswith('XmippProtReconstructHighRes'): errors.append("Relion local resolution protocol is not " "implemented for Xmipp - highres.") return errors
def importAngles(self, fileName, addAngles): if pwutils.exists(fileName): ext = pwutils.getExt(fileName) if ext == ".json": fnBase = pwutils.replaceBaseExt(fileName, 'hdf') keyName = 'tiltparams_micrographs/' + fnBase.replace( '_info', '') jsonAngDict = loadJson(fileName) if keyName in jsonAngDict: angles = jsonAngDict[keyName] tilt, y2, y = angles[:3] # y2=tilted, y=gamma(untilted) ang = Angles() # TODO: check this conversion ang.setAngles(y, y2, tilt) addAngles(ang) else: raise Exception( 'Unknown extension "%s" to import Eman tilt pair angles' % ext)
def importCoordinates3D(self, fileName, addCoordinate): from tomo.objects import Coordinate3D if pwutils.exists(fileName): ext = pwutils.getExt(fileName) if ext == ".json": jsonPosDict = loadJson(fileName) boxes = [] if "boxes_3d" in jsonPosDict: boxes = jsonPosDict["boxes_3d"] if boxes: for box in boxes: x, y, z = box[:3] coord = Coordinate3D() addCoordinate(coord, x, y, z) else: raise Exception( 'Unknown extension "%s" to import Eman coordinates' % ext)
def test_movies(self): print(magentaStr("\n==> Importing data - movies:")) protImport = self.runImportMovies() outputMovies = getattr(protImport, 'outputMovies', None) self.assertIsNotNone(outputMovies) inputMovies = protImport.outputMovies print(magentaStr("\n==> Testing cistem - unblur:")) prot = self.newProtocol(CistemProtUnblur, alignFrame0=2, alignFrameN=6) prot.inputMovies.set(inputMovies) self.launchProtocol(prot) outputMics = getattr(prot, 'outputMicrographsDoseWeighted', None) self.assertIsNotNone(outputMics) self.assertEqual(protImport.outputMovies.getSize(), outputMics.getSize()) for mic in outputMics: micFn = mic.getFileName() self.assertTrue(exists(self.proj.getPath(micFn)))
def createCtfModelStep(self): inputSet = self.inputParticles.get() partSet = self._createSetOfParticles() partSet.copyInfo(inputSet) for particle in inputSet: coord = particle.getCoordinate() if coord is None: continue x, y = coord.getPosition() if self.applyShifts: shifts = getShifts(particle.getTransform(), self.alignType) xCoor, yCoor = x - int(shifts[0]), y - int(shifts[1]) xNew, yNew = (xCoor * self.scale, yCoor * self.scale) else: xNew, yNew = (x * self.scale, y * self.scale) micBase = pwutils.removeBaseExt(coord.getMicName()) for key in self.matchingMics: micKey = pwutils.removeBaseExt(key.getFileName()) if micBase in micKey: # micName from mic and micName from coord may be different ctfFn = pwutils.join(self._getExtraPath(micKey), micKey + '_local.star') if pwutils.exists(ctfFn): mdFn = md.MetaData(ctfFn) for row in md.iterRows(mdFn): coordX = row.getValue(md.RLN_IMAGE_COORD_X) coordY = row.getValue(md.RLN_IMAGE_COORD_Y) if (int(xNew), int(yNew)) == (coordX, coordY): newPart = particle.clone() rowToCtfModel(row, newPart.getCTF()) partSet.append(newPart) self._defineOutputs(outputParticles=partSet) self._defineTransformRelation(inputSet, partSet)