def __init__(self, **kwargs): EMObject.__init__(self, **kwargs) self.grid = String() self.gridSquare = String() self.hole = String() self.x = Float() self.y = Float()
def updateItemMaxCC(self, item, row): from convert import locationToXmipp # ToDo: uncomment this lines when the output metadata has ITEM_ID # if item.getObjId() != row.getValue(xmipp.MDL_ITEM_ID): # raise Exception("The objId is not equal to ITEM_ID. Please, sort the metadata.") if isinstance(item, Class2D): img = item.getRepresentative() index, fn = img.getLocation() else: index, fn = item.getLocation() # objLoc = locationToXmipp(index, fn) # mdLoc = row.getValue(md.MDL_IMAGE) # if objLoc != mdLoc: # raise Exception("The image isn't the same. Please, sort the metadata.") if item.getObjId() != row.getValue(md.MDL_ITEM_ID): raise Exception( "The image isn't the same. Please, sort the metadata.") item._xmipp_imageRef = String(row.getValue(md.MDL_IMAGE_REF)) item._xmipp_image = String(row.getValue(md.MDL_IMAGE)) item._xmipp_imageResidual = String(row.getValue(md.MDL_IMAGE_RESIDUAL)) item._xmipp_maxCC = Float(row.getValue(md.MDL_MAXCC)) item._xmipp_cost = Float(row.getValue(md.MDL_COST)) if isinstance(item, Class2D): particle = item.getRepresentative() else: particle = item
def _setPsdFiles(self, ctfModel, micDir): ctfModel._psdFile = String(self._getFileName('psd', micDir=micDir)) ctfModel._xmipp_enhanced_psd = \ String(self._getFileName('enhanced_psd', micDir=micDir)) ctfModel._xmipp_ctfmodel_quadrant = \ String(self._getFileName('ctfmodel_quadrant', micDir=micDir)) ctfModel._xmipp_ctfmodel_halfplane = \ String(self._getFileName('ctfmodel_halfplane', micDir=micDir))
def __init__(self, **kwargs): OrderedObject.__init__(self, **kwargs) self.label = String(kwargs.get('label', None)) self.hostName = String(kwargs.get('hostName', None)) self.userName = String() self.password = String() self.hostPath = String() self.mpiCommand = String() self.scipionHome = String() self.scipionConfig = String() self.address = String() self.queueSystem = QueueSystemConfig()
def _loadInputCoords(self, micDict): """ Load coordinates from the input streaming. """ coordsFn = self.getCoords().getFileName() self.debug("Loading input db: %s" % coordsFn) coordSet = SetOfCoordinates(filename=coordsFn) # FIXME: Temporary to avoid loadAllPropertiesFail coordSet._xmippMd = String() coordSet.loadAllProperties() for micKey, mic in micDict.iteritems(): micId = mic.getObjId() coordList = [] self.debug("Loading coords for mic: %s (%s)" % (micId, micKey)) for coord in coordSet.iterItems(where='_micId=%s' % micId): # TODO: Check performance penalty of using this clone coordList.append(coord.clone()) self.debug(" Coords found: %s" % len(coordList)) if coordList: self.coordDict[micId] = coordList else: del micDict[micKey] self.coordsClosed = coordSet.isStreamClosed() coordSet.close() self.debug("Coords are closed? %s" % self.coordsClosed) self.debug("Closed db.") return micDict
def readSetOfCoordinates(outputDir, partSet, coordSet): """This method is used to store an edition of the coordinates obtained by localized reconstruction. Read from Xmipp .pos files. Params: outputDir: the directory where the .pos files are. It is also expected a file named: config.xmd in this directory where the box size can be read. micSet: the SetOfMicrographs to associate the .pos, which name should be the same of the micrographs. coordSet: the SetOfCoordinates that will be populated. """ # Read the boxSize from the config.xmd metadata configfile = join(outputDir, 'config.xmd') if exists(configfile): mdFn = md.MetaData('properties@' + join(outputDir, 'config.xmd')) boxSize = mdFn.getValue(md.MDL_PICKING_PARTICLE_SIZE, mdFn.firstObject()) coordSet.setBoxSize(boxSize) for part in partSet: baseFn = "%06d_at_%s" % (part.getIndex(), replaceBaseExt(part.getFileName(), 'pos')) posFile = join(outputDir, baseFn) readCoordinates(part, posFile, coordSet, outputDir) coordSet._xmippMd = String(outputDir)
def createChimeraScript(self, volume, pdb): """ Create a chimera script to visualize a pseudoatoms pdb obteined from a given EM 3d volume. A property will be set in the pdb object to store the location of the script. """ pseudoatoms = pdb.getFileName() scriptFile = pseudoatoms + '_chimera.cmd' pdb._chimeraScript = String(scriptFile) sampling = volume.getSamplingRate() radius = sampling * self.pseudoAtomRadius.get() fnIn = volume.getFileName() localInputFn = self._getBasePath(fnIn) createLink(fnIn, localInputFn) fhCmd = open(scriptFile, 'w') fhCmd.write("open %s\n" % basename(pseudoatoms)) fhCmd.write("rangecol bfactor,a 0 white 1 red\n") fhCmd.write("setattr a radius %f\n" % radius) fhCmd.write("represent sphere\n") fhCmd.write("open %s\n" % basename(localInputFn)) threshold = 0.01 if self.maskMode == NMA_MASK_THRE: self.maskThreshold.get() xdim = volume.getDim()[0] origin = xdim / 2 fhCmd.write( "volume #1 level %f transparency 0.5 voxelSize %f originIndex %d\n" % (threshold, sampling, origin)) fhCmd.close()
def _methods(self): methods = [] if not hasattr(self, 'outputVolumes'): methods.append("Output volumes not ready yet.") else: inputIsString = '' if isinstance(self.inputParticles.get(), SetOfParticles): methods.append( 'Set of %d particles %s was employed to create an initial volume using RCT method.' % (len(self.inputParticles.get()), self.getObjectTag('inputParticles'))) else: particlesArray = [len(s) for s in self.inputParticles.get()] particlesArrayString = String(particlesArray) methods.append( 'Set of %d classes %s was employed to create %d initial volumes using RCT method. ' % (len(self.inputParticles.get()), self.getObjectTag('inputParticles'), len(self.inputParticles.get()))) methods.append( 'For each initial volume were used respectively %s particles' % particlesArrayString) methods.append("Output volumes: %s" % self.getObjectTag('outputVolumes')) if self.doFilter.get(): methods.append("Output filtered volumes: %s" % self.getObjectTag('outputFilteredVolumes')) return methods
def createOutputStep(self): outputVols = self._createSetOfVolumes() imgSet = self.inputParticles.get() for i, vol in enumerate(self._iterInputVols()): volume = vol.clone() volDir = self._getVolDir(i + 1) volPrefix = 'vol%03d_' % (i + 1) validationMd = self._getExtraPath(volPrefix + 'validation.xmd') moveFile(join(volDir, 'validation.xmd'), validationMd) clusterMd = self._getExtraPath(volPrefix + 'clusteringTendency.xmd') moveFile(join(volDir, 'clusteringTendency.xmd'), clusterMd) outImgSet = self._createSetOfParticles(volPrefix) outImgSet.copyInfo(imgSet) outImgSet.copyItems(imgSet, updateItemCallback=self._setWeight, itemDataIterator=md.iterRows( clusterMd, sortByLabel=md.MDL_ITEM_ID)) mdValidatoin = md.MetaData(validationMd) weight = mdValidatoin.getValue(md.MDL_WEIGHT, mdValidatoin.firstObject()) volume.weight = Float(weight) volume.clusterMd = String(clusterMd) volume.cleanObjId( ) # clean objects id to assign new ones inside the set outputVols.append(volume) self._defineOutputs(outputParticles=outImgSet) outputVols.setSamplingRate(volume.getSamplingRate()) self._defineOutputs(outputVolumes=outputVols)
def __init__(self, **kwargs): OrderedObject.__init__(self, **kwargs) self.name = String('default') self.maxCores = Integer() self.allowMPI = Boolean() self.allowThreads = Boolean() self.maxHours = Integer()
def _checkNewInput(self): newMics = [] for idx, coordSet in enumerate(self.inputCoordinates): coorSet = SetOfCoordinates(filename=coordSet.get().getFileName()) coorSet._xmippMd = String() coorSet.loadAllProperties() self.streamClosed = coorSet.isStreamClosed() coorSet.close() if self.check[idx] == "": newMics.append( [c.clone() for c in coordSet.get().getMicrographs()]) else: newMics.append([ c.clone() for c in coordSet.get().getMicrographs().iterItems( orderBy='creation', where='creation>"' + str(self.check[idx]) + '"') ]) if len(newMics[idx]) < 1: continue else: for p in coordSet.get().getMicrographs().iterItems( orderBy='creation', direction='DESC'): self.check[idx] = p.getObjCreation() break newMics[:] = [item for item in newMics if len(item) > 0] if len(newMics) > 0: fDeps = self.insertNewCoorsSteps(min(newMics, key=len)) self.inputMics = self.inputMics + len(min(newMics, key=len)) outputStep = self._getFirstJoinStep() if outputStep is not None: outputStep.addPrerequisites(*fDeps) self.updateSteps()
def _updateParticle(self, item, row): item.setClassId(row.getValue(md.MDL_REF)) item.setTransform(rowToAlignment(row, ALIGN_2D)) if self.flag_relion: item._rlnLogLikeliContribution = Float(None) item._rlnMaxValueProbDistribution = Float(None) item._rlnGroupName = String(None) item._rlnNormCorrection = Float(None)
def _updateItem(self, particle, row): self.reader.setParticleTransform(particle, row) # FIXME: check if other attrs need saving particle._rlnImageOriginalName = String(row.rlnImageOriginalName) particle._rlnRandomSubset = Integer(row.rlnRandomSubset) newLoc = convert.relionToLocation(row.rlnImageName) particle.setLocation(newLoc)
def setPsdFiles(ctfModel, ctfRow): """ Set the PSD files of CTF estimation related to this ctfModel. The values will be read from the ctfRow if present. """ for attr, label in CTF_PSD_DICT.iteritems(): if ctfRow.containsLabel(label): setattr(ctfModel, attr, String(ctfRow.getValue(label)))
def getReadyMics(coordSet): coorSet = SetOfCoordinates(filename=coordSet.getFileName()) coorSet._xmippMd = String() coorSet.loadAllProperties() setClosed = coorSet.isStreamClosed() coorSet.close() currentPickMics = { micAgg["_micId"] for micAgg in coordSet.aggregate(["MAX"], "_micId", ["_micId"]) } return currentPickMics, setClosed
def _defineProcessParams(self, form): # First we customize the inputParticles param to fit our needs in this protocol form.getParam('inputParticles').pointerCondition = String('hasAlignment') form.getParam('inputParticles').help = String('Input images you want to filter. It is important that the images have alignment information with ' 'respect to the chosen set of classes. This is the standard situation ' 'after CL2D or ML2D.') form.addParam('inputClasses', PointerParam, label='Input Classes', important=True, pointerClass='SetOfClasses', help='Select the input classes for the basis construction against images will be projected to.') form.addSection(label='Basis construction') form.addParam('maxClasses', IntParam, default=128, label='Max. number of classes', expertLevel=LEVEL_ADVANCED, help='Maximum number of classes.') form.addParam('maxPCABases', IntParam, default=200, label='Number of PCA bases', expertLevel=LEVEL_ADVANCED, help='Number of PCA bases.') form.addSection(label='Denoising') form.addParam('PCABases2Project', IntParam, default=200, label='Number of PCA bases on which to project', expertLevel=LEVEL_ADVANCED, help='Number of PCA bases on which to project.')
def _loadInputCoords(self, micDict): """ Load coordinates from the input streaming. """ # TODO: this takes for ever if you are NOT # doing streaming and have several thousands of mics # so I add a counter to keep the user entertained import sys a = datetime.now() counter = 1 coordsFn = self.getCoords().getFileName() self.debug("Loading input db: %s" % coordsFn) coordSet = SetOfCoordinates(filename=coordsFn) # FIXME: Temporary to avoid loadAllPropertiesFail coordSet._xmippMd = String() coordSet.loadAllProperties() # TODO: horrible code. Rewrite using # for coord in coordSet.iterItems(orderBy='_micId', # direction='ASC'): # micId = coord.getMicId() # if micId != lastMicId: # lastMicId = micId # ... # ... for micKey, mic in micDict.iteritems(): if counter % 50 == 0: b = datetime.now() print(b - a, 'reading coordinates for mic number', "%06d" % counter) sys.stdout.flush() # force buffer to print counter += 1 micId = mic.getObjId() coordList = [] self.debug("Loading coords for mic: %s (%s)" % (micId, micKey)) for coord in coordSet.iterItems(where='_micId=%s' % micId): # TODO: Check performance penalty of using this clone coordList.append(coord.clone()) self.debug("Coords found: %s" % len(coordList)) if coordList: self.coordDict[micId] = coordList else: del micDict[micKey] self.coordsClosed = coordSet.isStreamClosed() coordSet.close() self.debug("Coords are closed? %s" % self.coordsClosed) self.debug("Closed db.") return micDict
def writeSetOfParticles(imgSet, starFile, stackFile): """ This function will write a SetOfImages as Bsoft metadata. Params: imgSet: the SetOfImages instance. starFile: the filename where to write the metadata. """ mdata = md.MetaData() mdata.setColumnFormat(False) imgRow = mdata.Row() imgRow.setValue("micrograph.id", int(1)) imgRow.setValue("particle.x_origin", str(stackFile)) imgRow.writeToMd(mdata, mdata.addObject()) imgSet._bsoftStar = String(starFile)
def createOutput(self): fnStruct = glob.glob(self._getExtraPath("atomStructIn*"))[0] for fnDir in glob.glob(self._getPath('glide-*')): fnBase = os.path.split(fnDir)[1] fnGrid = os.path.join(fnDir, '%s.zip' % fnBase) if os.path.exists(fnGrid): gridFile = SchrodingerGrid(filename=fnGrid) gridFile.structureFile = String(fnStruct) n = fnDir.split('glide-')[1] outputDict = {'outputGrid%s' % n: gridFile} self._defineOutputs(**outputDict) self._defineSourceRelation(self.inputStructure, gridFile)
def sortImages(self, inputId): imagesMd = self._getPath('images.xmd') writeSetOfParticles(self.inputParticles.get(), imagesMd) args = "-i Particles@%s --addToInput " % imagesMd if self.autoParRejection == self.REJ_MAXZSCORE: args += "--zcut " + str(self.maxZscore.get()) elif self.autoParRejection == self.REJ_PERCENTAGE: args += "--percent " + str(self.percentage.get()) self.runJob("xmipp_image_sort_by_statistics", args) self.outputMd = String(imagesMd)
def saveConfig(filename): from pyworkflow.mapper import SqliteMapper from pyworkflow.object import String, Integer mapper = SqliteMapper(filename) o = Config() for k, v in globals().iteritems(): if k.startswith('cfg'): if type(v) is str: value = String(v) else: value = Integer(v) setattr(o, k, value) mapper.insert(o) mapper.commit()
def createChimeraScript(self, volume, pdb): """ Create a chimera script to visualize a pseudoatoms pdb obteined from a given EM 3d volume. A property will be set in the pdb object to store the location of the script. """ pseudoatoms = pdb.getFileName() scriptFile = pseudoatoms + '_chimera.cmd' pdb._chimeraScript = String(scriptFile) sampling = volume.getSamplingRate() radius = sampling * self.pseudoAtomRadius.get() fnIn = getImageLocation(volume) if fnIn.endswith(":mrc"): fnIn = fnIn[:-4] x, y, z = volume.getOrigin(force=True).getShifts() xx, yy, zz = volume.getDim() dim = volume.getDim()[0] bildFileName = os.path.abspath(self._getExtraPath("axis.bild")) Chimera.createCoordinateAxisFile(dim, bildFileName=bildFileName, sampling=sampling) fhCmd = open(scriptFile, 'w') fhCmd.write("open %s\n" % basename(pseudoatoms)) fhCmd.write("rangecol bfactor,a 0 white 1 red\n") fhCmd.write("setattr a radius %f\n" % radius) fhCmd.write("represent sphere\n") fhCmd.write("open %s\n" % abspath(fnIn)) threshold = 0.01 if self.maskMode == NMA_MASK_THRE: self.maskThreshold.get() # set sampling fhCmd.write("volume #1 level %f transparency 0.5 voxelSize %f origin " "%0.2f,%0.2f,%0.2f\n" % (threshold, sampling, x, y, z)) fhCmd.write("open %s\n" % bildFileName) #fhCmd.write("move %0.2f,%0.2f,%0.2f model #0 coord #2\n" # % ((xx / 2. * sampling) - xv, # (yy / 2. * sampling) - yv, # (zz / 2. * sampling) - zv)) fhCmd.write("move %0.2f,%0.2f,%0.2f model #0 coord #2\n" % (x + (xx / 2. * sampling), y + (yy / 2. * sampling), z + (zz / 2. * sampling))) fhCmd.close()
def createOutputSingle(self, fnSite, fnStructureFile, score, dscore, srcObj): n = fnSite.split('@')[0] fnDir = self._getPath("grid-%s" % n) if os.path.exists(fnDir): fnBase = os.path.split(fnDir)[1] fnGrid = os.path.join(fnDir, '%s.zip' % fnBase) if os.path.exists(fnGrid): gridFile = SchrodingerGrid(filename=fnGrid) gridFile.structureFile = String(fnStructureFile) gridFile.bindingSiteScore = Float(score) gridFile.bindingSiteDScore = Float(dscore) n = fnDir.split('grid-')[1] outputDict = {'outputGrid%s' % n: gridFile} self._defineOutputs(**outputDict) self._defineSourceRelation(srcObj, gridFile)
def show(self, form, *params): protocol = form.protocol try: models = self.getModelsChainsStep(protocol) except Exception as e: print "ERROR: ", e.message return self.editionListOfChains(models) finalChainList = [] for i in self.chainList: finalChainList.append(String(i)) provider = ListTreeProviderString(finalChainList) dlg = dialog.ListDialog( form.root, "Model chains", provider, "Select one of the chains (model, chain, " "number of chain residues)") form.setVar('inputStructureChain', dlg.values[0].get())
def _updateParticle(self, item, row): item.setClassId(row.rlnClassNumber) self._reader.setParticleTransform(item, row) # Try to create extra objects only once if item is reused if not hasattr(item, '_rlnNormCorrection'): item._rlnNormCorrection = Float() item._rlnLogLikeliContribution = Float() item._rlnMaxValueProbDistribution = Float() item._rlnNormCorrection.set(row.rlnNormCorrection) item._rlnLogLikeliContribution.set(row.rlnLogLikeliContribution) item._rlnMaxValueProbDistribution.set(row.rlnMaxValueProbDistribution) if hasattr(item, '_rlnGroupName'): item._rlnGroupName.set(row.rlnGroupName) elif hasattr(row, 'rlnGroupName'): item._rlnGroupName = String(row.rlnGroupName)
def __init__(self, **kwargs): OrderedObject.__init__(self, **kwargs) self.name = String() # Number of cores from which the queue is mandatory # 0 means no mandatory at all # 1 will force to launch all jobs through the queue self.mandatory = Integer() self.queues = None # List for queue configurations self.submitCommand = String() # Allow to change the prefix of submission scripts # we used by default the ID.job, but in some clusters # the job script should start by a letter self.submitPrefix = String() self.checkCommand = String() self.cancelCommand = String() self.submitTemplate = String() self.jobDoneRegex = String()
def denoiseImages(self, inputId, inputClassesId): # We start preparing writing those elements we're using as input to keep them untouched imagesMd = self._getPath('images.xmd') writeSetOfParticles(self.inputParticles.get(), imagesMd) classesMd = self._getPath('classes.xmd') writeSetOfClasses2D(self.inputClasses.get(), classesMd) fnRoot = self._getExtraPath('pca') fnRootDenoised = self._getExtraPath('imagesDenoised') args = '-i Particles@%s --oroot %s --eigenvectors %d --maxImages %d' % (imagesMd, fnRoot, self.maxPCABases.get(), self.maxClasses.get()) self.runJob("xmipp_image_rotational_pca", args) N=min(self.maxPCABases.get(), self.PCABases2Project.get()) args='-i %s -o %s.stk --save_metadata_stack %s.xmd --basis %s.stk %d'\ % (imagesMd, fnRootDenoised, fnRootDenoised, fnRoot, N) self.runJob("xmipp_transform_filter", args) self.outputMd = String('%s.stk' % fnRootDenoised)
def createOutputStep(self): outputVols = self._createSetOfVolumes() for i, vol in enumerate(self._iterInputVols()): volume = vol.clone() volDir = self._getVolDir(i+1) volPrefix = 'vol%03d_' % (i+1) validationMd = self._getExtraPath(volPrefix + 'validation.xmd') moveFile(join(volDir, 'validation.xmd'), validationMd) clusterMd = self._getExtraPath(volPrefix + 'clusteringTendency.xmd') moveFile(join(volDir, 'clusteringTendency.xmd'), clusterMd) md = xmipp.MetaData(validationMd) weight = md.getValue(xmipp.MDL_WEIGHT, md.firstObject()) volume.weight = Float(weight) volume.clusterMd = String(clusterMd) volume.cleanObjId() # clean objects id to assign new ones inside the set outputVols.append(volume) outputVols.setSamplingRate(volume.getSamplingRate()) self._defineOutputs(outputVolumes=outputVols)
def _defineParams(self, form, fullForm=True): self._defineParams1(form,"t","Cp") if fullForm: form.addParam('fitType', params.EnumParam, choices=["Linear","Logarithmic","Relative"], label="Fit mode", default=1, help='Linear: sum (Cobserved-Cpredicted)^2\nLogarithmic: sum(log10(Cobserved)-log10(Cpredicted))^2\n'\ "Relative: sum ((Cobserved-Cpredicted)/Cobserved)^2") form.addParam('Nexp', params.IntParam, label="Number of exponentials", default=1, help='Number of exponentials to fit') else: self.fitType=Integer() self.fitType.set(1) self.Nexp=Integer() self.Nexp.set(1) form.addParam('bounds', params.StringParam, label="Amplitude and time constant bounds", default="", expertLevel=LEVEL_ADVANCED, help='Bounds for the c_i amplitudes and lambdas.\nExample 1: (0,10);(0,1e-2) -> c1 in (0,10), lambda1 in (0,1e-2)\n'\ 'Example 2: (0,10);(0,1e-2);(0,1);(0,1e-1) -> c1 in (0,10), lambda1 in (0,1e-2), c2 in (0,1), lambda2 in (0,1e-1)') form.addParam('confidenceInterval', params.FloatParam, label="Confidence interval=", default=95, expertLevel=LEVEL_ADVANCED, help='Confidence interval for the fitted parameters') if fullForm: form.addParam('reportX', params.StringParam, label="Evaluate at X=", default="", expertLevel=LEVEL_ADVANCED, help='Evaluate the model at these X values\nExample 1: [0,5,10,20,40,100]\nExample 2: 0:0.55:10, from 0 to 10 in steps of 0.5') else: self.reportX=String() self.reportX.set("")
def getTemplate(root): # Check if there is any .json.template in the template folder # get the template folder templateFolder = pw.getTemplatePath() # Get all ".json.template" there templates = [] for file in glob.glob1(templateFolder, "*.json.template"): templates.append(String(file)) if len(templates): if len(templates) == 1: chosen = templates[0].get() else: provider = ListTreeProviderString(templates) dlg = dialog.ListDialog(root, "Workflow templates", provider, "Select one of the templates.") chosen = dlg.values[0].get() chosen = os.path.join(templateFolder, chosen) print("Template to use: %s" % chosen) with open(chosen, 'r') as myfile: template = myfile.read() # Replace environment variables template = template % os.environ return template else: raise Exception("There isn't any *.json.template at %s" % templates)