def relionLabelString(): """ create an string that can be used for XMIPP_EXTRA_ALIASES for adding the labels of Relion. """ from xmipp import label2Str pairs = [] for k, v in XMIPP_RELION_LABELS.iteritems(): pairs.append('%s=%s' % (label2Str(k), v)) for k, v in XMIPP_RELION_LABELS_EXTRA.iteritems(): pairs.append('%s=%s' % (label2Str(k), v)) return ';'.join(pairs)
def objectToRow(obj, row, attrDict, extraLabels={}): """ This function will convert an EMObject into a XmippMdRow. Params: obj: the EMObject instance (input) row: the XmippMdRow instance (output) attrDict: dictionary with the map between obj attributes(keys) and row MDLabels in Xmipp (values). extraLabels: a list with extra labels that could be included as _xmipp_labelName """ if obj.isEnabled(): enabled = 1 else: enabled = -1 row.setValue(xmipp.MDL_ENABLED, enabled) for attr, label in attrDict.iteritems(): if hasattr(obj, attr): valueType = getLabelPythonType(label) row.setValue(label, valueType(getattr(obj, attr).get())) attrLabels = attrDict.values() for label in extraLabels: attrName = '_xmipp_%s' % xmipp.label2Str(label) if label not in attrLabels and hasattr(obj, attrName): value = obj.getAttributeValue(attrName) row.setValue(label, value)
def rowToObject(row, obj, attrDict, extraLabels={}): """ This function will convert from a XmippMdRow to an EMObject. Params: row: the XmippMdRow instance (input) obj: the EMObject instance (output) attrDict: dictionary with the map between obj attributes(keys) and row MDLabels in Xmipp (values). extraLabels: a list with extra labels that could be included as _xmipp_labelName """ obj.setEnabled(row.getValue(xmipp.MDL_ENABLED, 1) > 0) for attr, label in attrDict.iteritems(): value = row.getValue(label) if not hasattr(obj, attr): setattr(obj, attr, ObjectWrap(value)) else: getattr(obj, attr).set(value) attrLabels = attrDict.values() for label in extraLabels: if label not in attrLabels and row.hasLabel(label): labelStr = xmipp.label2Str(label) setattr(obj, '_xmipp_%s' % labelStr, row.getValueAsObject(label))
def findRow(md, label, value): """ Query the metadata for a row with label=value. Params: md: metadata to query. label: label to check value value: value for equal condition Returns: XmippMdRow object of the row found. None if no row is found with label=value """ mdQuery = xmipp.MetaData() # store result mdQuery.importObjects(md, xmipp.MDValueEQ(label, value)) n = mdQuery.size() if n == 0: row = None elif n == 1: row = XmippMdRow() row.readFromMd(mdQuery, mdQuery.firstObject()) else: raise Exception( "findRow: more than one row found matching the query %s = %s" % (xmipp.label2Str(label), value)) return row
def convertCtfparam(oldCtf): '''Convert the old format (Xmipp2.4) of the CTF and return a new MetaData''' f = open(oldCtf) md = MetaData() md.setColumnFormat(False) objId = md.addObject() for line in f: line = line.strip() if len(line) and not line.startswith('#'): parts = line.split('=') old_key = parts[0].strip() if old_key in OLD_CTF_BASIC_DICT: value = float(parts[1].strip()) label = OLD_CTF_BASIC_DICT[old_key] md.setValue(label, value, objId) else: print "WARNING: Ignoring old ctfparam key:", old_key f.close() #Change sign of defocusU, defocusV and Q0 labels = [label2Str(l) for l in [MDL_CTF_DEFOCUSU, MDL_CTF_DEFOCUSV, MDL_CTF_Q0]] exps = ["%s=%s*-1" % (l, l) for l in labels] expression = ','.join(exps) md.operate(expression) return md
def setXmippAttributes(obj, objRow, *labels): """ Set an attribute to obj from a label that is not basic ones. The new attribute will be named _xmipp_LabelName and the datatype will be set correctly. """ for label in labels: setattr(obj, '_xmipp_%s' % xmipp.label2Str(label), objRow.getValueAsObject(label))
def __setXmippImage(label): attr = '_xmipp_' + xmipp.label2Str(label) if not hasattr(particle, attr): img = Image() setattr(particle, attr, img) img.setSamplingRate(particle.getSamplingRate()) else: img = getattr(particle, attr) img.setLocation(xmippToLocation(row.getValue(label)))
def calculateDeviationsStep(self, it): """ Calculate both angles and shifts devitations for all iterations """ SL = xmipp.SymList() mdIter = xmipp.MetaData() #for it in self.allIters(): mdIter.clear() SL.readSymmetryFile(self._symmetry[it]) md1 = xmipp.MetaData(self.docFileInputAngles[it]) md2 = xmipp.MetaData(self.docFileInputAngles[it-1]) #ignore disabled, md1.removeDisabled() md2.removeDisabled() #first metadata file may not have shiftx and shifty if not md2.containsLabel(xmipp.MDL_SHIFT_X): md2.addLabel(xmipp.MDL_SHIFT_X) md2.addLabel(xmipp.MDL_SHIFT_Y) md2.fillConstant(xmipp.MDL_SHIFT_X,0.) md2.fillConstant(xmipp.MDL_SHIFT_Y,0.) oldLabels=[xmipp.MDL_ANGLE_ROT, xmipp.MDL_ANGLE_TILT, xmipp.MDL_ANGLE_PSI, xmipp.MDL_SHIFT_X, xmipp.MDL_SHIFT_Y] newLabels=[xmipp.MDL_ANGLE_ROT2, xmipp.MDL_ANGLE_TILT2, xmipp.MDL_ANGLE_PSI2, xmipp.MDL_SHIFT_X2, xmipp.MDL_SHIFT_Y2] md2.renameColumn(oldLabels,newLabels) md2.addLabel(xmipp.MDL_SHIFT_X_DIFF) md2.addLabel(xmipp.MDL_SHIFT_Y_DIFF) md2.addLabel(xmipp.MDL_SHIFT_DIFF) mdIter.join1(md1, md2, xmipp.MDL_IMAGE, xmipp.INNER_JOIN) SL.computeDistance(mdIter,False,False,False) xmipp.activateMathExtensions() #operate in sqlite shiftXLabel = xmipp.label2Str(xmipp.MDL_SHIFT_X) shiftX2Label = xmipp.label2Str(xmipp.MDL_SHIFT_X2) shiftXDiff = xmipp.label2Str(xmipp.MDL_SHIFT_X_DIFF) shiftYLabel = xmipp.label2Str(xmipp.MDL_SHIFT_Y) shiftY2Label = xmipp.label2Str(xmipp.MDL_SHIFT_Y2) shiftYDiff = xmipp.label2Str(xmipp.MDL_SHIFT_Y_DIFF) shiftDiff = xmipp.label2Str(xmipp.MDL_SHIFT_DIFF) #timeStr = str(dtBegin) operateString = shiftXDiff+"="+shiftXLabel+"-"+shiftX2Label operateString += "," + shiftYDiff+"="+shiftYLabel+"-"+shiftY2Label mdIter.operate(operateString) operateString = shiftDiff+"=sqrt(" \ +shiftXDiff+"*"+shiftXDiff+"+" \ +shiftYDiff+"*"+shiftYDiff+");" mdIter.operate(operateString) iterFile = self._mdDevitationsFn(it) mdIter.write(iterFile,xmipp.MD_APPEND) self._setLastIter(it)
def writeToMd(self, md, objId): """ Set back row values to a metadata row. """ for label, value in self._labelDict.iteritems(): # TODO: Check how to handle correctly unicode type # in Xmipp and Scipion if type(value) is unicode: value = str(value) try: md.setValue(label, value, objId) except Exception, ex: print >> sys.stderr, "XmippMdRow.writeToMd: Error writting value to metadata." print >> sys.stderr, " label: %s, value: %s, type(value): %s" % ( label2Str(label), value, type(value)) raise ex
def test_CTF(self): """ Test the conversion of a SetOfParticles to Xmipp metadata. """ mdCtf = xmipp.MetaData(self.dataset.getFile('ctfGold')) objId = mdCtf.firstObject() rowCtf = rowFromMd(mdCtf, objId) ctf = rowToCtfModel(rowCtf) ALL_CTF_LABELS = [ xmipp.MDL_CTF_CA, xmipp.MDL_CTF_ENERGY_LOSS, xmipp.MDL_CTF_LENS_STABILITY, xmipp.MDL_CTF_CONVERGENCE_CONE, xmipp.MDL_CTF_LONGITUDINAL_DISPLACEMENT, xmipp.MDL_CTF_TRANSVERSAL_DISPLACEMENT, xmipp.MDL_CTF_K, xmipp.MDL_CTF_BG_GAUSSIAN_K, xmipp.MDL_CTF_BG_GAUSSIAN_SIGMAU, xmipp.MDL_CTF_BG_GAUSSIAN_SIGMAV, xmipp.MDL_CTF_BG_GAUSSIAN_CU, xmipp.MDL_CTF_BG_GAUSSIAN_CV, xmipp.MDL_CTF_BG_SQRT_K, xmipp.MDL_CTF_BG_SQRT_U, xmipp.MDL_CTF_BG_SQRT_V, xmipp.MDL_CTF_BG_SQRT_ANGLE, xmipp.MDL_CTF_BG_BASELINE, xmipp.MDL_CTF_BG_GAUSSIAN2_K, xmipp.MDL_CTF_BG_GAUSSIAN2_SIGMAU, xmipp.MDL_CTF_BG_GAUSSIAN2_SIGMAV, xmipp.MDL_CTF_BG_GAUSSIAN2_CU, xmipp.MDL_CTF_BG_GAUSSIAN2_CV, xmipp.MDL_CTF_BG_GAUSSIAN2_ANGLE, xmipp.MDL_CTF_CRIT_FITTINGSCORE, xmipp.MDL_CTF_CRIT_FITTINGCORR13, xmipp.MDL_CTF_DOWNSAMPLE_PERFORMED, xmipp.MDL_CTF_CRIT_PSDVARIANCE, xmipp.MDL_CTF_CRIT_PSDPCA1VARIANCE, xmipp.MDL_CTF_CRIT_PSDPCARUNSTEST, xmipp.MDL_CTF_CRIT_FIRSTZEROAVG, xmipp.MDL_CTF_CRIT_DAMPING, xmipp.MDL_CTF_CRIT_FIRSTZERORATIO, xmipp.MDL_CTF_CRIT_PSDCORRELATION90, xmipp.MDL_CTF_CRIT_PSDRADIALINTEGRAL, xmipp.MDL_CTF_CRIT_NORMALITY, ] for label in ALL_CTF_LABELS: attrName = '_xmipp_%s' % xmipp.label2Str(label) self.assertAlmostEquals(mdCtf.getValue(label, objId), ctf.getAttributeValue(attrName))
def writeToMd(self, md, objId): """ Set back row values to a metadata row. """ for label, value in self._labelDict.iteritems(): # TODO: Check how to handle correctly unicode type # in Xmipp and Scipion if type(value) is unicode: value = str(value) try: md.setValue(label, value, objId) except Exception as ex: import sys print("XmippMdRow.writeToMd: Error writing value to metadata.", file=sys.stderr) print(" label: %s, value: %s, type(value): %s" % ( label2Str(label), value, type(value)), file=sys.stderr) raise ex
def test_CTF(self): """ Test the convertion of a SetOfParticles to Xmipp metadata. """ mdCtf = xmipp.MetaData(self.dataset.getFile('ctfGold')) objId = mdCtf.firstObject() rowCtf = rowFromMd(mdCtf, objId) ctf = rowToCtfModel(rowCtf) ALL_CTF_LABELS = [ xmipp.MDL_CTF_CA, xmipp.MDL_CTF_ENERGY_LOSS, xmipp.MDL_CTF_LENS_STABILITY, xmipp.MDL_CTF_CONVERGENCE_CONE, xmipp.MDL_CTF_LONGITUDINAL_DISPLACEMENT, xmipp.MDL_CTF_TRANSVERSAL_DISPLACEMENT, xmipp.MDL_CTF_K, xmipp.MDL_CTF_BG_GAUSSIAN_K, xmipp.MDL_CTF_BG_GAUSSIAN_SIGMAU, xmipp.MDL_CTF_BG_GAUSSIAN_SIGMAV, xmipp.MDL_CTF_BG_GAUSSIAN_CU, xmipp.MDL_CTF_BG_GAUSSIAN_CV, xmipp.MDL_CTF_BG_SQRT_K, xmipp.MDL_CTF_BG_SQRT_U, xmipp.MDL_CTF_BG_SQRT_V, xmipp.MDL_CTF_BG_SQRT_ANGLE, xmipp.MDL_CTF_BG_BASELINE, xmipp.MDL_CTF_BG_GAUSSIAN2_K, xmipp.MDL_CTF_BG_GAUSSIAN2_SIGMAU, xmipp.MDL_CTF_BG_GAUSSIAN2_SIGMAV, xmipp.MDL_CTF_BG_GAUSSIAN2_CU, xmipp.MDL_CTF_BG_GAUSSIAN2_CV, xmipp.MDL_CTF_BG_GAUSSIAN2_ANGLE, xmipp.MDL_CTF_CRIT_FITTINGSCORE, xmipp.MDL_CTF_CRIT_FITTINGCORR13, xmipp.MDL_CTF_DOWNSAMPLE_PERFORMED, xmipp.MDL_CTF_CRIT_PSDVARIANCE, xmipp.MDL_CTF_CRIT_PSDPCA1VARIANCE, xmipp.MDL_CTF_CRIT_PSDPCARUNSTEST, xmipp.MDL_CTF_CRIT_FIRSTZEROAVG, xmipp.MDL_CTF_CRIT_DAMPING, xmipp.MDL_CTF_CRIT_FIRSTZERORATIO, xmipp.MDL_CTF_CRIT_PSDCORRELATION90, xmipp.MDL_CTF_CRIT_PSDRADIALINTEGRAL, xmipp.MDL_CTF_CRIT_NORMALITY, ] for label in ALL_CTF_LABELS: attrName = '_xmipp_%s' % xmipp.label2Str(label) self.assertAlmostEquals(mdCtf.getValue(label, objId), ctf.getAttributeValue(attrName))
def _getMdString(self, filename, block=None): md = xmipp.MetaData() if block: md.read(block + '@' + filename) else: md.read(filename, 1) labels = md.getActiveLabels() msg = "Metadata items: *%d*\n" % md.getParsedLines() msg += "Metadata labels: " + ''.join(["\n - %s" % xmipp.label2Str(l) for l in labels]) imgPath = None for label in labels: if xmipp.labelIsImage(label): imgPath = self._getImgPath(filename, md.getValue(label, md.firstObject())) break if imgPath: self._imgPreview = self._getImagePreview(imgPath) self._imgInfo = self._getImageString(imgPath) return msg
def _getMdString(self, filename, block=None): md = xmipp.MetaData() if block: md.read(block + '@' + filename) else: md.read(filename, 1) labels = md.getActiveLabels() msg = "Metadata items: *%d*\n" % md.getParsedLines() msg += "Metadata labels: " + ''.join( ["\n - %s" % xmipp.label2Str(l) for l in labels]) imgPath = None for label in labels: if xmipp.labelIsImage(label): imgPath = self._getImgPath( filename, md.getValue(label, md.firstObject())) break if imgPath: self._imgPreview = self._getImagePreview(imgPath) self._imgInfo = self._getImageString(imgPath) return msg
def exportMdToRelion(md, outputRelion): """ This function will receive a Xmipp metadata and will convert to the one expected by Relion. All labels not recognized by Relion will be dropped. Params: md: input xmipp metadata. outputRelion: output filename to store the Relion star file. """ for label in md.getActiveLabels(): if not label in XMIPP_RELION_LABELS: md.removeLabel(label) tmpFile = outputRelion + '.tmp' md.write(tmpFile) # Create a dict with the names d = {} for k, v in XMIPP_RELION_LABELS.iteritems(): d[label2Str(k)] = v #print "dict: ", d renameMdLabels(tmpFile, outputRelion, d) from protlib_filesystem import deleteFile deleteFile(None, tmpFile)
def writeToMd(self, md, objId): """ Set back row values to a metadata row. """ for label, value in self._labelDict.iteritems(): # TODO: Check how to handle correctly unicode type # in Xmipp and Scipion t = type(value) if t is unicode: value = str(value) if t is int and xmipp.labelType(label) == xmipp.LABEL_SIZET: value = long(value) try: md.setValue(label, value, objId) except Exception as ex: print( "XmippMdRow.writeToMd: Error writting value to metadata.", file=sys.stderr) print( " label: %s, value: %s, type(value): %s" % (label2Str(label), value, type(value)), file=sys.stderr) raise ex
def __str__(self): s = "{" for k, v in self._labelDict.iteritems(): s += " %s = %s\n" % (xmipp.label2Str(k), v) return s + "}"
def writeToMd(self, md, objId): """ Set back row values to a metadata row. """ for label, value in self._labelDict.iteritems(): # TODO: Check how to handle correctly unicode type # in Xmipp and Scipion t = type(value) if t is unicode: value = str(value) if t is int and xmipp.labelType(label) == xmipp.LABEL_SIZET: value = long(value) try: md.setValue(label, value, objId) except Exception, ex: print >> sys.stderr, "XmippMdRow.writeToMd: Error writting value to metadata." print >> sys.stderr, " label: %s, value: %s, type(value): %s" % (label2Str(label), value, type(value)) raise ex
def _convertLabelToColumn(self, label, md): """ From an Xmipp label, create the corresponding column. """ return ds.Column(xmipp.label2Str(label), getLabelPythonType(label), renderType=self._getLabelRenderType(label, md))
def __str__(self): s = '{' for k, v in self._labelDict.iteritems(): s += ' %s = %s\n' % (xmipp.label2Str(k), v) return s + '}'
def __str__(self): s = '{' for k, v in self._labelDict.iteritems(): s += ' %s = %s\n' % (label2Str(k), v) return s + '}'
def findRow(md, label, value): """ Query the metadata for a row with label=value. Params: md: metadata to query. label: label to check value value: value for equal condition Returns: XmippMdRow object of the row found. None if no row is found with label=value """ mdQuery = xmipp.MetaData() # store result mdQuery.importObjects(md, xmipp.MDValueEQ(label, value)) n = mdQuery.size() if n == 0: row = None elif n == 1: row = XmippMdRow() row.readFromMd(mdQuery, mdQuery.firstObject()) else: raise Exception("findRow: more than one row found matching the query %s = %s" % (xmipp.label2Str(label), value)) return row
def _convertLabelToColumn(self, label): """ From an Xmipp label, create the corresponding column. """ return ds.Column(xmipp.label2Str(label), getLabelPythonType(label), renderType=self._getLabelRenderType(label))
def _plotHistogramAngularMovement(self, paramName=None): from numpy import arange from matplotlib.ticker import FormatStrFormatter plots = [] colors = ['g', 'b', 'r', 'y', 'c', 'm', 'k'] lenColors = len(colors) numberOfBins = self.numberOfBins.get() md = xmipp.MetaData() for it in self._iterations: mdFn = self.protocol._mdDevitationsFn(it) if xmipp.existsBlockInMetaDataFile(mdFn): md.read(mdFn) if not self.usePsi: md.fillConstant(xmipp.MDL_ANGLE_PSI, 0.) nrefs = len(self._refsList) gridsize = self._getGridSize(nrefs) xplotterShift = XmippPlotter(*gridsize, mainTitle='Iteration_%d\n' % it, windowTitle="ShiftDistribution") xplotter = XmippPlotter(*gridsize, mainTitle='Iteration_%d' % it, windowTitle="AngularDistribution") for ref3d in self._refsList: mDoutRef3D = xmipp.MetaData() mDoutRef3D.importObjects( md, xmipp.MDValueEQ(xmipp.MDL_REF3D, ref3d)) _frequency = "Frequency (%d)" % mDoutRef3D.size() xplotterShift.createSubPlot( "%s_ref3D_%d" % (xmipp.label2Str(xmipp.MDL_SHIFT_DIFF), ref3d), "pixels", _frequency) xplotter.createSubPlot( "%s_ref3D_%d" % (xmipp.label2Str(xmipp.MDL_ANGLE_DIFF), ref3d), "degrees", _frequency) #mDoutRef3D.write("*****@*****.**",MD_APPEND) xplotter.plotMd( mDoutRef3D, xmipp.MDL_ANGLE_DIFF, xmipp.MDL_ANGLE_DIFF, color=colors[ref3d % lenColors], #nbins=50 nbins=int(numberOfBins) ) #if nbins is present do an histogram xplotterShift.plotMd( mDoutRef3D, xmipp.MDL_SHIFT_DIFF, xmipp.MDL_SHIFT_DIFF, color=colors[ref3d % lenColors], nbins=int(numberOfBins )) #if nbins is present do an histogram if self.angleSort: mDoutRef3D.sort(xmipp.MDL_ANGLE_DIFF) fn = xmipp.FileName() baseFileName = self.protocol._getTmpPath( "angle_sort.xmd") fn = self.protocol._getRefBlockFileName( "angle_iter", it, "ref3D", ref3d, baseFileName) mDoutRef3D.write(fn, xmipp.MD_APPEND) print "File with sorted angles saved in:", fn if self.shiftSort: mDoutRef3D.sort(xmipp.MDL_SHIFT_DIFF) fn = xmipp.FileName() baseFileName = self.protocol._getTmpPath( "angle_sort.xmd") fn = self.protocol._getRefBlockFileName( "shift_iter", it, "ref3D", ref3d, baseFileName) mDoutRef3D.write(fn, xmipp.MD_APPEND) print "File with sorted shifts saved in:", fn plots.append(xplotterShift) plots.append(xplotter) else: print "File %s does not exist" % mdFn return plots
def _plotHistogramAngularMovement(self, paramName=None): from numpy import arange from matplotlib.ticker import FormatStrFormatter plots = [] colors = ['g', 'b', 'r', 'y', 'c', 'm', 'k'] lenColors=len(colors) numberOfBins = self.numberOfBins.get() md = xmipp.MetaData() for it in self._iterations: mdFn = self.protocol._mdDevitationsFn(it) if xmipp.existsBlockInMetaDataFile(mdFn): md.read(mdFn) if not self.usePsi: md.fillConstant(xmipp.MDL_ANGLE_PSI,0.) nrefs = len(self._refsList) gridsize = self._getGridSize(nrefs) xplotterShift = XmippPlotter(*gridsize, mainTitle='Iteration_%d\n' % it, windowTitle="ShiftDistribution") xplotter = XmippPlotter(*gridsize, mainTitle='Iteration_%d' % it, windowTitle="AngularDistribution") for ref3d in self._refsList: mDoutRef3D = xmipp.MetaData() mDoutRef3D.importObjects(md, xmipp.MDValueEQ(xmipp.MDL_REF3D, ref3d)) _frequency = "Frequency (%d)" % mDoutRef3D.size() xplotterShift.createSubPlot("%s_ref3D_%d"%(xmipp.label2Str(xmipp.MDL_SHIFT_DIFF),ref3d), "pixels", _frequency) xplotter.createSubPlot("%s_ref3D_%d"%(xmipp.label2Str(xmipp.MDL_ANGLE_DIFF),ref3d), "degrees", _frequency) #mDoutRef3D.write("*****@*****.**",MD_APPEND) xplotter.plotMd(mDoutRef3D, xmipp.MDL_ANGLE_DIFF, xmipp.MDL_ANGLE_DIFF, color=colors[ref3d%lenColors], #nbins=50 nbins=int(numberOfBins) )#if nbins is present do an histogram xplotterShift.plotMd(mDoutRef3D, xmipp.MDL_SHIFT_DIFF, xmipp.MDL_SHIFT_DIFF, color=colors[ref3d%lenColors], nbins=int(numberOfBins) )#if nbins is present do an histogram if self.angleSort: mDoutRef3D.sort(xmipp.MDL_ANGLE_DIFF) fn = xmipp.FileName() baseFileName = self.protocol._getTmpPath("angle_sort.xmd") fn = self.protocol._getRefBlockFileName("angle_iter", it, "ref3D", ref3d, baseFileName) mDoutRef3D.write(fn, xmipp.MD_APPEND) print "File with sorted angles saved in:", fn if self.shiftSort: mDoutRef3D.sort(xmipp.MDL_SHIFT_DIFF) fn = xmipp.FileName() baseFileName = self.protocol._getTmpPath("angle_sort.xmd") fn = self.protocol._getRefBlockFileName("shift_iter", it, "ref3D", ref3d, baseFileName) mDoutRef3D.write(fn, xmipp.MD_APPEND) print "File with sorted shifts saved in:", fn plots.append(xplotterShift) plots.append(xplotter) else: print "File %s does not exist" % mdFn return plots