def test_iterRows(self): print("Checking iterRows...") dataFile = testfile('star', 'refine3d', 'run_it016_data.star') table = Table(fileName=dataFile, tableName='particles') # Let's open again the same file for iteration with open(dataFile) as f: tableReader = Table.Reader(f, tableName='particles') for c1, c2 in zip(table.getColumns(), tableReader.getColumns()): self.assertEqual(c1, c2, "Column c1 (%s) differs from c2 (%s)" % (c1, c2)) for r1, r2 in zip(table, tableReader): self.assertEqual(r1, r2) # Now try directly with iterRows function for r1, r2 in zip(table, Table.iterRows(dataFile, tableName='particles')): self.assertEqual(r1, r2) defocusSorted = sorted(float(r.rlnDefocusU) for r in table) for d1, row in zip(defocusSorted, Table.iterRows(dataFile, tableName='particles', key=lambda r: r.rlnDefocusU)): self.assertAlmostEqual(d1, row.rlnDefocusU) # Test sorting by imageName column, also using getColumnValues and sort() imageIds = table.getColumnValues('rlnImageName') imageIds.sort() # Check sorted iteration give the total amount of rows rows = [r for r in Table.iterRows(dataFile, tableName='particles', key='rlnImageName')] self.assertEqual(len(imageIds), len(rows)) for id1, row in zip(imageIds, Table.iterRows(dataFile, tableName='particles', key='rlnImageName')): self.assertEqual(id1, row.rlnImageName) def getIter(): """ Test a function to get an iterator. """ return Table.iterRows(dataFile, tableName='particles', key='rlnImageName') iterByIds = getIter() for id1, row in zip(imageIds, iterByIds): self.assertEqual(id1, row.rlnImageName)
def _fillDataFromIter(self, imgSet, iteration): tableName = 'particles@' if self.IS_GT30() else '' outImgsFn = self._getFileName('data', iter=iteration) imgSet.setAlignmentProj() self.reader = convert.createReader(alignType=ALIGN_PROJ, pixelSize=imgSet.getSamplingRate()) mdIter = Table.iterRows(tableName + outImgsFn, key='rlnImageId') imgSet.copyItems(self._getInputParticles(), doClone=False, updateItemCallback=self._updateParticle, itemDataIterator=mdIter)
def createFinalFilesStep(self): # -----metadata to save all final models------- finalModel = self._getFileName('finalModel') finalModelMd = self._getMetadata() # -----metadata to save all final particles----- finalData = self._getFileName('finalData') fn = self._getFileName('rawFinalData') print("FN: ", fn) tableIn = Table(fileName=fn, tableName='particles') cols = [str(c) for c in tableIn.getColumnNames()] ouTable = Table(columns=cols, tableName='particles') for rLev in self._getRLevList(): it = self._lastIter(rLev) modelFn = self._getFileName('model', iter=it, lev=self._level, rLev=rLev) modelMd = self._getMetadata('model_classes@' + modelFn) refLabel = md.RLN_MLMODEL_REF_IMAGE imgRow = md.getFirstRow(modelMd) fn = imgRow.getValue(refLabel) mapId = self._getRunLevId(rLev=rLev) newMap = self._getMapById(mapId) imgRow.setValue(refLabel, newMap) copyFile(fn, newMap) self._mapsDict[fn] = mapId imgRow.addToMd(finalModelMd) dataFn = self._getFileName('data', iter=it, lev=self._level, rLev=rLev) pTable = Table() for row in pTable.iterRows(dataFn, tableName='particles'): newRow = row._replace(rlnClassNumber=rLev) ouTable.addRow(*newRow) self.writeStar(finalData, ouTable) finalModelMd.write('model_classes@' + finalModel)
def _fillClassesFromIter(self, clsSet): from ..convert import createReader """ Create the SetOfClasses3D from a given iteration. """ self._loadClassesInfo() dataStar = self._getFileName('finalData') pixelSize = self.inputParticles.get().getSamplingRate() if (self.IS_3D): self._reader = createReader(alignType=ALIGN_PROJ, pixelSize=pixelSize) else: self._reader = createReader(alignType=ALIGN_2D, #JV pixelSize=pixelSize) mdIter = Table.iterRows('particles@' + dataStar, key='rlnImageId') clsSet.classifyItems(updateItemCallback=self._updateParticle, updateClassCallback=self._updateClass, itemDataIterator=mdIter, doClone=False) #JV
def mergeClassesStep(self): if self.doGrouping: from cryomethods.functions import NumpyImgHandler npIh = NumpyImgHandler() makePath(self._getLevelPath(self._level)) listVol = self._getFinalMaps() matrix = npIh.getAllNpList(listVol, 2) labels = self._clusteringData(matrix) clsChange = 0 prevStar = self._getFileName('rawFinalData') pTable = Table() origStar = self._getFileName('input_star', lev=1, rLev=1) opticsTable = Table(fileName=origStar, tableName='optics') print("OPTABLE: ", origStar, opticsTable.size()) for row in pTable.iterRows(prevStar, key="rlnClassNumber", tableName='particles'): clsPart = row.rlnClassNumber newClass = labels[clsPart - 1] + 1 newRow = row._replace(rlnClassNumber=newClass) if not newClass == clsChange: if not clsChange == 0: self.writeStar(fn, ouTable, opticsTable) clsChange = newClass fn = self._getFileName('input_star', lev=self._level, rLev=newClass) tableIn = Table(fileName=prevStar, tableName='particles') cols = [str(c) for c in tableIn.getColumnNames()] ouTable = Table(columns=cols, tableName='particles') ouTable.addRow(*newRow) print("mergeClassesStep ouTable.size: ", ouTable.size()) self.writeStar(fn, ouTable, opticsTable) else: prevData = self._getFileName('rawFinalData') finalData = self._getFileName('finalData') prevModel = self._getFileName('rawFinalModel') finalModel = self._getFileName('finalModel') copyFile(prevData, finalData) copyFile(prevModel, finalModel)
def getIter(): """ Test a function to get an iterator. """ return Table.iterRows(dataFile, tableName='particles', key='rlnImageName')
def _mergeDataStar(self, rLev, callback): def _getMapId(rMap): try: return self._mapsDict[rMap] except: return None iters = self._lastIter(rLev) #metadata to save all particles that continues outData = self._getFileName('outputData', lev=self._level) #metadata to save all final particles finalData = self._getFileName('rawFinalData') imgStar = self._getFileName('data', iter=iters, lev=self._level, rLev=rLev) opTable = Table(filename=imgStar, tableName='optics') tableIn = Table(fileName=imgStar, tableName='particles') print("IMGSTAR: ", imgStar, "PARTS: ", tableIn.size()) cols = [str(c) for c in tableIn.getColumnNames()] outTable = Table(columns=cols, tableName='particles') finalTable = Table(columns=cols, tableName='particles') if os.path.exists(outData): print("Exists ", outData) tmpTable = Table() for row in tmpTable.iterRows(outData, tableName='particles'): outTable.addRow(*row) if os.path.exists(finalData): print("Exists ", finalData) tpTable = Table() for row in tpTable.iterRows(finalData, tableName='particles'): finalTable.addRow(*row) pTable = Table() for row in pTable.iterRows(imgStar, key="rlnClassNumber", tableName='particles'): clsPart = row.rlnClassNumber rMap = callback(iters, rLev, clsPart) mapId = _getMapId(rMap) while mapId is None: for clsPart in range(1, self.numberOfClasses.get()+1): rMap = callback(iters, rLev, clsPart) mapId = _getMapId(rMap) if mapId is not None: break if self.stopDict[mapId]: # if mapId != newMapId: # if newMapId != '00.000': # print(mdClass) # mdClass.write(classMd) # classMd = self._getFileName('mdataForClass', id=mapId) # mdClass = self._getMetadata(classMd) # newMapId = mapId classId = self._clsIdDict[mapId] newRow = row._replace(rlnClassNumber=classId) finalTable.addRow(*newRow) else: classId = int(mapId.split('.')[1]) newRow = row._replace(rlnClassNumber=classId) outTable.addRow(*newRow) # if self.stopDict[mapId]: # if mdClass.size() != 0: # mdClass.write(classMd) if finalTable.size() != 0: print("finalTable.size: ", finalTable.size()) self.writeStar(finalData, finalTable) if outTable.size() != 0: print("outTable.size: ", outTable.size()) self.writeStar(outData, outTable, opTable)
def convertInputStep(self, resetDeps, copyAlignment): import random """ Create the input file in STAR format as expected by Relion. If the input particles comes from Relion, just link the file. """ if self._level == 0: makePath(self._getRunPath(self._level, 1)) imgStar = self._getFileName('input_star', lev=self._level, rLev=0) self._convertStar(copyAlignment, imgStar) opticsTable = Table(fileName=imgStar, tableName='optics') partsTable = Table(fileName=imgStar, tableName='particles') self._convertVol(ImageHandler(), self.inputVolumes.get()) mdSize = partsTable.size() for i in range(9, 1, -1): makePath(self._getRunPath(self._level, i)) mStar = self._getFileName('input_star', lev=self._level, rLev=i) size = 10000 * i if mdSize >= 100000 else int(mdSize * 0.1 * i) print("partsTable: ", size, i, mdSize) partsTable._rows = random.sample(partsTable._rows, k=size) self.writeStar(mStar, partsTable, opticsTable) elif self._level == 1: imgStar = self._getFileName('input_star', lev=self._level, rLev=1) makePath(self._getRunPath(self._level, 1)) self._convertStar(copyAlignment, imgStar) # find a clever way to avoid volume conversion if its already done. self._convertVol(ImageHandler(), self.inputVolumes.get()) else: lastCls = None prevStar = self._getFileName('outputData', lev=self._level - 1) firstStarFn = self._getFileName('input_star', lev=1, rLev=1) # mdData = md.MetaData(prevStar) opTable = Table(fileName=firstStarFn, tableName='optics') tableIn = Table(fileName=prevStar, tableName='particles') cols = [str(c) for c in tableIn.getColumnNames()] pTable = Table() for row in pTable.iterRows(prevStar, key="rlnClassNumber", tableName='particles'): clsPart = row.rlnClassNumber if clsPart != lastCls: makePath(self._getRunPath(self._level, clsPart)) if lastCls is not None: print("writing %s" % fn) # mdInput.write(fn) self.writeStar(fn, newPTable, opTable) paths = self._getRunPath(self._level, clsPart) makePath(paths) print ("Path: %s and newRlev: %d" % (paths, clsPart)) lastCls = clsPart newPTable = Table(columns=cols, tableName='particles') fn = self._getFileName('input_star', lev=self._level, rLev=clsPart) # objId = mdInput.addObject() newPTable.addRow(*row) # row.writeToMd(mdInput, objId) print("writing %s and ending the loop" % fn) self.writeStar(fn, newPTable, opTable)