def test_iterRows(self):
print("Checking iterRows...")
dataFile = testfile('star', 'refine3d', 'run_it016_data.star')
table = Table(fileName=dataFile, tableName='particles')
# Let's open again the same file for iteration
with open(dataFile) as f:
tableReader = Table.Reader(f, tableName='particles')
for c1, c2 in zip(table.getColumns(), tableReader.getColumns()):
self.assertEqual(c1, c2, "Column c1 (%s) differs from c2 (%s)"
% (c1, c2))
for r1, r2 in zip(table, tableReader):
self.assertEqual(r1, r2)
# Now try directly with iterRows function
for r1, r2 in zip(table,
Table.iterRows(dataFile, tableName='particles')):
self.assertEqual(r1, r2)
defocusSorted = sorted(float(r.rlnDefocusU) for r in table)
for d1, row in zip(defocusSorted,
Table.iterRows(dataFile,
tableName='particles',
key=lambda r: r.rlnDefocusU)):
self.assertAlmostEqual(d1, row.rlnDefocusU)
# Test sorting by imageName column, also using getColumnValues and sort()
imageIds = table.getColumnValues('rlnImageName')
imageIds.sort()
# Check sorted iteration give the total amount of rows
rows = [r for r in Table.iterRows(dataFile,
tableName='particles',
key='rlnImageName')]
self.assertEqual(len(imageIds), len(rows))
for id1, row in zip(imageIds,
Table.iterRows(dataFile,
tableName='particles',
key='rlnImageName')):
self.assertEqual(id1, row.rlnImageName)
def getIter():
""" Test a function to get an iterator. """
return Table.iterRows(dataFile,
tableName='particles', key='rlnImageName')
iterByIds = getIter()
for id1, row in zip(imageIds, iterByIds):
self.assertEqual(id1, row.rlnImageName)
def _fillDataFromIter(self, imgSet, iteration):
tableName = 'particles@' if self.IS_GT30() else ''
outImgsFn = self._getFileName('data', iter=iteration)
imgSet.setAlignmentProj()
self.reader = convert.createReader(alignType=ALIGN_PROJ,
pixelSize=imgSet.getSamplingRate())
mdIter = Table.iterRows(tableName + outImgsFn, key='rlnImageId')
imgSet.copyItems(self._getInputParticles(), doClone=False,
updateItemCallback=self._updateParticle,
itemDataIterator=mdIter)
def createFinalFilesStep(self):
# -----metadata to save all final models-------
finalModel = self._getFileName('finalModel')
finalModelMd = self._getMetadata()
# -----metadata to save all final particles-----
finalData = self._getFileName('finalData')
fn = self._getFileName('rawFinalData')
print("FN: ", fn)
tableIn = Table(fileName=fn, tableName='particles')
cols = [str(c) for c in tableIn.getColumnNames()]
ouTable = Table(columns=cols, tableName='particles')
for rLev in self._getRLevList():
it = self._lastIter(rLev)
modelFn = self._getFileName('model', iter=it,
lev=self._level, rLev=rLev)
modelMd = self._getMetadata('model_classes@' + modelFn)
refLabel = md.RLN_MLMODEL_REF_IMAGE
imgRow = md.getFirstRow(modelMd)
fn = imgRow.getValue(refLabel)
mapId = self._getRunLevId(rLev=rLev)
newMap = self._getMapById(mapId)
imgRow.setValue(refLabel, newMap)
copyFile(fn, newMap)
self._mapsDict[fn] = mapId
imgRow.addToMd(finalModelMd)
dataFn = self._getFileName('data', iter=it,
lev=self._level, rLev=rLev)
pTable = Table()
for row in pTable.iterRows(dataFn, tableName='particles'):
newRow = row._replace(rlnClassNumber=rLev)
ouTable.addRow(*newRow)
self.writeStar(finalData, ouTable)
finalModelMd.write('model_classes@' + finalModel)
def _fillClassesFromIter(self, clsSet):
from ..convert import createReader
""" Create the SetOfClasses3D from a given iteration. """
self._loadClassesInfo()
dataStar = self._getFileName('finalData')
pixelSize = self.inputParticles.get().getSamplingRate()
if (self.IS_3D):
self._reader = createReader(alignType=ALIGN_PROJ,
pixelSize=pixelSize)
else:
self._reader = createReader(alignType=ALIGN_2D, #JV
pixelSize=pixelSize)
mdIter = Table.iterRows('particles@' + dataStar, key='rlnImageId')
clsSet.classifyItems(updateItemCallback=self._updateParticle,
updateClassCallback=self._updateClass,
itemDataIterator=mdIter,
doClone=False) #JV
def mergeClassesStep(self):
if self.doGrouping:
from cryomethods.functions import NumpyImgHandler
npIh = NumpyImgHandler()
makePath(self._getLevelPath(self._level))
listVol = self._getFinalMaps()
matrix = npIh.getAllNpList(listVol, 2)
labels = self._clusteringData(matrix)
clsChange = 0
prevStar = self._getFileName('rawFinalData')
pTable = Table()
origStar = self._getFileName('input_star', lev=1, rLev=1)
opticsTable = Table(fileName=origStar, tableName='optics')
print("OPTABLE: ", origStar, opticsTable.size())
for row in pTable.iterRows(prevStar, key="rlnClassNumber",
tableName='particles'):
clsPart = row.rlnClassNumber
newClass = labels[clsPart - 1] + 1
newRow = row._replace(rlnClassNumber=newClass)
if not newClass == clsChange:
if not clsChange == 0:
self.writeStar(fn, ouTable, opticsTable)
clsChange = newClass
fn = self._getFileName('input_star', lev=self._level,
rLev=newClass)
tableIn = Table(fileName=prevStar, tableName='particles')
cols = [str(c) for c in tableIn.getColumnNames()]
ouTable = Table(columns=cols, tableName='particles')
ouTable.addRow(*newRow)
print("mergeClassesStep ouTable.size: ", ouTable.size())
self.writeStar(fn, ouTable, opticsTable)
else:
prevData = self._getFileName('rawFinalData')
finalData = self._getFileName('finalData')
prevModel = self._getFileName('rawFinalModel')
finalModel = self._getFileName('finalModel')
copyFile(prevData, finalData)
copyFile(prevModel, finalModel)
def getIter():
""" Test a function to get an iterator. """
return Table.iterRows(dataFile,
tableName='particles', key='rlnImageName')
def _mergeDataStar(self, rLev, callback):
def _getMapId(rMap):
try:
return self._mapsDict[rMap]
except:
return None
iters = self._lastIter(rLev)
#metadata to save all particles that continues
outData = self._getFileName('outputData', lev=self._level)
#metadata to save all final particles
finalData = self._getFileName('rawFinalData')
imgStar = self._getFileName('data', iter=iters,
lev=self._level, rLev=rLev)
opTable = Table(filename=imgStar, tableName='optics')
tableIn = Table(fileName=imgStar, tableName='particles')
print("IMGSTAR: ", imgStar, "PARTS: ", tableIn.size())
cols = [str(c) for c in tableIn.getColumnNames()]
outTable = Table(columns=cols, tableName='particles')
finalTable = Table(columns=cols, tableName='particles')
if os.path.exists(outData):
print("Exists ", outData)
tmpTable = Table()
for row in tmpTable.iterRows(outData, tableName='particles'):
outTable.addRow(*row)
if os.path.exists(finalData):
print("Exists ", finalData)
tpTable = Table()
for row in tpTable.iterRows(finalData, tableName='particles'):
finalTable.addRow(*row)
pTable = Table()
for row in pTable.iterRows(imgStar, key="rlnClassNumber",
tableName='particles'):
clsPart = row.rlnClassNumber
rMap = callback(iters, rLev, clsPart)
mapId = _getMapId(rMap)
while mapId is None:
for clsPart in range(1, self.numberOfClasses.get()+1):
rMap = callback(iters, rLev, clsPart)
mapId = _getMapId(rMap)
if mapId is not None:
break
if self.stopDict[mapId]:
# if mapId != newMapId:
# if newMapId != '00.000':
# print(mdClass)
# mdClass.write(classMd)
# classMd = self._getFileName('mdataForClass', id=mapId)
# mdClass = self._getMetadata(classMd)
# newMapId = mapId
classId = self._clsIdDict[mapId]
newRow = row._replace(rlnClassNumber=classId)
finalTable.addRow(*newRow)
else:
classId = int(mapId.split('.')[1])
newRow = row._replace(rlnClassNumber=classId)
outTable.addRow(*newRow)
# if self.stopDict[mapId]:
# if mdClass.size() != 0:
# mdClass.write(classMd)
if finalTable.size() != 0:
print("finalTable.size: ", finalTable.size())
self.writeStar(finalData, finalTable)
if outTable.size() != 0:
print("outTable.size: ", outTable.size())
self.writeStar(outData, outTable, opTable)
def convertInputStep(self, resetDeps, copyAlignment):
import random
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
"""
if self._level == 0:
makePath(self._getRunPath(self._level, 1))
imgStar = self._getFileName('input_star', lev=self._level, rLev=0)
self._convertStar(copyAlignment, imgStar)
opticsTable = Table(fileName=imgStar, tableName='optics')
partsTable = Table(fileName=imgStar, tableName='particles')
self._convertVol(ImageHandler(), self.inputVolumes.get())
mdSize = partsTable.size()
for i in range(9, 1, -1):
makePath(self._getRunPath(self._level, i))
mStar = self._getFileName('input_star', lev=self._level, rLev=i)
size = 10000 * i if mdSize >= 100000 else int(mdSize * 0.1 * i)
print("partsTable: ", size, i, mdSize)
partsTable._rows = random.sample(partsTable._rows, k=size)
self.writeStar(mStar, partsTable, opticsTable)
elif self._level == 1:
imgStar = self._getFileName('input_star', lev=self._level, rLev=1)
makePath(self._getRunPath(self._level, 1))
self._convertStar(copyAlignment, imgStar)
# find a clever way to avoid volume conversion if its already done.
self._convertVol(ImageHandler(), self.inputVolumes.get())
else:
lastCls = None
prevStar = self._getFileName('outputData', lev=self._level - 1)
firstStarFn = self._getFileName('input_star', lev=1, rLev=1)
# mdData = md.MetaData(prevStar)
opTable = Table(fileName=firstStarFn, tableName='optics')
tableIn = Table(fileName=prevStar, tableName='particles')
cols = [str(c) for c in tableIn.getColumnNames()]
pTable = Table()
for row in pTable.iterRows(prevStar, key="rlnClassNumber",
tableName='particles'):
clsPart = row.rlnClassNumber
if clsPart != lastCls:
makePath(self._getRunPath(self._level, clsPart))
if lastCls is not None:
print("writing %s" % fn)
# mdInput.write(fn)
self.writeStar(fn, newPTable, opTable)
paths = self._getRunPath(self._level, clsPart)
makePath(paths)
print ("Path: %s and newRlev: %d" % (paths, clsPart))
lastCls = clsPart
newPTable = Table(columns=cols, tableName='particles')
fn = self._getFileName('input_star', lev=self._level,
rLev=clsPart)
# objId = mdInput.addObject()
newPTable.addRow(*row)
# row.writeToMd(mdInput, objId)
print("writing %s and ending the loop" % fn)
self.writeStar(fn, newPTable, opTable)
