def test_TableRowSetValue(self):
table = emc.Table([
Column(1, "col1", emc.typeFloat),
Column(2, "col2", emc.typeInt16),
Column(3, "col3", emc.typeString)
])
for i, t in enumerate([(1, "col1", emc.typeFloat),
(2, "col2", emc.typeInt16),
(3, "col3", emc.typeString)]):
col = table.getColumnByIndex(i)
self.assertEqual(t[0], col.getId())
self.assertEqual(t[1], col.getName())
self.assertEqual(t[2], col.getType())
row = table.createRow()
print("Row (before set) >>> ", row)
# Define some constants values to set and check
NAMES = ['My name', 'Second name', 'Yet another name']
FLOATS = [3.1416]
INTS = [300, 400]
row[1] = FLOATS[0]
row[2] = INTS[0]
row[3] = NAMES[0]
print("Adding row to table...")
table.addRow(row)
print("Accessing row by index. ")
row2 = table[0]
def read_emcore():
import emcore as emc
dataFile = testfile('star', 'refine3d', 'run_it016_sampling.star')
tables = []
for i in range(N):
t = emc.Table()
t.read('sampling_directions', dataFile)
tables.append(t)
memory_usage()
def _checkPrefix(self, tableFn, expectedPrefix):
table = emc.Table()
table.read(tableFn)
row = table[0]
imgPath = str(row['rlnImageName'])
im = emv.utils.ImageManager()
realPath = imgPath.split('@')[1]
imgPrefix = im.findImagePrefix(realPath, tableFn)
self.assertEqual(imgPrefix, expectedPrefix)
def test_ColumnsBasic(self):
c1Name = "firstCol"
c1 = Column(c1Name, emc.typeFloat)
self.assertEqual(c1.getId(), Column.NO_ID)
self.assertEqual(c1.getName(), c1Name)
self.assertEqual(c1.getType(), emc.typeFloat)
c2Name = "secondCol"
c2 = Column(c2Name, emc.typeInt16)
self.assertEqual(c2.getId(), Column.NO_ID)
self.assertEqual(c2.getName(), c2Name)
self.assertEqual(c2.getType(), emc.typeInt16)
colMap = emc.Table()
self.assertEqual(0, colMap.addColumn(c1))
self.assertEqual(1, colMap.addColumn(c2))
# Get columns stored and do some validations
rc1 = colMap.getColumnByIndex(0)
rc2 = colMap.getColumnByIndex(1)
self.assertEqual(rc1.getId(), 1)
self.assertEqual(rc2.getId(), 2)
self.assertEqual(0, colMap.getIndex(c1.getName()))
self.assertEqual(1, colMap.getIndex(c2.getName()))
self.assertEqual(0, colMap.getIndex(rc1.getId()))
self.assertEqual(1, colMap.getIndex(rc2.getId()))
#self.assertRaises(emc.Error, lambda: colMap.getIndex(100))
#self.assertRaises(emc.Error, lambda: colMap.getIndex("noColumn"))
# Add more columns with and without IDs
bigId = 100
c3index = colMap.addColumn(Column(bigId, "thirdCol", emc.typeFloat))
rc3 = colMap.getColumnByIndex(c3index)
self.assertEqual(bigId, rc3.getId())
self.assertEqual("thirdCol", rc3.getName())
colMap.addColumn(Column("forthCol", emc.typeFloat))
rc4 = colMap.getColumnByIndex(c3index + 1)
self.assertEqual(bigId + 1, rc4.getId())
self.assertEqual("forthCol", rc4.getName())
# Let's insert a new column, all indexes before should not be
# changed, but the ones after the position should be increased by 1
c3bindex = colMap.insertColumn(Column("thirdBCol", emc.typeFloat),
c3index + 1)
self.assertEqual(c3bindex, c3index + 1)
for i, colName in enumerate(["firstCol", "secondCol", "thirdCol"]):
self.assertEqual(colMap.getIndex(colName), i)
self.assertEqual(colMap.getIndex("forthCol"), c3bindex + 1)
def _getCoordsList(self, micId):
""" Return the coordinates list of a given micrograph. """
mic = self.getMicrograph(micId)
if micId not in self._loadedMics:
baseFn = dv.utils.removeBaseExt(mic.getPath())
coordsFn = self._input('Movies/%s_autopick.star' % baseFn)
coordsTable = emc.Table()
coordsTable.read(coordsFn)
mic._coordinates.extend(self._coordFromRow(row) for row in coordsTable)
self._loadedMics.add(micId)
return mic._coordinates
def test_ReadSingleRow(self):
self.assertTrue(emc.TableFile.hasImpl('star'))
td = TestData()
root = td.get("relion_tutorial/import/")
fn1 = root + "case1/classify3d_small_it038_optimiser.star"
print("Reading star: ", fn1)
t = emc.Table()
t.read("optimiser_general", fn1)
self.assertEqual(t.getColumnsSize(), 53)
self.assertEqual(t.getSize(), 1)
def __init__(self, tableSource, **kwargs):
""" Create a new instance of EmTableModel.
Args:
tableSource: Input from where table will be retrieved,
it can be one of the following options:
* emc.Table: just a single table that will be used, not
other tables will be loaded in this case
* string: This should be the path from where to read
the table(s). The first table will be loaded by default.
* tuple (string, string): Here you can specify the path and
the name of the table that you want to be loaded by default.
Keyword Arguments:
imageManager=value Provide an ImageManager that can be used
to read images referenced from this table.
"""
if isinstance(tableSource, emc.Table):
self._table = tableSource
self._tableIO = None
# Define only a single table name ''
tableName = ''
self._path = None
self._tableNames = [tableName]
else: # In this variant we will create a emc.TableFile to read data
if isinstance(tableSource, py23.str):
if '@' in tableSource:
tableName, path = tableSource.split('@')
else:
tableName, path = None, tableSource
elif isinstance(tableSource, tuple):
path, tableName = tableSource
else:
raise Exception("Invalid tableSource input '%s' (type %s)"
% (tableSource, type(tableSource)))
self._path = os.path.abspath(path)
self._tableIO = emc.TableFile()
self._tableIO.open(self._path, emc.File.Mode.READ_ONLY)
self._table = emc.Table()
self._tableNames = self._tableIO.getTableNames()
# If not tableName provided, load first table
tableName = tableName or self._tableNames[0]
# Create an ImageManager if none is provided
self._imageManager = kwargs.get('imageManager', ImageManager())
# Use a dictionary for checking the prefix path of the
# images columns data
self._imagePrefixes = kwargs.get('imagePrefixes', {})
self.loadTable(tableName)
def test_Copy(self):
n = 10
table10 = self.createTable(n)
table10copy = emc.Table(table10)
self.checkColumns(table10copy)
self.assertEqual(table10copy.getSize(), table10.getSize())
self.assertEqual(table10copy.getSize(), n)
colNames = [col.getName() for col in table10.iterColumns()]
for i in range(n):
row1 = table10[i]
row2 = table10copy[i]
for cn in colNames:
self.assertEqual(row1[cn], row2[cn])
def createTable(self, nRows):
table = emc.Table([
Column(1, "col1", emc.typeSizeT),
Column(2, "col2", emc.typeFloat),
Column(3, "col3", emc.typeString)
])
row = table.createRow()
for i in range(nRows):
row["col1"] = i
row["col2"] = float(i) / 100
row["col3"] = "image_%03d" % i
table.addRow(row)
return table
def createEmptyTableModel(cls, columns=[]):
"""
Creates an TableModel instance, initializing the table header from the
given ColumnInfo list
Args:
columns: (list) List of ColumnInfo for table header initialization
Returns:
A :class:`TableModel <datavis.models.TableModel>` instance
"""
Column = emc.Table.Column
cols = []
for i, info in enumerate(columns):
if isinstance(info, models.ColumnInfo):
cols.append(
Column(i + 1, info.getName(),
EmType.toModel(info.getType())))
else:
raise Exception("Invalid ColumnInfo.")
return EmTableModel(emc.Table(cols))
def test_ReadXmd(self):
self.assertTrue(emc.TableFile.hasImpl('star'))
td = TestData()
fn1 = td.get("xmipp_tutorial/gold/images200k.xmd")
print("Reading xmd: ", fn1)
tio = emc.TableFile()
table = emc.Table()
t = Timer()
t.tic()
tio.open(fn1)
tio.read("noname", table)
t.toc()
print("Size: ", table.getSize())
tio.close()
refColNames = [
'itemId', 'xcoor', 'ycoor', 'image', 'micrograph', 'enabled',
'ctfDefocusU', 'ctfDefocusV', 'ctfDefocusAngle', 'ctfQ0',
'ctfSphericalAberration', 'ctfVoltage', 'zScore', 'zScoreShape1',
'zScoreShape2', 'zScoreSNR1', 'zScoreSNR2', 'zScoreHistogram'
]
# Get column by index
for i in range(table.getColumnsSize()):
col = table.getColumnByIndex(i)
self.assertEqual(refColNames[i], col.getName())
# Iterate through the columns
for i, col1 in enumerate(table.iterColumns()):
col2 = table.getColumnByIndex(i)
self.assertEqual(refColNames[i], col1.getName())
self.assertEqual(refColNames[i], col2.getName())
self.assertEqual(col1, col2)
def _loadData(self):
notes = self._input('note.txt')
summary = self._input('summary.star')
if not os.path.exists(notes):
raise Exception("Expecting file '%s'" % notes)
if not os.path.exists(summary):
raise Exception("Expecting summary file '%s'" % summary)
self._summaryTable = emc.Table()
self._summaryTable.read(summary)
projectRoot = None
for row in self._summaryTable:
micFn = str(row['rlnMicrographName'])
if projectRoot is None:
projectRoot = self._imageManager.findImagePrefix(
micFn, os.path.abspath(self._inputDir))
mic = dv.models.Micrograph(path=os.path.join(projectRoot, micFn))
self.addMicrograph(mic)
self._loadedMics = set()
def createView(self):
kwargs = dict()
kwargs['selectionMode'] = dv.views.PagingView.SINGLE_SELECTION
kwargs['boxSize'] = 300
kwargs['pickerMode'] = dv.views.DEFAULT_MODE
kwargs['shape'] = dv.views.SHAPE_CIRCLE
kwargs['removeRois'] = True
kwargs['roiAspectLocked'] = True
kwargs['roiCentered'] = True
projectFolder = self.projectFolder #'/Users/josem/work/data/relion30_tutorial_precalculated_results/'
pickingFolder = self.pickingFolder # 'AutoPick/LoG_based'
pickingPath = os.path.join(projectFolder, pickingFolder)
micsStar = self.micStar # 'Select/job005/micrographs_selected.star'
# For some reason Relion store input micrographs star filename
# in the following star file, that is not a STAR file
# suffixMicFn = os.path.join(pickingPath, 'coords_suffix_autopick.star')
#
# if not os.path.exists(suffixMicFn):
# raise Exception("Missing expected file: %s" % suffixMicFn)
#
# with open(suffixMicFn) as f:
# micsStar = f.readline().strip()
micsStarPath = os.path.join(projectFolder, micsStar)
if not os.path.exists(micsStarPath):
raise Exception("Missing expected file %s" % micsStarPath)
print("Relion Project: %s" % projectFolder)
print(" Micrographs: %s" % micsStar)
print(" Picking: %s" % pickingFolder)
# FIXME: Read Table data from the constructor
table = emc.Table()
coordTable = emc.Table()
table.read(micsStarPath)
model = emv.models.EmPickerModel()
model.setBoxSize(64)
def _getMicPath(micName):
micPath = os.path.join(projectFolder, micName)
if os.path.exists(micPath):
return micPath
micPath = os.path.join(pickingPath, micName)
if os.path.exists(micPath):
return micPath
raise Exception("Can not find root path for mic: %s" % micName)
for i, row in enumerate(table):
micPath = _getMicPath(str(row['rlnMicrographName']))
mic = dv.models.Micrograph(i + 1, micPath)
micBase = os.path.basename(micPath).replace(
".mrc", "_autopick.star")
micCoordsFn = os.path.join(pickingPath, micBase)
model.addMicrograph(mic)
if os.path.exists(micCoordsFn):
coordTable.read(micCoordsFn)
for coordRow in coordTable:
coord = model.createCoordinate(
round(float(coordRow['rlnCoordinateX'])),
round(float(coordRow['rlnCoordinateY'])), '')
model.addCoordinates(mic.getId(), [coord])
return dv.views.PickerView(model, **kwargs)
def test_Read(self):
td = TestData()
self.assertTrue(emc.TableFile.hasImpl('star'))
root = td.get("relion_tutorial/import/")
fn1 = root + "case1/classify3d_small_it038_data.star"
print("Reading star: ", fn1)
tio = emc.TableFile()
tio.open(fn1)
tableNames = tio.getTableNames()
print("Tables: ", tableNames)
tio.close()
t = emc.Table()
t.read(fn1) # should read first table by default
t = emc.Table([
Column(1, "col1", emc.typeFloat),
Column(2, "col2", emc.typeInt16),
Column(3, "col3", emc.typeString)
])
self.assertEqual(t.getColumnsSize(), 3)
self.assertTrue(t.isEmpty())
t.read("images", fn1)
print("Size: ", t.getSize())
self.assertEqual(t.getSize(), 79)
refColNames = [
"rlnVoltage", "rlnDefocusU", "rlnSphericalAberration",
"rlnAmplitudeContrast", "rlnImageName", "rlnNormCorrection",
"rlnMicrographName", "rlnGroupNumber", "rlnOriginX", "rlnOriginY",
"rlnAngleRot", "rlnAngleTilt", "rlnAnglePsi", "rlnClassNumber",
"rlnLogLikeliContribution", "rlnNrOfSignificantSamples",
"rlnMaxValueProbDistribution"
]
"""
loop_
_rlnVoltage #1
_rlnDefocusU #2
_rlnSphericalAberration #3
_rlnAmplitudeContrast #4
_rlnImageName #5
_rlnNormCorrection #6
_rlnMicrographName #7
_rlnGroupNumber #8
_rlnOriginX #9
_rlnOriginY #10
_rlnAngleRot #11
_rlnAngleTilt #12
_rlnAnglePsi #13
_rlnClassNumber #14
_rlnLogLikeliContribution #15
_rlnNrOfSignificantSamples #16
_rlnMaxValueProbDistribution #17
300.000000 15060.500000 2.260000 0.100000 000080@cluster/wind2/win_05677.dat
0.771391 cluster/wind2/win_00849.dat 1 -2.968653 -22.218653 -20.283924
175.359562 126.884867 5 24711.149348 1 0.962371
"""
goldValues = [('rlnGroupNumber', [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21
]),
('rlnDefocusU', [
15060.5, 16009.5, 18642.2, 24642.4, 24772.8, 25580.8,
26833.9, 27164.9, 30135.1, 33744.2, 35465.5, 13923.4,
17895.1, 19206.2, 24165.1, 24650.4, 25866.7, 26527.1,
27915.1, 29939.5, 30062.1
]),
('rlnImageName', [
'000080@cluster/wind2/win_05677.dat',
'000023@cluster/wind2/win_05883.dat',
'000042@cluster/wind2/win_04377.dat',
'000065@cluster/wind2/win_00139.dat',
'000038@cluster/wind2/win_04884.dat',
'000046@cluster/wind2/win_02565.dat',
'000023@cluster/wind2/win_00161.dat',
'000026@cluster/wind2/win_03758.dat',
'000058@cluster/wind2/win_03862.dat',
'000024@cluster/wind2/win_04246.dat',
'000073@cluster/wind2/win_02382.dat',
'000021@cluster/wind2/win_04587.dat',
'000007@cluster/wind2/win_03186.dat',
'000056@cluster/wind2/win_04919.dat',
'000076@cluster/wind2/win_03268.dat',
'000048@cluster/wind2/win_04346.dat',
'000073@cluster/wind2/win_03740.dat',
'000036@cluster/wind2/win_04583.dat',
'000015@cluster/wind2/win_05453.dat',
'000021@cluster/wind2/win_00846.dat',
'000080@cluster/wind2/win_04165.dat'
])]
for i, col in enumerate(t.iterColumns()):
self.assertEqual(refColNames[i], col.getName())
print(col)
for label, values in goldValues:
mdValues = [row[label] for i, row in enumerate(t) if i < 20]
for v1, v2 in zip(values, mdValues):
self.assertEqual(v2, v1)
def test_TableBasic(self):
table = emc.Table([
Column(1, "col1", emc.typeFloat),
Column(2, "col2", emc.typeInt16),
Column(3, "col3", emc.typeString)
])
for i, t in enumerate([(1, "col1", emc.typeFloat),
(2, "col2", emc.typeInt16),
(3, "col3", emc.typeString)]):
col = table.getColumnByIndex(i)
self.assertEqual(t[0], col.getId())
self.assertEqual(t[1], col.getName())
self.assertEqual(t[2], col.getType())
row = table.createRow()
print("Row (before set) >>> ", row)
# Define some constants values to set and check
NAMES = ['My name', 'Second name', 'Yet another name']
FLOATS = [3.1416]
INTS = [300, 400]
row[1] = FLOATS[0]
row[2] = INTS[0]
row[3] = NAMES[0]
print("Row (after set) >>> ", row)
# Check values are properly set
self.assertAlmostEqual(float(row[1]), FLOATS[0], 3)
self.assertEqual(int(row[2]), INTS[0])
self.assertEqual(str(row[3]), NAMES[0])
# Check we get same values using Column IDs or Names
self.assertEqual(row[1], row["col1"])
self.assertEqual(row[2], row["col2"])
self.assertEqual(row[3], row["col3"])
# Check we can retrieve values in normal variables
x = int(row[2])
self.assertEqual(x, INTS[0])
# Lets add another row, based on first one, but with some changes
row2 = Row(row)
row2["col2"] = INTS[1]
row2["col3"] = NAMES[1]
x = int(row2[2])
self.assertEqual(x, INTS[1])
print("Row2 >>> ", row2)
# Check that row was not modified after the copy
self.assertAlmostEqual(float(row[1]), FLOATS[0], 3)
self.assertEqual(int(row[2]), INTS[0])
self.assertEqual(str(row[3]), NAMES[0])
# Let's add both rows to the table
table.addRow(row)
table.addRow(row2)
for row in table:
row["col3"] = NAMES[2]
row["col2"] = int(row["col2"]) / 10
print("Table: ", table)
self.assertEqual(2, table.getSize())
self.assertFalse(table.isEmpty())
# Check row values are as expected (using iterator)
for i, row in enumerate(table):
self.assertAlmostEqual(row["col1"], FLOATS[0], 3)
self.assertEqual(row["col2"], INTS[i] / 10)
self.assertEqual(row["col3"], NAMES[2])
# Check row values are as expected (using indexing)
for i in range(table.getSize()):
row = table[i]
self.assertAlmostEqual(row["col1"], FLOATS[0], 3)
self.assertEqual(row["col2"], INTS[i] / 10)
self.assertEqual(row["col3"], NAMES[2])
table.clear()
self.assertEqual(0, table.getSize())
self.assertTrue(table.isEmpty())
import emcore as emc
import datavis as dv
import emvis as emv
from datavis.views import TableModel
testDataPath = os.environ.get("EM_TEST_DATA", None)
print("hasImpl('star'): ", emc.TableFile.hasImpl('star'))
if testDataPath is not None:
# use the code below when yue have a properly configured environment
fn1 = os.path.join(testDataPath, "relion_tutorial", "import", "refine3d",
"extra", "relion_it025_data.star")
print("Reading star: ", fn1)
t = emc.Table()
tio = emc.TableFile()
tio.open(fn1)
tio.read("images", t)
tio.close()
refColNames = [
"rlnVoltage", "rlnDefocusU", "rlnSphericalAberration",
"rlnAmplitudeContrast", "rlnImageName", "rlnNormCorrection",
"rlnMicrographName", "rlnGroupNumber", "rlnOriginX", "rlnOriginY",
"rlnAngleRot", "rlnAngleTilt", "rlnAnglePsi", "rlnClassNumber",
"rlnLogLikeliContribution", "rlnNrOfSignificantSamples",
"rlnMaxValueProbDistribution"
]
model = emv.models.ModelsFactory.createTableModel(fn1)
