def findMaskPoints(self, image, mask, subscriber=0):
"""
Returns a table of masked points with each row
giving a tuple (coordinate_1, ..., coordindate_n, value).
"""
self.check(image, mask)
subscriber %= 10.0
index = (mask.data == FEATURE_COLOR).nonzero()
zVal = image.data[index]
subscriber %= 60.0
fields = []
for dim, coord in enumerate(index):
newField = FieldContainer(
image.dimensions[dim].data[coord],
image.dimensions[dim].unit,
longname=image.dimensions[dim].longname + " %i" % dim,
shortname=image.dimensions[dim].shortname
)
fields.append(newField)
fields.append(FieldContainer(zVal, image.unit,
longname=image.longname,
shortname=image.shortname)
)
res = SampleContainer(
fields,
u"Points from %s at %s"%(image.longname, mask.longname),
u"X1"
)
res.seal()
subscriber %= 100.0
return res
def setUp(self):
super(SampleContainerInSampleContainerTestCase, self).setUp()
sample1 = SampleContainer([self.dependent, self.field],
longname='First Sample',
shortname='X',
attributes=copy.copy(self.attributes).update(
{'isSample': 'It seems so.'}))
self.independent2 = FieldContainer(
0.3 * numpy.linspace(0, 1, self.testData.shape[0] * 10),
longname='independent variable',
shortname='x',
unit=Quantity('1 mg'),
attributes=copy.copy(self.attributes).update({'independent':
True}))
self.dependent2 = FieldContainer(9.81 * self.independent2.data,
dimensions=[self.independent2],
longname='dependent variable',
shortname='f',
unit=Quantity('9.81 nN'),
attributes=copy.copy(
self.attributes).update(
{'independent': False}))
sample2 = SampleContainer([self.dependent2, self.independent2],
longname='Second Sample',
shortname='Y',
attributes=copy.copy(self.attributes).update(
{'sample Nr.': 2}))
self.sample = SampleContainer([sample1, sample2],
longname='SampleContainer with Samples',
shortname='(X,Y)',
attributes=copy.copy(
self.attributes).update(
{'isSample': 'It seems so.'}))
self.sample.seal()
def makeSC(column_data, longnames, shortnames, longname, shortname, attributes={}):
unzipped = zip(*column_data)
assert len(unzipped) == len(longnames) == len(shortnames)
def get_column_fc(col, ln, sn):
try:
from pyphant.quantities import Quantity
unit = Quantity(1.0, col[0].unit)
data = [quant.value for quant in col]
except (KeyError, AttributeError):
unit = 1
data = col
from numpy import array
from pyphant.core.DataContainer import FieldContainer
fc = FieldContainer(data=array(data), unit=unit, longname=ln, shortname=sn)
return fc
columns = [get_column_fc(col, ln, sn) for col, ln, sn in zip(unzipped, longnames, shortnames)]
from pyphant.core.DataContainer import SampleContainer
sc = SampleContainer(longname=longname, shortname=shortname, attributes=attributes, columns=columns)
sc.seal()
return sc
def checkExpected(self, columns, result):
expected = SampleContainer(columns,
longname='Table',
shortname='T',
attributes=deepcopy(result.attributes))
expected.seal()
self.assertEqual(result, expected)
def makeSC(column_data,
longnames,
shortnames,
longname,
shortname,
attributes=None):
if attributes is None:
attributes = {}
unzipped = zip(*column_data)
assert len(unzipped) == len(longnames) == len(shortnames)
def get_column_fc(col, ln, sn):
try:
from pyphant.quantities import Quantity
unit = Quantity(1.0, col[0].unit)
data = [quant.value for quant in col]
except (KeyError, AttributeError):
unit = 1
data = col
from numpy import array
from pyphant.core.DataContainer import FieldContainer
fc = FieldContainer(data=array(data),
unit=unit,
longname=ln,
shortname=sn)
return fc
columns = [get_column_fc(col, ln, sn) for col, ln, sn \
in zip(unzipped, longnames, shortnames)]
from pyphant.core.DataContainer import SampleContainer
sc = SampleContainer(longname=longname,
shortname=shortname,
attributes=attributes,
columns=columns)
sc.seal()
return sc
class SampleContainerTestCase(ContainerTestCase):
def setUp(self):
super(SampleContainerTestCase, self).setUp()
self.independent = FieldContainer(
0.3 * numpy.linspace(0, 1, self.testData.shape[0]),
longname='independent variable',
shortname='x',
unit=Quantity('1 mg'),
attributes=copy.copy(self.attributes).update({'independent':
True}))
self.dependent = FieldContainer(9.81 * self.independent.data,
dimensions=[self.independent],
longname='dependent variable',
shortname='f',
unit=Quantity('9.81 nN'),
attributes=copy.copy(
self.attributes).update(
{'independent': False}))
self.sample = SampleContainer([self.dependent, self.field],
longname='Sample',
shortname='X',
attributes=copy.copy(
self.attributes).update(
{'isSample': 'It seems so.'}))
self.sample.seal()
def testSaveRestore(self):
self.eln.create_group(self.eln.root, 'testSaveRestoreSample')
saveSample(self.eln, self.eln.root.testSaveRestoreSample, self.sample)
restoredSample = loadSample(self.eln,
self.eln.root.testSaveRestoreSample)
self.assertEqual(restoredSample, self.sample)
def findMaskPoints(self, image, mask, subscriber=0):
"""
Returns a table of masked points with each row
giving a tuple (coordinate_1, ..., coordindate_n, value).
"""
self.check(image, mask)
subscriber %= 10.0
index = (mask.data == FEATURE_COLOR).nonzero()
zVal = image.data[index]
subscriber %= 60.0
fields = []
for dim, coord in enumerate(index):
newField = FieldContainer(
image.dimensions[dim].data[coord],
image.dimensions[dim].unit,
longname=image.dimensions[dim].longname + " %i" % dim,
shortname=image.dimensions[dim].shortname)
fields.append(newField)
fields.append(
FieldContainer(zVal,
image.unit,
longname=image.longname,
shortname=image.shortname))
res = SampleContainer(
fields, u"Points from %s at %s" % (image.longname, mask.longname),
u"X1")
res.seal()
subscriber %= 100.0
return res
def testSCwithSCColumn(self):
fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10)))
fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20)))
sc_child = SampleContainer(longname='sc_child', columns=[fc_child1])
sc_parent = SampleContainer(longname='sc_parent', columns=[sc_child,
fc_child2])
sc_parent.seal()
km = KnowledgeManager.getInstance()
km.registerDataContainer(sc_parent, temporary=True)
lnlist = km.search(['longname'], {'col_of':{'longname':'sc_parent'}})
lnlist = [entry[0] for entry in lnlist]
assert len(lnlist) == 2
assert 'fc_child2' in lnlist
assert 'sc_child' in lnlist
def testSCwithSCColumn(self):
fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10)))
fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20)))
sc_child = SampleContainer(longname='sc_child', columns=[fc_child1])
sc_parent = SampleContainer(longname='sc_parent',
columns=[sc_child, fc_child2])
sc_parent.seal()
km = KnowledgeManager.getInstance()
km.registerDataContainer(sc_parent, temporary=True)
lnlist = km.search(['longname'], {'col_of': {'longname': 'sc_parent'}})
lnlist = [entry[0] for entry in lnlist]
assert len(lnlist) == 2
assert 'fc_child2' in lnlist
assert 'sc_child' in lnlist
def setUp(self):
super(SampleContainerTestCase,self).setUp()
self.independent = FieldContainer(0.3*numpy.linspace(0,1,self.testData.shape[0]),
longname='independent variable',
shortname='x',
unit = Quantity('1 mg'),
attributes = copy.copy(self.attributes).update({'independent':True}))
self.dependent = FieldContainer(9.81*self.independent.data,
dimensions=[self.independent],
longname='dependent variable',
shortname='f',
unit = Quantity('9.81 nN'),
attributes = copy.copy(self.attributes).update({'independent':False}))
self.sample = SampleContainer([self.dependent,self.field],longname='Sample',shortname='X',
attributes = copy.copy(self.attributes).update({'isSample':'It seems so.'}))
self.sample.seal()
def setUp(self):
SampleContainerTest.setUp(self)
time_data = numpy.array([10.0, 20.0, 30.0, 5.0, 9000.0])
time_error = numpy.array([1.0, 2.0, 3.0, 0.5, 900.0])
time_unit = Quantity("2s")
time_FC = FieldContainer(time_data, time_unit, time_error, None, None, "Zeit", "t", None, False)
length_data = numpy.array([-20.0, 0.0, 20.0, 10.0, 5.5])
length_error = numpy.array([2.0, 0.1, 2.0, 1.0, 0.5])
length_unit = Quantity("1000m")
length_FC = FieldContainer(length_data, length_unit, length_error, None, None, "Strecke", "l", None, False)
temperature_data = numpy.array(
[[10.1, 10.2, 10.3], [20.1, 20.2, 20.3], [30.1, 30.2, 30.3], [40.1, 40.2, 40.3], [50.1, 50.2, 50.3]]
)
temperature_error = numpy.array(
[[0.1, 0.2, 0.3], [1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3], [4.1, 4.2, 4.3]]
)
temperature_unit = Quantity("1mK")
temperature_FC = FieldContainer(
temperature_data, temperature_unit, temperature_error, None, None, "Temperatur", "T", None, False
)
self.sc2d = SampleContainer([length_FC, temperature_FC, time_FC], "Test Container", "TestC")
self.sc2d["t"].dimensions[0].unit = Quantity("1m")
self.sc2d["t"].dimensions[0].data = numpy.array([-20, -10, 0, 10, 20])
self.sc2d["l"].dimensions[0].unit = Quantity("5m")
self.sc2d["l"].dimensions[0].data = numpy.array([-4, -2, 0, 2, 4])
self.sc2d["T"].dimensions[0].unit = Quantity("2m")
self.sc2d["T"].dimensions[0].data = numpy.array([-10, -5, 0, 5, 10])
self.sc2d["T"].dimensions[1].unit = Quantity("10nm")
self.sc2d["T"].dimensions[1].data = numpy.array([-1, 0, 1])
def getStatistics(self, zstack, subscriber=0):
# Initialization:
longname = zstack.longname
attributes = zstack.attributes
data = zstack.data.astype(float)
zsu = ZSUnits(zstack)
del zstack
labelData = None
zsp = ZSParams(self)
zsp.estimateThreshold(data, subscriber) # 80%
inclusionList = []
inclusionDict = {}
# Calculate autofocus results:
maxi = data.shape[0]
for zid, sliceData in enumerate(data):
labelData = self.autofocus(sliceData, zsp, zsu, zid, inclusionList,
inclusionDict, labelData)
subscriber %= 80 + int(float(zid) / float(maxi) * 10.0)
# Extract statistics:
inclusionList.sort(key=lambda x: x.focus, reverse=True)
infoList = []
base = numpy.zeros(data.shape[1:], dtype=bool)
for inclusion in inclusionList:
if not inclusion.valid:
continue
infoList.append(inclusion.getInfo(zsu))
base[inclusion.fpSlices] |= inclusion.footprint
inclusion.invalidateConnected()
baseSum = float(base.sum())
baseSize = float(base.size)
baseArea = baseSize * zsu.amul * zsu.AQUANT
projectedArea = baseSum * zsu.amul * zsu.AQUANT
# Put info into SampleContainer:
attributes = {
'ZStackType': 'StatisticsSC',
#'threshold': zsp.threshold,
#'ZStackAttributes': attributes,
'detectedInclusions': len(infoList),
'baseArea': baseArea,
'projectedArea': projectedArea
}
longname = 'Statistics_' + longname
if len(infoList) == 0:
return SampleContainer(longname=longname,
columns=[],
attributes=attributes)
columnlns = [
'zPos', 'yPos', 'xPos', 'diameter', 'focus', 'label', 'zIndex',
'ySliceStart', 'ySliceStop', 'xSliceStart', 'xSliceStop'
]
columnsns = [
'z', 'y', 'x', 'd', 'f', 'l', 'zi', 'yt', 'yp', 'xt', 'xp'
]
from pyphant.core.Helpers import makeSC
resultSC = makeSC(infoList, columnlns, columnsns, longname, 's',
attributes)
subscriber %= 100
return resultSC
def setUp(self):
data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0])
unit = PQ('3s')
error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2])
longname = u'Test: FieldContainer H5FileHandler'
shortname = u'TestH5FC'
attributes = {'custom1': u'testing1...', 'custom2': u'testing2...'}
self.fc = FieldContainer(data, unit, error, None, None, longname,
shortname, attributes)
self.fc.seal()
fc2 = FieldContainer(NPArray([4003.2, 5.3, 600.9]), PQ('0.2m'), None,
None, None, 'FieldContainer 2', 'FC2')
fc2.seal()
columns = [self.fc, fc2]
longname = u'Test: SampleContainer H5FileHandler'
shortname = u'TestH5SC'
self.sc = SampleContainer(columns, longname, shortname, attributes)
self.sc.seal()
class SampleContainerTestCase(unittest.TestCase):
def setUp(self):
data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0])
unit = PQ('3s')
error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2])
longname = u'Test: FieldContainer H5FileHandler'
shortname = u'TestH5FC'
attributes = {'custom1':u'testing1...', 'custom2':u'testing2...'}
self.fc = FieldContainer(data, unit, error, None, None, longname,
shortname, attributes)
self.fc.seal()
fc2 = FieldContainer(NPArray([4003.2, 5.3, 600.9]), PQ('0.2m'), None,
None, None, 'FieldContainer 2', 'FC2')
fc2.seal()
columns = [self.fc, fc2]
longname = u'Test: SampleContainer H5FileHandler'
shortname = u'TestH5SC'
self.sc = SampleContainer(columns, longname, shortname, attributes)
self.sc.seal()
def testDimensions(self):
fc1 = FieldContainer(
numpy.array([4, 5, 6]),
unit=Quantity("1 m"),
dimensions=[FieldContainer(numpy.array([1, 2, 3]), unit=Quantity("1 s"))],
shortname="s1",
)
fc2 = FieldContainer(
numpy.array([1, 2, 3]),
unit=Quantity("1 km"),
dimensions=[FieldContainer(numpy.array([10, 20, 30]), unit=Quantity("0.1 s"))],
shortname="s2",
)
sc = SampleContainer(columns=[fc1, fc2])
expr = "col('s1') + col('s2')"
dims = sc.calcColumn(expr, "Add", "a").dimensions
self.assertEqual(dims, fc1.dimensions)
self.assertEqual(dims, fc2.dimensions)
fc2.dimensions[0].unit = Quantity("0.11 s")
self.assertRaises(ValueError, sc.calcColumn, expr, "Add", "a")
class SampleContainerInSampleContainerTestCase(SampleContainerTestCase):
def setUp(self):
super(SampleContainerInSampleContainerTestCase,self).setUp()
sample1 = SampleContainer([self.dependent,self.field],longname='First Sample',shortname='X',
attributes = copy.copy(self.attributes).update({'isSample':'It seems so.'}))
self.independent2 = FieldContainer(0.3*numpy.linspace(0,1,self.testData.shape[0]*10),
longname='independent variable',
shortname='x',
unit = Quantity('1 mg'),
attributes = copy.copy(self.attributes).update({'independent':True}))
self.dependent2 = FieldContainer(9.81*self.independent2.data,
dimensions=[self.independent2],
longname='dependent variable',
shortname='f',
unit = Quantity('9.81 nN'),
attributes = copy.copy(self.attributes).update({'independent':False}))
sample2 = SampleContainer([self.dependent2,self.independent2],longname='Second Sample',shortname='Y',
attributes = copy.copy(self.attributes).update({'sample Nr.': 2}))
self.sample = SampleContainer([sample1,sample2],longname='SampleContainer with Samples',shortname='(X,Y)',
attributes = copy.copy(self.attributes).update({'isSample':'It seems so.'}))
self.sample.seal()
class SampleContainerTestCase(ContainerTestCase):
def setUp(self):
super(SampleContainerTestCase,self).setUp()
self.independent = FieldContainer(0.3*numpy.linspace(0,1,self.testData.shape[0]),
longname='independent variable',
shortname='x',
unit = Quantity('1 mg'),
attributes = copy.copy(self.attributes).update({'independent':True}))
self.dependent = FieldContainer(9.81*self.independent.data,
dimensions=[self.independent],
longname='dependent variable',
shortname='f',
unit = Quantity('9.81 nN'),
attributes = copy.copy(self.attributes).update({'independent':False}))
self.sample = SampleContainer([self.dependent,self.field],longname='Sample',shortname='X',
attributes = copy.copy(self.attributes).update({'isSample':'It seems so.'}))
self.sample.seal()
def testSaveRestore(self):
self.eln.createGroup(self.eln.root,'testSaveRestoreSample')
saveSample(self.eln,self.eln.root.testSaveRestoreSample,self.sample)
restoredSample = loadSample(self.eln,self.eln.root.testSaveRestoreSample)
self.assertEqual(restoredSample,self.sample)
class SampleContainerSlicingTests(SampleContainerTest):
def setUp(self):
SampleContainerTest.setUp(self)
time_data = numpy.array([10.0, 20.0, 30.0, 5.0, 9000.0])
time_error = numpy.array([1.0, 2.0, 3.0, 0.5, 900.0])
time_unit = Quantity("2s")
time_FC = FieldContainer(time_data, time_unit, time_error, None, None, "Zeit", "t", None, False)
length_data = numpy.array([-20.0, 0.0, 20.0, 10.0, 5.5])
length_error = numpy.array([2.0, 0.1, 2.0, 1.0, 0.5])
length_unit = Quantity("1000m")
length_FC = FieldContainer(length_data, length_unit, length_error, None, None, "Strecke", "l", None, False)
temperature_data = numpy.array(
[[10.1, 10.2, 10.3], [20.1, 20.2, 20.3], [30.1, 30.2, 30.3], [40.1, 40.2, 40.3], [50.1, 50.2, 50.3]]
)
temperature_error = numpy.array(
[[0.1, 0.2, 0.3], [1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3], [4.1, 4.2, 4.3]]
)
temperature_unit = Quantity("1mK")
temperature_FC = FieldContainer(
temperature_data, temperature_unit, temperature_error, None, None, "Temperatur", "T", None, False
)
self.sc2d = SampleContainer([length_FC, temperature_FC, time_FC], "Test Container", "TestC")
self.sc2d["t"].dimensions[0].unit = Quantity("1m")
self.sc2d["t"].dimensions[0].data = numpy.array([-20, -10, 0, 10, 20])
self.sc2d["l"].dimensions[0].unit = Quantity("5m")
self.sc2d["l"].dimensions[0].data = numpy.array([-4, -2, 0, 2, 4])
self.sc2d["T"].dimensions[0].unit = Quantity("2m")
self.sc2d["T"].dimensions[0].data = numpy.array([-10, -5, 0, 5, 10])
self.sc2d["T"].dimensions[1].unit = Quantity("10nm")
self.sc2d["T"].dimensions[1].data = numpy.array([-1, 0, 1])
# purely one dimensional Tests:
def testConsistancy(self):
result1 = self.sampleContainer.filter('"20m" < col("i") and "80m" > col("i")')
result2 = self.sampleContainer.filter('"20m" < col("i") < "80m"')
self.assertEqual(result1[0], result2[0])
self.assertEqual(result1[1], result2[1])
def testSimpleUnicodeExpression(self):
result = self.sampleContainer.filter(u'"50m" <= col("i") < "57m"')
self.assertEqual(len(result.columns), 2)
self.assertEqual(len(result[0].data), 7)
self.assertEqual(len(result[1].data), 7)
expected = self.sampleContainer["i"][50:57]
expected.attributes = {}
result.attributes = {}
self.assertEqual(result[0], expected)
expected = self.sampleContainer["t"][50:57]
expected.attributes = {}
result.attributes = {}
self.assertEqual(result[1], expected)
def testANDExpression(self):
result = self.sampleContainer.filter('col("i") >= "20m" and col("t") <= "98.5s"')
expectedi = self.sampleContainer["i"][20:98]
expectedt = self.sampleContainer["t"][20:98]
expectedi.attributes = {}
expectedt.attributes = {}
result[0].attributes = {}
result[1].attributes = {}
self.assertEqual(result[0], expectedi)
self.assertEqual(result[1], expectedt)
# tests involving 2 dimensional FieldContainers:
def _compareExpected(self, expression, ind):
indices = numpy.array(ind)
result = self.sc2d.filter(expression)
expectedSC = copy.deepcopy(self.sc2d)
for FC in expectedSC:
FC.data = FC.data[indices]
FC.error = FC.error[indices]
FC.dimensions[0].data = FC.dimensions[0].data[indices]
self.assertEqual(result, expectedSC)
return result
def testEmpty2dExpression(self):
result = self.sc2d.filter("")
self.assertEqual(result, self.sc2d)
def testAtomar2dExpressions(self):
self._compareExpected('col("t") <= "40.0s"', [True, True, False, True, False])
self._compareExpected('col("l") < "10000m"', [True, True, False, False, True])
self._compareExpected('col("Zeit") >= "20.0s"', [True, True, True, False, True])
self._compareExpected('col("l") > "5500m"', [False, False, True, True, False])
self._compareExpected('col("t") == "18000s"', [False, False, False, False, True])
self._compareExpected('col("Strecke") != "20000m"', [True, True, False, True, True])
def testNot2dExpression(self):
self._compareExpected('not col("t") == "10s"', [True, True, True, False, True])
def testAnd2dExpression(self):
self._compareExpected('col("Zeit") == "60s" and "20000m" == col("Strecke")', [False, False, True, False, False])
def testMultiAnd2dExpression(self):
self._compareExpected(
"col('l') >= '0 km' and col('l') < '20 km' and col('t') <= '60s'", [False, True, False, True, False]
)
def testOr2dExpression(self):
self._compareExpected('col("Zeit") < "60s" or col("Strecke") == "5500m"', [True, True, False, True, True])
def testMultiOr2dExpression(self):
self._compareExpected(
"col('t') == '20s' or col('l') == '0 km' or col('t') > '1000.2 s'", [True, True, False, False, True]
)
def testMultipleCompareOpPrecedence2dExpression(self):
self._compareExpected('not "0m" <= col("l") <= "10000m"', [True, False, True, False, False])
def testColumnToColumn2dExpression(self):
self._compareExpected('col("l") == col("Strecke")', [True, True, True, True, True])
self._compareExpected('col("t") != col("Zeit")', [False, False, False, False, False])
def testIncompatibleDimensionsExpression(self):
fc1 = FieldContainer(
numpy.array([1, 2, 3]),
dimensions=[FieldContainer(numpy.array([4, 5, 6]), unit=Quantity("1m"))],
shortname="t1",
)
fc2 = copy.deepcopy(fc1)
fc2.dimensions[0].data[0] = 4.01
fc2.shortname = "t2"
incompatibleSC = SampleContainer(columns=[fc1, fc2])
self.assertRaises(ValueError, incompatibleSC.filter, "col('t1') > col('t2')")
def testStringSample(self):
# string = numpy.rec.fromrecords([(s,) for s in [u'Hello',u'World!',u'Bäh!']])
# strings =
uField = FieldContainer(scipy.array([u"Hello", u"World!", u"Bäh!"]), longname=u"Text", shortname="\gamma")
sample = SampleContainer([uField])
sample.seal()
def checkExpected(self, columns, result):
expected = SampleContainer(columns, longname="Table", shortname="T", attributes=deepcopy(result.attributes))
expected.seal()
self.assertEqual(result, expected)