def findMaskPoints(self, image, mask, subscriber=0): """ Returns a table of masked points with each row giving a tuple (coordinate_1, ..., coordindate_n, value). """ self.check(image, mask) subscriber %= 10.0 index = (mask.data == FEATURE_COLOR).nonzero() zVal = image.data[index] subscriber %= 60.0 fields = [] for dim, coord in enumerate(index): newField = FieldContainer( image.dimensions[dim].data[coord], image.dimensions[dim].unit, longname=image.dimensions[dim].longname + " %i" % dim, shortname=image.dimensions[dim].shortname ) fields.append(newField) fields.append(FieldContainer(zVal, image.unit, longname=image.longname, shortname=image.shortname) ) res = SampleContainer( fields, u"Points from %s at %s"%(image.longname, mask.longname), u"X1" ) res.seal() subscriber %= 100.0 return res
def setUp(self): super(SampleContainerInSampleContainerTestCase, self).setUp() sample1 = SampleContainer([self.dependent, self.field], longname='First Sample', shortname='X', attributes=copy.copy(self.attributes).update( {'isSample': 'It seems so.'})) self.independent2 = FieldContainer( 0.3 * numpy.linspace(0, 1, self.testData.shape[0] * 10), longname='independent variable', shortname='x', unit=Quantity('1 mg'), attributes=copy.copy(self.attributes).update({'independent': True})) self.dependent2 = FieldContainer(9.81 * self.independent2.data, dimensions=[self.independent2], longname='dependent variable', shortname='f', unit=Quantity('9.81 nN'), attributes=copy.copy( self.attributes).update( {'independent': False})) sample2 = SampleContainer([self.dependent2, self.independent2], longname='Second Sample', shortname='Y', attributes=copy.copy(self.attributes).update( {'sample Nr.': 2})) self.sample = SampleContainer([sample1, sample2], longname='SampleContainer with Samples', shortname='(X,Y)', attributes=copy.copy( self.attributes).update( {'isSample': 'It seems so.'})) self.sample.seal()
def makeSC(column_data, longnames, shortnames, longname, shortname, attributes={}): unzipped = zip(*column_data) assert len(unzipped) == len(longnames) == len(shortnames) def get_column_fc(col, ln, sn): try: from pyphant.quantities import Quantity unit = Quantity(1.0, col[0].unit) data = [quant.value for quant in col] except (KeyError, AttributeError): unit = 1 data = col from numpy import array from pyphant.core.DataContainer import FieldContainer fc = FieldContainer(data=array(data), unit=unit, longname=ln, shortname=sn) return fc columns = [get_column_fc(col, ln, sn) for col, ln, sn in zip(unzipped, longnames, shortnames)] from pyphant.core.DataContainer import SampleContainer sc = SampleContainer(longname=longname, shortname=shortname, attributes=attributes, columns=columns) sc.seal() return sc
def checkExpected(self, columns, result): expected = SampleContainer(columns, longname='Table', shortname='T', attributes=deepcopy(result.attributes)) expected.seal() self.assertEqual(result, expected)
def makeSC(column_data, longnames, shortnames, longname, shortname, attributes=None): if attributes is None: attributes = {} unzipped = zip(*column_data) assert len(unzipped) == len(longnames) == len(shortnames) def get_column_fc(col, ln, sn): try: from pyphant.quantities import Quantity unit = Quantity(1.0, col[0].unit) data = [quant.value for quant in col] except (KeyError, AttributeError): unit = 1 data = col from numpy import array from pyphant.core.DataContainer import FieldContainer fc = FieldContainer(data=array(data), unit=unit, longname=ln, shortname=sn) return fc columns = [get_column_fc(col, ln, sn) for col, ln, sn \ in zip(unzipped, longnames, shortnames)] from pyphant.core.DataContainer import SampleContainer sc = SampleContainer(longname=longname, shortname=shortname, attributes=attributes, columns=columns) sc.seal() return sc
class SampleContainerTestCase(ContainerTestCase): def setUp(self): super(SampleContainerTestCase, self).setUp() self.independent = FieldContainer( 0.3 * numpy.linspace(0, 1, self.testData.shape[0]), longname='independent variable', shortname='x', unit=Quantity('1 mg'), attributes=copy.copy(self.attributes).update({'independent': True})) self.dependent = FieldContainer(9.81 * self.independent.data, dimensions=[self.independent], longname='dependent variable', shortname='f', unit=Quantity('9.81 nN'), attributes=copy.copy( self.attributes).update( {'independent': False})) self.sample = SampleContainer([self.dependent, self.field], longname='Sample', shortname='X', attributes=copy.copy( self.attributes).update( {'isSample': 'It seems so.'})) self.sample.seal() def testSaveRestore(self): self.eln.create_group(self.eln.root, 'testSaveRestoreSample') saveSample(self.eln, self.eln.root.testSaveRestoreSample, self.sample) restoredSample = loadSample(self.eln, self.eln.root.testSaveRestoreSample) self.assertEqual(restoredSample, self.sample)
def findMaskPoints(self, image, mask, subscriber=0): """ Returns a table of masked points with each row giving a tuple (coordinate_1, ..., coordindate_n, value). """ self.check(image, mask) subscriber %= 10.0 index = (mask.data == FEATURE_COLOR).nonzero() zVal = image.data[index] subscriber %= 60.0 fields = [] for dim, coord in enumerate(index): newField = FieldContainer( image.dimensions[dim].data[coord], image.dimensions[dim].unit, longname=image.dimensions[dim].longname + " %i" % dim, shortname=image.dimensions[dim].shortname) fields.append(newField) fields.append( FieldContainer(zVal, image.unit, longname=image.longname, shortname=image.shortname)) res = SampleContainer( fields, u"Points from %s at %s" % (image.longname, mask.longname), u"X1") res.seal() subscriber %= 100.0 return res
def testSCwithSCColumn(self): fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10))) fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20))) sc_child = SampleContainer(longname='sc_child', columns=[fc_child1]) sc_parent = SampleContainer(longname='sc_parent', columns=[sc_child, fc_child2]) sc_parent.seal() km = KnowledgeManager.getInstance() km.registerDataContainer(sc_parent, temporary=True) lnlist = km.search(['longname'], {'col_of':{'longname':'sc_parent'}}) lnlist = [entry[0] for entry in lnlist] assert len(lnlist) == 2 assert 'fc_child2' in lnlist assert 'sc_child' in lnlist
def testSCwithSCColumn(self): fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10))) fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20))) sc_child = SampleContainer(longname='sc_child', columns=[fc_child1]) sc_parent = SampleContainer(longname='sc_parent', columns=[sc_child, fc_child2]) sc_parent.seal() km = KnowledgeManager.getInstance() km.registerDataContainer(sc_parent, temporary=True) lnlist = km.search(['longname'], {'col_of': {'longname': 'sc_parent'}}) lnlist = [entry[0] for entry in lnlist] assert len(lnlist) == 2 assert 'fc_child2' in lnlist assert 'sc_child' in lnlist
def setUp(self): super(SampleContainerTestCase,self).setUp() self.independent = FieldContainer(0.3*numpy.linspace(0,1,self.testData.shape[0]), longname='independent variable', shortname='x', unit = Quantity('1 mg'), attributes = copy.copy(self.attributes).update({'independent':True})) self.dependent = FieldContainer(9.81*self.independent.data, dimensions=[self.independent], longname='dependent variable', shortname='f', unit = Quantity('9.81 nN'), attributes = copy.copy(self.attributes).update({'independent':False})) self.sample = SampleContainer([self.dependent,self.field],longname='Sample',shortname='X', attributes = copy.copy(self.attributes).update({'isSample':'It seems so.'})) self.sample.seal()
def setUp(self): SampleContainerTest.setUp(self) time_data = numpy.array([10.0, 20.0, 30.0, 5.0, 9000.0]) time_error = numpy.array([1.0, 2.0, 3.0, 0.5, 900.0]) time_unit = Quantity("2s") time_FC = FieldContainer(time_data, time_unit, time_error, None, None, "Zeit", "t", None, False) length_data = numpy.array([-20.0, 0.0, 20.0, 10.0, 5.5]) length_error = numpy.array([2.0, 0.1, 2.0, 1.0, 0.5]) length_unit = Quantity("1000m") length_FC = FieldContainer(length_data, length_unit, length_error, None, None, "Strecke", "l", None, False) temperature_data = numpy.array( [[10.1, 10.2, 10.3], [20.1, 20.2, 20.3], [30.1, 30.2, 30.3], [40.1, 40.2, 40.3], [50.1, 50.2, 50.3]] ) temperature_error = numpy.array( [[0.1, 0.2, 0.3], [1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3], [4.1, 4.2, 4.3]] ) temperature_unit = Quantity("1mK") temperature_FC = FieldContainer( temperature_data, temperature_unit, temperature_error, None, None, "Temperatur", "T", None, False ) self.sc2d = SampleContainer([length_FC, temperature_FC, time_FC], "Test Container", "TestC") self.sc2d["t"].dimensions[0].unit = Quantity("1m") self.sc2d["t"].dimensions[0].data = numpy.array([-20, -10, 0, 10, 20]) self.sc2d["l"].dimensions[0].unit = Quantity("5m") self.sc2d["l"].dimensions[0].data = numpy.array([-4, -2, 0, 2, 4]) self.sc2d["T"].dimensions[0].unit = Quantity("2m") self.sc2d["T"].dimensions[0].data = numpy.array([-10, -5, 0, 5, 10]) self.sc2d["T"].dimensions[1].unit = Quantity("10nm") self.sc2d["T"].dimensions[1].data = numpy.array([-1, 0, 1])
def getStatistics(self, zstack, subscriber=0): # Initialization: longname = zstack.longname attributes = zstack.attributes data = zstack.data.astype(float) zsu = ZSUnits(zstack) del zstack labelData = None zsp = ZSParams(self) zsp.estimateThreshold(data, subscriber) # 80% inclusionList = [] inclusionDict = {} # Calculate autofocus results: maxi = data.shape[0] for zid, sliceData in enumerate(data): labelData = self.autofocus(sliceData, zsp, zsu, zid, inclusionList, inclusionDict, labelData) subscriber %= 80 + int(float(zid) / float(maxi) * 10.0) # Extract statistics: inclusionList.sort(key=lambda x: x.focus, reverse=True) infoList = [] base = numpy.zeros(data.shape[1:], dtype=bool) for inclusion in inclusionList: if not inclusion.valid: continue infoList.append(inclusion.getInfo(zsu)) base[inclusion.fpSlices] |= inclusion.footprint inclusion.invalidateConnected() baseSum = float(base.sum()) baseSize = float(base.size) baseArea = baseSize * zsu.amul * zsu.AQUANT projectedArea = baseSum * zsu.amul * zsu.AQUANT # Put info into SampleContainer: attributes = { 'ZStackType': 'StatisticsSC', #'threshold': zsp.threshold, #'ZStackAttributes': attributes, 'detectedInclusions': len(infoList), 'baseArea': baseArea, 'projectedArea': projectedArea } longname = 'Statistics_' + longname if len(infoList) == 0: return SampleContainer(longname=longname, columns=[], attributes=attributes) columnlns = [ 'zPos', 'yPos', 'xPos', 'diameter', 'focus', 'label', 'zIndex', 'ySliceStart', 'ySliceStop', 'xSliceStart', 'xSliceStop' ] columnsns = [ 'z', 'y', 'x', 'd', 'f', 'l', 'zi', 'yt', 'yp', 'xt', 'xp' ] from pyphant.core.Helpers import makeSC resultSC = makeSC(infoList, columnlns, columnsns, longname, 's', attributes) subscriber %= 100 return resultSC
def setUp(self): data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0]) unit = PQ('3s') error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2]) longname = u'Test: FieldContainer H5FileHandler' shortname = u'TestH5FC' attributes = {'custom1': u'testing1...', 'custom2': u'testing2...'} self.fc = FieldContainer(data, unit, error, None, None, longname, shortname, attributes) self.fc.seal() fc2 = FieldContainer(NPArray([4003.2, 5.3, 600.9]), PQ('0.2m'), None, None, None, 'FieldContainer 2', 'FC2') fc2.seal() columns = [self.fc, fc2] longname = u'Test: SampleContainer H5FileHandler' shortname = u'TestH5SC' self.sc = SampleContainer(columns, longname, shortname, attributes) self.sc.seal()
class SampleContainerTestCase(unittest.TestCase): def setUp(self): data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0]) unit = PQ('3s') error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2]) longname = u'Test: FieldContainer H5FileHandler' shortname = u'TestH5FC' attributes = {'custom1':u'testing1...', 'custom2':u'testing2...'} self.fc = FieldContainer(data, unit, error, None, None, longname, shortname, attributes) self.fc.seal() fc2 = FieldContainer(NPArray([4003.2, 5.3, 600.9]), PQ('0.2m'), None, None, None, 'FieldContainer 2', 'FC2') fc2.seal() columns = [self.fc, fc2] longname = u'Test: SampleContainer H5FileHandler' shortname = u'TestH5SC' self.sc = SampleContainer(columns, longname, shortname, attributes) self.sc.seal()
def testDimensions(self): fc1 = FieldContainer( numpy.array([4, 5, 6]), unit=Quantity("1 m"), dimensions=[FieldContainer(numpy.array([1, 2, 3]), unit=Quantity("1 s"))], shortname="s1", ) fc2 = FieldContainer( numpy.array([1, 2, 3]), unit=Quantity("1 km"), dimensions=[FieldContainer(numpy.array([10, 20, 30]), unit=Quantity("0.1 s"))], shortname="s2", ) sc = SampleContainer(columns=[fc1, fc2]) expr = "col('s1') + col('s2')" dims = sc.calcColumn(expr, "Add", "a").dimensions self.assertEqual(dims, fc1.dimensions) self.assertEqual(dims, fc2.dimensions) fc2.dimensions[0].unit = Quantity("0.11 s") self.assertRaises(ValueError, sc.calcColumn, expr, "Add", "a")
class SampleContainerInSampleContainerTestCase(SampleContainerTestCase): def setUp(self): super(SampleContainerInSampleContainerTestCase,self).setUp() sample1 = SampleContainer([self.dependent,self.field],longname='First Sample',shortname='X', attributes = copy.copy(self.attributes).update({'isSample':'It seems so.'})) self.independent2 = FieldContainer(0.3*numpy.linspace(0,1,self.testData.shape[0]*10), longname='independent variable', shortname='x', unit = Quantity('1 mg'), attributes = copy.copy(self.attributes).update({'independent':True})) self.dependent2 = FieldContainer(9.81*self.independent2.data, dimensions=[self.independent2], longname='dependent variable', shortname='f', unit = Quantity('9.81 nN'), attributes = copy.copy(self.attributes).update({'independent':False})) sample2 = SampleContainer([self.dependent2,self.independent2],longname='Second Sample',shortname='Y', attributes = copy.copy(self.attributes).update({'sample Nr.': 2})) self.sample = SampleContainer([sample1,sample2],longname='SampleContainer with Samples',shortname='(X,Y)', attributes = copy.copy(self.attributes).update({'isSample':'It seems so.'})) self.sample.seal()
class SampleContainerTestCase(ContainerTestCase): def setUp(self): super(SampleContainerTestCase,self).setUp() self.independent = FieldContainer(0.3*numpy.linspace(0,1,self.testData.shape[0]), longname='independent variable', shortname='x', unit = Quantity('1 mg'), attributes = copy.copy(self.attributes).update({'independent':True})) self.dependent = FieldContainer(9.81*self.independent.data, dimensions=[self.independent], longname='dependent variable', shortname='f', unit = Quantity('9.81 nN'), attributes = copy.copy(self.attributes).update({'independent':False})) self.sample = SampleContainer([self.dependent,self.field],longname='Sample',shortname='X', attributes = copy.copy(self.attributes).update({'isSample':'It seems so.'})) self.sample.seal() def testSaveRestore(self): self.eln.createGroup(self.eln.root,'testSaveRestoreSample') saveSample(self.eln,self.eln.root.testSaveRestoreSample,self.sample) restoredSample = loadSample(self.eln,self.eln.root.testSaveRestoreSample) self.assertEqual(restoredSample,self.sample)
class SampleContainerSlicingTests(SampleContainerTest): def setUp(self): SampleContainerTest.setUp(self) time_data = numpy.array([10.0, 20.0, 30.0, 5.0, 9000.0]) time_error = numpy.array([1.0, 2.0, 3.0, 0.5, 900.0]) time_unit = Quantity("2s") time_FC = FieldContainer(time_data, time_unit, time_error, None, None, "Zeit", "t", None, False) length_data = numpy.array([-20.0, 0.0, 20.0, 10.0, 5.5]) length_error = numpy.array([2.0, 0.1, 2.0, 1.0, 0.5]) length_unit = Quantity("1000m") length_FC = FieldContainer(length_data, length_unit, length_error, None, None, "Strecke", "l", None, False) temperature_data = numpy.array( [[10.1, 10.2, 10.3], [20.1, 20.2, 20.3], [30.1, 30.2, 30.3], [40.1, 40.2, 40.3], [50.1, 50.2, 50.3]] ) temperature_error = numpy.array( [[0.1, 0.2, 0.3], [1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3], [4.1, 4.2, 4.3]] ) temperature_unit = Quantity("1mK") temperature_FC = FieldContainer( temperature_data, temperature_unit, temperature_error, None, None, "Temperatur", "T", None, False ) self.sc2d = SampleContainer([length_FC, temperature_FC, time_FC], "Test Container", "TestC") self.sc2d["t"].dimensions[0].unit = Quantity("1m") self.sc2d["t"].dimensions[0].data = numpy.array([-20, -10, 0, 10, 20]) self.sc2d["l"].dimensions[0].unit = Quantity("5m") self.sc2d["l"].dimensions[0].data = numpy.array([-4, -2, 0, 2, 4]) self.sc2d["T"].dimensions[0].unit = Quantity("2m") self.sc2d["T"].dimensions[0].data = numpy.array([-10, -5, 0, 5, 10]) self.sc2d["T"].dimensions[1].unit = Quantity("10nm") self.sc2d["T"].dimensions[1].data = numpy.array([-1, 0, 1]) # purely one dimensional Tests: def testConsistancy(self): result1 = self.sampleContainer.filter('"20m" < col("i") and "80m" > col("i")') result2 = self.sampleContainer.filter('"20m" < col("i") < "80m"') self.assertEqual(result1[0], result2[0]) self.assertEqual(result1[1], result2[1]) def testSimpleUnicodeExpression(self): result = self.sampleContainer.filter(u'"50m" <= col("i") < "57m"') self.assertEqual(len(result.columns), 2) self.assertEqual(len(result[0].data), 7) self.assertEqual(len(result[1].data), 7) expected = self.sampleContainer["i"][50:57] expected.attributes = {} result.attributes = {} self.assertEqual(result[0], expected) expected = self.sampleContainer["t"][50:57] expected.attributes = {} result.attributes = {} self.assertEqual(result[1], expected) def testANDExpression(self): result = self.sampleContainer.filter('col("i") >= "20m" and col("t") <= "98.5s"') expectedi = self.sampleContainer["i"][20:98] expectedt = self.sampleContainer["t"][20:98] expectedi.attributes = {} expectedt.attributes = {} result[0].attributes = {} result[1].attributes = {} self.assertEqual(result[0], expectedi) self.assertEqual(result[1], expectedt) # tests involving 2 dimensional FieldContainers: def _compareExpected(self, expression, ind): indices = numpy.array(ind) result = self.sc2d.filter(expression) expectedSC = copy.deepcopy(self.sc2d) for FC in expectedSC: FC.data = FC.data[indices] FC.error = FC.error[indices] FC.dimensions[0].data = FC.dimensions[0].data[indices] self.assertEqual(result, expectedSC) return result def testEmpty2dExpression(self): result = self.sc2d.filter("") self.assertEqual(result, self.sc2d) def testAtomar2dExpressions(self): self._compareExpected('col("t") <= "40.0s"', [True, True, False, True, False]) self._compareExpected('col("l") < "10000m"', [True, True, False, False, True]) self._compareExpected('col("Zeit") >= "20.0s"', [True, True, True, False, True]) self._compareExpected('col("l") > "5500m"', [False, False, True, True, False]) self._compareExpected('col("t") == "18000s"', [False, False, False, False, True]) self._compareExpected('col("Strecke") != "20000m"', [True, True, False, True, True]) def testNot2dExpression(self): self._compareExpected('not col("t") == "10s"', [True, True, True, False, True]) def testAnd2dExpression(self): self._compareExpected('col("Zeit") == "60s" and "20000m" == col("Strecke")', [False, False, True, False, False]) def testMultiAnd2dExpression(self): self._compareExpected( "col('l') >= '0 km' and col('l') < '20 km' and col('t') <= '60s'", [False, True, False, True, False] ) def testOr2dExpression(self): self._compareExpected('col("Zeit") < "60s" or col("Strecke") == "5500m"', [True, True, False, True, True]) def testMultiOr2dExpression(self): self._compareExpected( "col('t') == '20s' or col('l') == '0 km' or col('t') > '1000.2 s'", [True, True, False, False, True] ) def testMultipleCompareOpPrecedence2dExpression(self): self._compareExpected('not "0m" <= col("l") <= "10000m"', [True, False, True, False, False]) def testColumnToColumn2dExpression(self): self._compareExpected('col("l") == col("Strecke")', [True, True, True, True, True]) self._compareExpected('col("t") != col("Zeit")', [False, False, False, False, False]) def testIncompatibleDimensionsExpression(self): fc1 = FieldContainer( numpy.array([1, 2, 3]), dimensions=[FieldContainer(numpy.array([4, 5, 6]), unit=Quantity("1m"))], shortname="t1", ) fc2 = copy.deepcopy(fc1) fc2.dimensions[0].data[0] = 4.01 fc2.shortname = "t2" incompatibleSC = SampleContainer(columns=[fc1, fc2]) self.assertRaises(ValueError, incompatibleSC.filter, "col('t1') > col('t2')")
def testStringSample(self): # string = numpy.rec.fromrecords([(s,) for s in [u'Hello',u'World!',u'Bäh!']]) # strings = uField = FieldContainer(scipy.array([u"Hello", u"World!", u"Bäh!"]), longname=u"Text", shortname="\gamma") sample = SampleContainer([uField]) sample.seal()
def checkExpected(self, columns, result): expected = SampleContainer(columns, longname="Table", shortname="T", attributes=deepcopy(result.attributes)) expected.seal() self.assertEqual(result, expected)