def gradientWorker(self, image, subscriber=0):
d0 = image.dimensions[0]
dims = [d0] + [d.inUnitsOf(d0) for d in image.dimensions[1:]]
data = image.data.astype(float)
gradient = np.gradient(data)
magnitude = np.zeros_like(data)
for i, (dim, grad) in enumerate(zip(dims, gradient)):
shape = [1, ] * len(dims)
shape[i] = dim.data.shape[0]
grad /= np.gradient(dim.data).reshape(shape)
magnitude += grad ** 2
magnitude = np.sqrt(magnitude)
longname = "Gradient"
from pyphant.core.DataContainer import FieldContainer
result = FieldContainer(
magnitude,
image.unit / d0.unit,
None,
copy.deepcopy(image.mask),
copy.deepcopy(image.dimensions),
longname,
image.shortname,
copy.deepcopy(image.attributes),
False)
result.seal()
return result
class CoverageTestCase(unittest.TestCase):
def setUp(self):
from pyphant.core.DataContainer import FieldContainer
data = numpy.arange(0, 256, 1).reshape((16, 16))
self.image = FieldContainer(data, unit=1e10)
self.image.seal()
def testRatiosUpToTenPercentError(self):
delta = .1
from ImageProcessing.CoverageWorker import CoverageWorker
from ImageProcessing import FEATURE_COLOR, BACKGROUND_COLOR
from pyphant.quantities import Quantity
cworker = CoverageWorker()
for rho1Fac in [0.1, 0.25, 0.5, 1.0, 1.5, 2.0, 5.0, 10.0]:
rho1 = '%s mC / m ** 3' % (3000. * rho1Fac, )
rho2 = '3 C / m ** 3'
cworker.paramRho1.value = rho1
cworker.paramRho2.value = rho2
rho1 = Quantity(rho1)
rho2 = Quantity(rho2)
for ratio in numpy.arange(0.0, 1.01, 0.01):
cworker.paramW1.value = '%s%%' % (ratio * 100., )
result = cworker.threshold(self.image)
self.assertEqual(result.dimensions, self.image.dimensions)
stuff1 = numpy.where(result.data == FEATURE_COLOR,
True, False).sum()
stuff2 = numpy.where(result.data == BACKGROUND_COLOR,
True, False).sum()
self.assertEqual(stuff1 + stuff2, result.data.size)
stuff1 = (stuff1 * rho1).inUnitsOf('C / m ** 3').value
stuff2 = (stuff2 * rho2).inUnitsOf('C / m ** 3').value
actualRatio = stuff1 / (stuff1 + stuff2)
self.assertTrue(abs(ratio - actualRatio) <= delta)
def testNonUniformAxes(self):
im = np.array(
[
[0., 1., 2.],
[30., 10., 50.],
[8., 9., 6.],
[-10., 0., 22.]
]
)
x = FieldContainer(np.array([1., 10., 200.]), unit=Quantity('1 m'))
y = FieldContainer(np.array([0., 2., 4., 8.]), unit=Quantity('1 cm'))
fc = FieldContainer(im, unit=Quantity('5 V'), dimensions=[y, x])
fc.seal()
grad_y, grad_x = np.gradient(fc.data)
grad_y /= np.gradient(y.data).reshape((4, 1))
grad_x /= np.gradient(x.data).reshape((1, 3))
grad_y = FieldContainer(
grad_y, unit=fc.unit / y.unit,
dimensions=copy.deepcopy(fc.dimensions)
)
grad_x = FieldContainer(
grad_x, unit=fc.unit / x.unit,
dimensions=copy.deepcopy(fc.dimensions)
)
grad_x = grad_x.inUnitsOf(grad_y)
expected_result = FieldContainer(
(grad_x.data ** 2 + grad_y.data ** 2) ** 0.5,
unit=copy.deepcopy(grad_y.unit),
dimensions=copy.deepcopy(fc.dimensions)
)
result = Gradient().gradientWorker(fc)
self.assertEqual(expected_result, result)
class CoverageTestCase(unittest.TestCase):
def setUp(self):
from pyphant.core.DataContainer import FieldContainer
data = numpy.arange(0, 256, 1).reshape((16, 16))
self.image = FieldContainer(data, unit=1e10)
self.image.seal()
def testRatiosUpToTenPercentError(self):
delta = .1
from ImageProcessing.CoverageWorker import CoverageWorker
from ImageProcessing import FEATURE_COLOR, BACKGROUND_COLOR
from pyphant.quantities import Quantity
cworker = CoverageWorker()
for rho1Fac in [0.1, 0.25, 0.5, 1.0, 1.5, 2.0, 5.0, 10.0]:
rho1 = '%s mC / m ** 3' % (3000. * rho1Fac, )
rho2 = '3 C / m ** 3'
cworker.paramRho1.value = rho1
cworker.paramRho2.value = rho2
rho1 = Quantity(rho1)
rho2 = Quantity(rho2)
for ratio in numpy.arange(0.0, 1.01, 0.01):
cworker.paramW1.value = '%s%%' % (ratio * 100., )
result = cworker.threshold(self.image)
self.assertEqual(result.dimensions, self.image.dimensions)
stuff1 = numpy.where(result.data == FEATURE_COLOR, True,
False).sum()
stuff2 = numpy.where(result.data == BACKGROUND_COLOR, True,
False).sum()
self.assertEqual(stuff1 + stuff2, result.data.size)
stuff1 = (stuff1 * rho1).inUnitsOf('C / m ** 3').value
stuff2 = (stuff2 * rho2).inUnitsOf('C / m ** 3').value
actualRatio = stuff1 / (stuff1 + stuff2)
self.assertTrue(abs(ratio - actualRatio) <= delta)
def testSeal(self):
field = FieldContainer(self.testData, 1, longname=self.longname, shortname=self.shortname)
field.seal()
self.assertNotEqual(field.id, None)
self.assertNotEqual(field.hash, None)
try:
field.data = scipy.array([1, 2, 3])
except TypeError, e:
pass
def testUnits(self):
data = (np.arange(0, 256, .01)).reshape((80, 320))
image = FieldContainer(data, unit=Quantity('1 mJ'))
for dim in image.dimensions:
dim.unit = Quantity('1 cm')
image.seal()
gradient = Gradient()
result = gradient.gradientWorker(image)
self.assertEqual(result.dimensions, image.dimensions)
self.assertEqual(result.unit, Quantity('1 mJ / cm'))
class FieldContainerTestCase(unittest.TestCase):
def setUp(self):
data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0])
unit = PQ('3s')
error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2])
longname = u'Test: FieldContainer H5FileHandler'
shortname = u'TestH5FC'
attributes = {'custom1':u'testing1...', 'custom2':u'testing2...'}
self.fc = FieldContainer(data, unit, error, None, None, longname,
shortname, attributes)
self.fc.seal()
def testDeepcopy(self):
field = FieldContainer(self.testData, 1, longname=self.longname, shortname=self.shortname)
copiedField = copy.deepcopy(field)
self.assertEqual(field, copiedField)
field.seal()
copiedField.seal()
self.assertEqual(field, copiedField)
# equal because only real data is considered:
self.assertEqual(field.hash, copiedField.hash)
# unique due to timestamp in dimension ids:
self.assertNotEqual(field.id, copiedField.id)
def testDateTime(self):
"""Test the correct saving and restoring of object arrays composed from datetime objects."""
objectArray = numpy.array([datetime.datetime.now() for i in range(10)])
objectField = FieldContainer(objectArray,longname=u"timestamp",
shortname='t')
objectField.seal()
self.eln.createGroup(self.eln.root,'testObjectFields')
saveField(self.eln,self.eln.root.testObjectFields,objectField)
restoredField = loadField(self.eln,self.eln.root.testObjectFields)
for i,j in zip(restoredField.data.tolist(),objectField.data.tolist()):
self.assertEqual(i,j,'Expected %s but got %s!' % (j,i))
class FieldContainerTestCase(unittest.TestCase):
def setUp(self):
data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0])
unit = PQ('3s')
error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2])
longname = u'Test: FieldContainer H5FileHandler'
shortname = u'TestH5FC'
attributes = {'custom1': u'testing1...', 'custom2': u'testing2...'}
self.fc = FieldContainer(data, unit, error, None, None, longname,
shortname, attributes)
self.fc.seal()
def find(self, image, subscriber=0):
newdata = self.findExtrema(image.data)
longname = "FindLocalExtrema"
from pyphant.core.DataContainer import FieldContainer
result = FieldContainer(newdata, copy.deepcopy(image.unit),
copy.deepcopy(image.error),
copy.deepcopy(image.mask),
copy.deepcopy(image.dimensions),
longname, image.shortname,
copy.deepcopy(image.attributes), False)
result.seal()
return result
def testUnits(self):
from ImageProcessing.Gradient import Gradient
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = (numpy.arange(0, 256, .01)).reshape((80, 320))
image = FieldContainer(data, unit=Quantity('1 mJ'))
for dim in image.dimensions:
dim.unit = Quantity('1 cm')
image.seal()
gradient = Gradient()
result = gradient.gradientWorker(image)
self.assertEqual(result.dimensions, image.dimensions)
self.assertEqual(result.unit, Quantity('1 mJ / cm'))
def invert(self, image, subscriber=0):
max = scipy.amax(image.data)
min = scipy.amin(image.data)
data = max + min - image.data
from pyphant.core.DataContainer import FieldContainer
result = FieldContainer(data, unit=image.unit,
dimensions=copy.deepcopy(image.dimensions),
mask=copy.deepcopy(image.mask),
error=copy.deepcopy(image.error),
longname=image.longname,
shortname=image.shortname)
result.seal()
return result
def testUnicodeFields(self):
self.field.seal()
unicodeArray = numpy.array([u'Hallo World!',u'Hallo Wörld!'])
unicodeField = FieldContainer(unicodeArray,longname=u"blabla",
shortname=self.shortname,
unit = 1,
attributes = self.attributes
)
unicodeField.seal()
self.eln.createGroup(self.eln.root,'testUnicodeFields')
saveField(self.eln,self.eln.root.testUnicodeFields,unicodeField)
restoredField = loadField(self.eln,self.eln.root.testUnicodeFields)
self.assertEqual(restoredField,unicodeField,
"Restored unicode string is %s (%s) but is expected to be %s (%s)." % (restoredField.data,restoredField.data.dtype,unicodeField.data,unicodeField.data.dtype))
def invert(self, image, subscriber=0):
max = scipy.amax(image.data)
min = scipy.amin(image.data)
data = max + min - image.data
from pyphant.core.DataContainer import FieldContainer
result = FieldContainer(data,
unit=image.unit,
dimensions=copy.deepcopy(image.dimensions),
mask=copy.deepcopy(image.mask),
error=copy.deepcopy(image.error),
longname=image.longname,
shortname=image.shortname)
result.seal()
return result
def loadImageAsGreyScale(self, subscriber=0):
import os
import re
from scipy.misc import imread
import numpy
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
path = os.path.realpath(self.paramPath.value)
if os.path.isfile(path):
path = os.path.dirname(path)
pattern = re.compile(self.paramRegex.value)
filenames = filter(lambda x: pattern.match(x) is not None,
os.listdir(path))
filenames.sort()
filenames = [os.path.join(path, fname) for fname in filenames]
print path
zClip = self.getClip(self.paramZClip.value)
filenames = filenames[zClip[0]:zClip[1]]
assert len(filenames) >= 1
yClip = self.getClip(self.paramYClip.value)
xClip = self.getClip(self.paramXClip.value)
dtype = self.paramDtype.value
data = []
for i, fn in enumerate(filenames):
subscriber %= 1 + 99 * i / len(filenames)
data.append(imread(fn, True)[yClip[0]:yClip[1], xClip[0]:xClip[1]])
data = numpy.array(data, dtype=dtype)
axes = ['z', 'y', 'x']
dimensions = [
self.getDimension(a, data.shape[i]) for i, a in enumerate(axes)
]
try:
unit = Quantity(self.paramFieldUnit.value)
except AttributeError:
unit = self.paramFieldUnit.value
longname = self.paramLongname.value
shortname = self.paramShortname.value
image = FieldContainer(data=data,
dimensions=dimensions,
unit=unit,
longname=longname,
shortname=shortname,
attributes={
'yFactor': Quantity(self.paramDy.value),
'xFactor': Quantity(self.paramDx.value)
})
image.seal()
subscriber %= 100
return image
def find(self, image, subscriber=0):
newdata = self.findExtrema(image.data)
longname = "FindLocalExtrema"
from pyphant.core.DataContainer import FieldContainer
result = FieldContainer(
newdata,
copy.deepcopy(image.unit),
copy.deepcopy(image.error),
copy.deepcopy(image.mask),
copy.deepcopy(image.dimensions),
longname,
image.shortname,
copy.deepcopy(image.attributes),
False)
result.seal()
return result
def testInvert(self):
from ImageProcessing.InvertWorker import InvertWorker
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = numpy.ones((100, 100), dtype='uint8') * 112
data[0][0] = 0
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('1 cm')
image.seal()
invert = InvertWorker()
result = invert.invert(image)
self.assertEqual(result.dimensions, image.dimensions)
expected = numpy.zeros((100, 100), dtype='uint8')
expected[0][0] = 112
self.assertTrue((result.data == expected).all())
def testInvert(self):
from ImageProcessing.InvertWorker import InvertWorker
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = numpy.ones((100, 100), dtype='uint8') * 112
data[0][0] = 0
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('1 cm')
image.seal()
invert = InvertWorker()
result = invert.invert(image)
self.assertEqual(result.dimensions, image.dimensions)
expected = numpy.zeros((100, 100), dtype='uint8')
expected[0][0] = 112
self.assertTrue((result.data == expected).all())
def loadImageAsGreyScale(self, subscriber=0):
import os
import re
from scipy.misc import imread
import numpy
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
path = os.path.realpath(self.paramPath.value)
if os.path.isfile(path):
path = os.path.dirname(path)
pattern = re.compile(self.paramRegex.value)
filenames = filter(
lambda x: pattern.match(x) is not None, os.listdir(path)
)
filenames.sort()
filenames = [os.path.join(path, fname) for fname in filenames]
print path
zClip = self.getClip(self.paramZClip.value)
filenames = filenames[zClip[0]:zClip[1]]
assert len(filenames) >= 1
yClip = self.getClip(self.paramYClip.value)
xClip = self.getClip(self.paramXClip.value)
dtype = self.paramDtype.value
data = []
for i, fn in enumerate(filenames):
subscriber %= 1 + 99 * i / len(filenames)
data.append(imread(fn, True)[yClip[0]:yClip[1], xClip[0]:xClip[1]])
data = numpy.array(data, dtype=dtype)
axes = ['z', 'y', 'x']
dimensions = [
self.getDimension(a, data.shape[i]) for i, a in enumerate(axes)
]
try:
unit = Quantity(self.paramFieldUnit.value)
except AttributeError:
unit = self.paramFieldUnit.value
longname = self.paramLongname.value
shortname = self.paramShortname.value
image = FieldContainer(
data=data, dimensions=dimensions, unit=unit,
longname=longname, shortname=shortname,
attributes={'yFactor':Quantity(self.paramDy.value),
'xFactor':Quantity(self.paramDx.value)}
)
image.seal()
subscriber %= 100
return image
def testFindExtrPoint(self):
from ImageProcessing.FindLocalExtrema import FindLocalExtrema
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = numpy.ones((100, 100), dtype='uint8') * 112
data[10][20] = 255
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('2 mum')
image.seal()
fle = FindLocalExtrema()
fle.paramExcolor.value = 1
result = fle.find(image)
self.assertEqual(result.dimensions, image.dimensions)
expected = numpy.zeros((100, 100), dtype='uint8')
expected[10][20] = 1
self.assertTrue((result.data == expected).all())
def testFindExtrPoint(self):
from ImageProcessing.FindLocalExtrema import FindLocalExtrema
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = numpy.ones((100, 100), dtype='uint8') * 112
data[10][20] = 255
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('2 mum')
image.seal()
fle = FindLocalExtrema()
fle.paramExcolor.value = 1
result = fle.find(image)
self.assertEqual(result.dimensions, image.dimensions)
expected = numpy.zeros((100, 100), dtype='uint8')
expected[10][20] = 1
self.assertTrue((result.data == expected).all())
def setUp(self):
data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0])
unit = PQ('3s')
error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2])
longname = u'Test: FieldContainer H5FileHandler'
shortname = u'TestH5FC'
attributes = {'custom1':u'testing1...', 'custom2':u'testing2...'}
self.fc = FieldContainer(data, unit, error, None, None, longname,
shortname, attributes)
self.fc.seal()
fc2 = FieldContainer(NPArray([4003.2, 5.3, 600.9]), PQ('0.2m'), None,
None, None, 'FieldContainer 2', 'FC2')
fc2.seal()
columns = [self.fc, fc2]
longname = u'Test: SampleContainer H5FileHandler'
shortname = u'TestH5SC'
self.sc = SampleContainer(columns, longname, shortname, attributes)
self.sc.seal()
def setUp(self):
data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0])
unit = PQ('3s')
error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2])
longname = u'Test: FieldContainer H5FileHandler'
shortname = u'TestH5FC'
attributes = {'custom1': u'testing1...', 'custom2': u'testing2...'}
self.fc = FieldContainer(data, unit, error, None, None, longname,
shortname, attributes)
self.fc.seal()
fc2 = FieldContainer(NPArray([4003.2, 5.3, 600.9]), PQ('0.2m'), None,
None, None, 'FieldContainer 2', 'FC2')
fc2.seal()
columns = [self.fc, fc2]
longname = u'Test: SampleContainer H5FileHandler'
shortname = u'TestH5SC'
self.sc = SampleContainer(columns, longname, shortname, attributes)
self.sc.seal()
def add_noise(self, input_fc, subscriber=0):
width = parseFCUnit(self.paramWidth.value)
scale = float(width / input_fc.unit)
noisy_data = input_fc.data + normal(
scale=scale, size=input_fc.data.shape
)
output_fc = FieldContainer(
data=noisy_data,
unit=deepcopy(input_fc.unit),
dimensions=deepcopy(input_fc.dimensions),
longname=input_fc.longname + u" with noise",
shortname=input_fc.shortname,
error=deepcopy(input_fc.error),
mask=deepcopy(input_fc.mask),
attributes=deepcopy(input_fc.attributes)
)
output_fc.seal()
return output_fc
def threshold(self, image, subscriber=0):
from pyphant.quantities.ParseQuantities import parseQuantity
w1 = parseQuantity(self.paramW1.value)[0]
rho1 = parseQuantity(self.paramRho1.value)[0]
rho2 = parseQuantity(self.paramRho2.value)[0]
coveragePercent = weight2Coverage(w1, rho1, rho2)
th = calculateThreshold(image, coveragePercent)
import scipy
from ImageProcessing import (FEATURE_COLOR, BACKGROUND_COLOR)
import copy
from pyphant.core.DataContainer import FieldContainer
resultArray = scipy.where(image.data < th, FEATURE_COLOR,
BACKGROUND_COLOR)
result = FieldContainer(resultArray,
dimensions=copy.deepcopy(image.dimensions),
longname=u"Binary Image",
shortname=u"B")
result.seal()
return result
def testETFR(self):
from ImageProcessing.EdgeTouchingFeatureRemover \
import EdgeTouchingFeatureRemover
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
from ImageProcessing import BACKGROUND_COLOR, FEATURE_COLOR
data = numpy.ones((10, 10), dtype='uint8') * BACKGROUND_COLOR
data[5:10, 3:8] = FEATURE_COLOR
data[1:4, 3:8] = FEATURE_COLOR
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('2 mum')
image.seal()
etfr = EdgeTouchingFeatureRemover()
result = etfr.fillFeatures(image)
self.assertEqual(result.dimensions, image.dimensions)
expected = numpy.ones((10, 10), dtype='uint8') * BACKGROUND_COLOR
expected[1:4, 3:8] = FEATURE_COLOR
self.assertTrue((expected == result.data).all())
def gradientWorker(self, image, subscriber=0):
for dim in image.dimensions:
assert dim.unit == image.dimensions[0].unit, \
"Non-uniform dimensions!"
newdata = gradient(image.data.astype(float))
longname = "Gradient"
from pyphant.core.DataContainer import FieldContainer
result = FieldContainer(
newdata,
image.unit / image.dimensions[0].unit,
None,
copy.deepcopy(image.mask),
copy.deepcopy(image.dimensions),
longname,
image.shortname,
copy.deepcopy(image.attributes),
False)
result.seal()
return result
def testETFR(self):
from ImageProcessing.EdgeTouchingFeatureRemover \
import EdgeTouchingFeatureRemover
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
from ImageProcessing import BACKGROUND_COLOR, FEATURE_COLOR
data = numpy.ones((10, 10), dtype='uint8') * BACKGROUND_COLOR
data[5:10,3:8] = FEATURE_COLOR
data[1:4,3:8] = FEATURE_COLOR
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('2 mum')
image.seal()
etfr = EdgeTouchingFeatureRemover()
result = etfr.fillFeatures(image)
self.assertEqual(result.dimensions, image.dimensions)
expected = numpy.ones((10, 10), dtype='uint8') * BACKGROUND_COLOR
expected[1:4,3:8] = FEATURE_COLOR
self.assertTrue((expected == result.data).all())
def threshold(self, image, subscriber=0):
from pyphant.quantities.ParseQuantities import parseQuantity
w1 = parseQuantity(self.paramW1.value)[0]
rho1 = parseQuantity(self.paramRho1.value)[0]
rho2 = parseQuantity(self.paramRho2.value)[0]
coveragePercent = weight2Coverage(w1, rho1, rho2)
th = calculateThreshold(image, coveragePercent)
import scipy
from ImageProcessing import (FEATURE_COLOR, BACKGROUND_COLOR)
import copy
from pyphant.core.DataContainer import FieldContainer
resultArray = scipy.where(image.data < th,
FEATURE_COLOR,
BACKGROUND_COLOR)
result = FieldContainer(resultArray,
dimensions=copy.deepcopy(image.dimensions),
longname=u"Binary Image", shortname=u"B")
result.seal()
return result
def testUltimatePoints(self):
from ImageProcessing.UltimatePointsCalculator \
import UltimatePointsCalculator
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = numpy.zeros((10, 10), dtype='uint8')
data[1][2] = 1
data[5][3] = 2
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('1 mum')
image.seal()
upc = UltimatePointsCalculator()
result = upc.findUltimatePoints(image)
self.assertEqual(result['i'].unit, Quantity('1 mum'))
self.assertEqual(result['j'].unit, Quantity('1 mum'))
indices = zip(result['j'].data, result['i'].data)
self.assertTrue((1, 2) in indices)
self.assertTrue((5, 3) in indices)
self.assertEqual(len(indices), 2)
def testUltimatePoints(self):
from ImageProcessing.UltimatePointsCalculator \
import UltimatePointsCalculator
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = numpy.zeros((10, 10), dtype='uint8')
data[1][2] = 1
data[5][3] = 2
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('1 mum')
image.seal()
upc = UltimatePointsCalculator()
result = upc.findUltimatePoints(image)
self.assertEqual(result['i'].unit, Quantity('1 mum'))
self.assertEqual(result['j'].unit, Quantity('1 mum'))
indices = zip(result['j'].data, result['i'].data)
self.assertTrue((1, 2) in indices)
self.assertTrue((5, 3) in indices)
self.assertEqual(len(indices), 2)
def testWatershed(self):
from ImageProcessing.Watershed import Watershed
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = numpy.zeros((10, 10), dtype='uint8')
data[2:8, 2:8] = 1
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('1 mum')
image.seal()
data = numpy.zeros((10, 10), dtype='uint8')
data[3][3] = 1
data[4][6] = 2
markers = FieldContainer(data)
for dim in markers.dimensions:
dim.unit = Quantity('1 mum')
wshed = Watershed()
result = wshed.wsworker(image, markers)
self.assertEqual(result.dimensions, image.dimensions)
from scipy.ndimage import label
self.assertEqual(label(result.data)[1], 2)
def testWatershed(self):
from ImageProcessing.Watershed import Watershed
from pyphant.core.DataContainer import FieldContainer
from pyphant.quantities import Quantity
data = numpy.zeros((10, 10), dtype='uint8')
data[2:8, 2:8] = 1
image = FieldContainer(data)
for dim in image.dimensions:
dim.unit = Quantity('1 mum')
image.seal()
data = numpy.zeros((10, 10), dtype='uint8')
data[3][3] = 1
data[4][6] = 2
markers = FieldContainer(data)
for dim in markers.dimensions:
dim.unit = Quantity('1 mum')
wshed = Watershed()
result = wshed.wsworker(image, markers)
self.assertEqual(result.dimensions, image.dimensions)
from scipy.ndimage import label
self.assertEqual(label(result.data)[1], 2)
def gradientWorker(self, image, subscriber=0):
d0 = image.dimensions[0]
dims = [d0] + [d.inUnitsOf(d0) for d in image.dimensions[1:]]
data = image.data.astype(float)
gradient = np.gradient(data)
magnitude = np.zeros_like(data)
for i, (dim, grad) in enumerate(zip(dims, gradient)):
shape = [
1,
] * len(dims)
shape[i] = dim.data.shape[0]
grad /= np.gradient(dim.data).reshape(shape)
magnitude += grad**2
magnitude = np.sqrt(magnitude)
longname = "Gradient"
from pyphant.core.DataContainer import FieldContainer
result = FieldContainer(magnitude, image.unit / d0.unit, None,
copy.deepcopy(image.mask),
copy.deepcopy(image.dimensions),
longname, image.shortname,
copy.deepcopy(image.attributes), False)
result.seal()
return result
class KnowledgeManagerTestCase(unittest.TestCase):
def setUp(self):
a = N.array([0, 1, 2, 3])
self._fc = FieldContainer(a)
self._fc.seal()
def testGetLocalFile(self):
h5fileid, h5name = tempfile.mkstemp(suffix='.h5', prefix='test-')
os.close(h5fileid)
h5 = open_file(h5name, 'w')
resultsGroup = h5.create_group("/", "results")
ptp.saveResult(self._fc, h5)
h5.close()
km = KnowledgeManager.getInstance()
from urllib import pathname2url
url = pathname2url(h5name)
if not url.startswith('///'):
url = '//' + url
url = 'file:' + url
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
os.remove(h5name)
def testGetHTTPFile(self):
host = "pyphant.sourceforge.net"
remote_dir = ""
url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5"
# Get remote file and load DataContainer
filename, headers = urllib.urlretrieve(url)
h5 = open_file(filename, 'r')
for g in h5.walk_groups("/results"):
if (len(g._v_attrs.TITLE)>0) \
and (r"\Psi" in g._v_attrs.shortname):
http_fc = ptp.loadField(h5, g)
h5.close()
km = KnowledgeManager.getInstance()
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(http_fc.id)
self.assertEqual(http_fc, km_fc)
os.remove(filename)
def testGetDataContainer(self):
km = KnowledgeManager.getInstance()
km.registerDataContainer(self._fc, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
def testSCwithSCColumn(self):
fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10)))
fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20)))
sc_child = SampleContainer(longname='sc_child', columns=[fc_child1])
sc_parent = SampleContainer(longname='sc_parent',
columns=[sc_child, fc_child2])
sc_parent.seal()
km = KnowledgeManager.getInstance()
km.registerDataContainer(sc_parent, temporary=True)
lnlist = km.search(['longname'], {'col_of': {'longname': 'sc_parent'}})
lnlist = [entry[0] for entry in lnlist]
assert len(lnlist) == 2
assert 'fc_child2' in lnlist
assert 'sc_child' in lnlist
def testExceptions(self):
km = KnowledgeManager.getInstance()
#TODO:
#invalid id
#DataContainer not sealed
#Local file not readable
#Register empty hdf
def testRegisterFMF(self):
km = KnowledgeManager.getInstance()
fileid, filename = tempfile.mkstemp(suffix='.fmf', prefix='test-')
os.close(fileid)
handler = open(filename, 'w')
fmfstring = """; -*- fmf-version: 1.0 -*-
[*reference]
title: Knowledge Manager FMF Test
creator: Alexander Held
created: 2009-05-25 08:45:00+02:00
place: Uni Freiburg
[*data definitions]
voltage: V [V]
current: I(V) [A]
[*data]
1.0\t0.5
2.0\t1.0
3.0\t1.5
"""
handler.write(fmfstring)
handler.close()
dc_id = km.registerFMF(filename, temporary=True)
os.remove(filename)
km.getDataContainer(dc_id)
def testCache(self):
print "Preparing FCs for cache test (cache size: %d MB)..."\
% (CACHE_MAX_SIZE / 1024 / 1024)
km = KnowledgeManager.getInstance()
sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)]
byte_sizes = [reduce(lambda x, y: x * y, size) * 8 for size in sizes]
ids = []
rand_id_pool = []
assert_dict = {}
for num in xrange(2 * 20):
fc = FieldContainer(N.ones((500, 500)))
fc.data.flat[0] = num
fc.seal()
km.registerDataContainer(fc, temporary=True)
rand_id_pool.append(fc.id)
assert_dict[fc.id] = num
for size in sizes:
fc = FieldContainer(N.ones(size))
fc.seal()
km.registerDataContainer(fc, temporary=True)
ids.append(fc.id)
for id, bytes in zip(ids, byte_sizes):
uc_acc_time = 0.0
c_acc_time = 0.0
reps = 30
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
km._cache = []
km._cache_size = 0
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
print "Avr. access time for %0.2f kB sequential read: "\
"%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024,
uc_avr, c_avr)
km._cache = []
km._cache_size = 0
rand_ids = []
reps = 500
import pyphant.core.KnowledgeManager
pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20
for run in xrange(reps):
rand_ids.append(rand_id_pool[random.randint(
0,
len(rand_id_pool) - 1)])
uc_acc_time = 0.0
c_acc_time = 0.0
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
assert km._cache_size >= 0
assert km._cache_size <= CACHE_MAX_SIZE
assert len(km._cache) <= CACHE_MAX_NUMBER
assert assert_dict[id] == fc.data.flat[0]
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
bytes = float(500 * 500 * 8) / 1024
print "Avr. access time for %0.2f kB random read: "\
"%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr)
print "------ End Cache Test ------"
def testCache(self):
print "Preparing FCs for cache test (cache size: %d MB)..."\
% (CACHE_MAX_SIZE / 1024 / 1024)
km = KnowledgeManager.getInstance()
sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)]
byte_sizes = [reduce(lambda x, y: x * y, size) * 8 for size in sizes]
ids = []
rand_id_pool = []
assert_dict = {}
for num in xrange(2 * 20):
fc = FieldContainer(N.ones((500, 500)))
fc.data.flat[0] = num
fc.seal()
km.registerDataContainer(fc, temporary=True)
rand_id_pool.append(fc.id)
assert_dict[fc.id] = num
for size in sizes:
fc = FieldContainer(N.ones(size))
fc.seal()
km.registerDataContainer(fc, temporary=True)
ids.append(fc.id)
for id, bytes in zip(ids, byte_sizes):
uc_acc_time = 0.0
c_acc_time = 0.0
reps = 30
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
km._cache = []
km._cache_size = 0
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
print "Avr. access time for %0.2f kB sequential read: "\
"%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024,
uc_avr, c_avr)
km._cache = []
km._cache_size = 0
rand_ids = []
reps = 500
import pyphant.core.KnowledgeManager
pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20
for run in xrange(reps):
rand_ids.append(rand_id_pool[random.randint(
0,
len(rand_id_pool) - 1)])
uc_acc_time = 0.0
c_acc_time = 0.0
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
assert km._cache_size >= 0
assert km._cache_size <= CACHE_MAX_SIZE
assert len(km._cache) <= CACHE_MAX_NUMBER
assert assert_dict[id] == fc.data.flat[0]
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
bytes = float(500 * 500 * 8) / 1024
print "Avr. access time for %0.2f kB random read: "\
"%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr)
print "------ End Cache Test ------"
class KnowledgeManagerTestCase(unittest.TestCase):
def setUp(self):
a = N.array([0, 1, 2, 3])
self._fc = FieldContainer(a)
self._fc.seal()
def testGetLocalFile(self):
h5fileid, h5name = tempfile.mkstemp(suffix='.h5',prefix='test-')
os.close(h5fileid)
h5 = tables.openFile(h5name,'w')
resultsGroup = h5.createGroup("/", "results")
ptp.saveResult(self._fc, h5)
h5.close()
km = KnowledgeManager.getInstance()
from urllib import pathname2url
url = pathname2url(h5name)
if not url.startswith('///'):
url = '//' + url
url = 'file:' + url
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
os.remove(h5name)
def testGetHTTPFile(self):
host = "pyphant.sourceforge.net"
remote_dir = ""
url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5"
# Get remote file and load DataContainer
filename, headers = urllib.urlretrieve(url)
h5 = tables.openFile(filename, 'r')
for g in h5.walkGroups("/results"):
if (len(g._v_attrs.TITLE)>0) \
and (r"\Psi" in g._v_attrs.shortname):
http_fc = ptp.loadField(h5,g)
h5.close()
km = KnowledgeManager.getInstance()
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(http_fc.id)
self.assertEqual(http_fc, km_fc)
os.remove(filename)
def testGetDataContainer(self):
km = KnowledgeManager.getInstance()
km.registerDataContainer(self._fc, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
def testSCwithSCColumn(self):
fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10)))
fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20)))
sc_child = SampleContainer(longname='sc_child', columns=[fc_child1])
sc_parent = SampleContainer(longname='sc_parent', columns=[sc_child,
fc_child2])
sc_parent.seal()
km = KnowledgeManager.getInstance()
km.registerDataContainer(sc_parent, temporary=True)
lnlist = km.search(['longname'], {'col_of':{'longname':'sc_parent'}})
lnlist = [entry[0] for entry in lnlist]
assert len(lnlist) == 2
assert 'fc_child2' in lnlist
assert 'sc_child' in lnlist
def testExceptions(self):
km = KnowledgeManager.getInstance()
#TODO:
#invalid id
#DataContainer not sealed
#Local file not readable
#Register empty hdf
def testRegisterFMF(self):
km = KnowledgeManager.getInstance()
fileid, filename = tempfile.mkstemp(suffix='.fmf', prefix='test-')
os.close(fileid)
handler = open(filename, 'w')
fmfstring = """; -*- fmf-version: 1.0 -*-
[*reference]
title: Knowledge Manager FMF Test
creator: Alexander Held
created: 2009-05-25 08:45:00+02:00
place: Uni Freiburg
[*data definitions]
voltage: V [V]
current: I(V) [A]
[*data]
1.0\t0.5
2.0\t1.0
3.0\t1.5
"""
handler.write(fmfstring)
handler.close()
dc_id = km.registerFMF(filename, temporary=True)
os.remove(filename)
km.getDataContainer(dc_id)
def testCache(self):
print "Preparing FCs for cache test (cache size: %d MB)..."\
% (CACHE_MAX_SIZE / 1024 / 1024)
km = KnowledgeManager.getInstance()
sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)]
byte_sizes = [reduce(lambda x, y:x * y, size) * 8 for size in sizes]
ids = []
rand_id_pool = []
assert_dict = {}
for num in xrange(2 * 20):
fc = FieldContainer(N.ones((500, 500)))
fc.data.flat[0] = num
fc.seal()
km.registerDataContainer(fc, temporary=True)
rand_id_pool.append(fc.id)
assert_dict[fc.id] = num
for size in sizes:
fc = FieldContainer(N.ones(size))
fc.seal()
km.registerDataContainer(fc, temporary=True)
ids.append(fc.id)
for id, bytes in zip(ids, byte_sizes):
uc_acc_time = 0.0
c_acc_time = 0.0
reps = 30
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
km._cache = []
km._cache_size = 0
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
print "Avr. access time for %0.2f kB sequential read: "\
"%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024,
uc_avr, c_avr)
km._cache = []
km._cache_size = 0
rand_ids = []
reps = 500
import pyphant.core.KnowledgeManager
pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20
for run in xrange(reps):
rand_ids.append(rand_id_pool[
random.randint(0, len(rand_id_pool) - 1)])
uc_acc_time = 0.0
c_acc_time = 0.0
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
assert km._cache_size >= 0
assert km._cache_size <= CACHE_MAX_SIZE
assert len(km._cache) <= CACHE_MAX_NUMBER
assert assert_dict[id] == fc.data.flat[0]
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
bytes = float(500 * 500 * 8) / 1024
print "Avr. access time for %0.2f kB random read: "\
"%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr)
print "------ End Cache Test ------"
def testCache(self):
print "Preparing FCs for cache test (cache size: %d MB)..."\
% (CACHE_MAX_SIZE / 1024 / 1024)
km = KnowledgeManager.getInstance()
sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)]
byte_sizes = [reduce(lambda x, y:x * y, size) * 8 for size in sizes]
ids = []
rand_id_pool = []
assert_dict = {}
for num in xrange(2 * 20):
fc = FieldContainer(N.ones((500, 500)))
fc.data.flat[0] = num
fc.seal()
km.registerDataContainer(fc, temporary=True)
rand_id_pool.append(fc.id)
assert_dict[fc.id] = num
for size in sizes:
fc = FieldContainer(N.ones(size))
fc.seal()
km.registerDataContainer(fc, temporary=True)
ids.append(fc.id)
for id, bytes in zip(ids, byte_sizes):
uc_acc_time = 0.0
c_acc_time = 0.0
reps = 30
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
km._cache = []
km._cache_size = 0
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
print "Avr. access time for %0.2f kB sequential read: "\
"%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024,
uc_avr, c_avr)
km._cache = []
km._cache_size = 0
rand_ids = []
reps = 500
import pyphant.core.KnowledgeManager
pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20
for run in xrange(reps):
rand_ids.append(rand_id_pool[
random.randint(0, len(rand_id_pool) - 1)])
uc_acc_time = 0.0
c_acc_time = 0.0
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
assert km._cache_size >= 0
assert km._cache_size <= CACHE_MAX_SIZE
assert len(km._cache) <= CACHE_MAX_NUMBER
assert assert_dict[id] == fc.data.flat[0]
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
bytes = float(500 * 500 * 8) / 1024
print "Avr. access time for %0.2f kB random read: "\
"%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr)
print "------ End Cache Test ------"