def gradientWorker(self, image, subscriber=0): d0 = image.dimensions[0] dims = [d0] + [d.inUnitsOf(d0) for d in image.dimensions[1:]] data = image.data.astype(float) gradient = np.gradient(data) magnitude = np.zeros_like(data) for i, (dim, grad) in enumerate(zip(dims, gradient)): shape = [1, ] * len(dims) shape[i] = dim.data.shape[0] grad /= np.gradient(dim.data).reshape(shape) magnitude += grad ** 2 magnitude = np.sqrt(magnitude) longname = "Gradient" from pyphant.core.DataContainer import FieldContainer result = FieldContainer( magnitude, image.unit / d0.unit, None, copy.deepcopy(image.mask), copy.deepcopy(image.dimensions), longname, image.shortname, copy.deepcopy(image.attributes), False) result.seal() return result
class CoverageTestCase(unittest.TestCase): def setUp(self): from pyphant.core.DataContainer import FieldContainer data = numpy.arange(0, 256, 1).reshape((16, 16)) self.image = FieldContainer(data, unit=1e10) self.image.seal() def testRatiosUpToTenPercentError(self): delta = .1 from ImageProcessing.CoverageWorker import CoverageWorker from ImageProcessing import FEATURE_COLOR, BACKGROUND_COLOR from pyphant.quantities import Quantity cworker = CoverageWorker() for rho1Fac in [0.1, 0.25, 0.5, 1.0, 1.5, 2.0, 5.0, 10.0]: rho1 = '%s mC / m ** 3' % (3000. * rho1Fac, ) rho2 = '3 C / m ** 3' cworker.paramRho1.value = rho1 cworker.paramRho2.value = rho2 rho1 = Quantity(rho1) rho2 = Quantity(rho2) for ratio in numpy.arange(0.0, 1.01, 0.01): cworker.paramW1.value = '%s%%' % (ratio * 100., ) result = cworker.threshold(self.image) self.assertEqual(result.dimensions, self.image.dimensions) stuff1 = numpy.where(result.data == FEATURE_COLOR, True, False).sum() stuff2 = numpy.where(result.data == BACKGROUND_COLOR, True, False).sum() self.assertEqual(stuff1 + stuff2, result.data.size) stuff1 = (stuff1 * rho1).inUnitsOf('C / m ** 3').value stuff2 = (stuff2 * rho2).inUnitsOf('C / m ** 3').value actualRatio = stuff1 / (stuff1 + stuff2) self.assertTrue(abs(ratio - actualRatio) <= delta)
def testNonUniformAxes(self): im = np.array( [ [0., 1., 2.], [30., 10., 50.], [8., 9., 6.], [-10., 0., 22.] ] ) x = FieldContainer(np.array([1., 10., 200.]), unit=Quantity('1 m')) y = FieldContainer(np.array([0., 2., 4., 8.]), unit=Quantity('1 cm')) fc = FieldContainer(im, unit=Quantity('5 V'), dimensions=[y, x]) fc.seal() grad_y, grad_x = np.gradient(fc.data) grad_y /= np.gradient(y.data).reshape((4, 1)) grad_x /= np.gradient(x.data).reshape((1, 3)) grad_y = FieldContainer( grad_y, unit=fc.unit / y.unit, dimensions=copy.deepcopy(fc.dimensions) ) grad_x = FieldContainer( grad_x, unit=fc.unit / x.unit, dimensions=copy.deepcopy(fc.dimensions) ) grad_x = grad_x.inUnitsOf(grad_y) expected_result = FieldContainer( (grad_x.data ** 2 + grad_y.data ** 2) ** 0.5, unit=copy.deepcopy(grad_y.unit), dimensions=copy.deepcopy(fc.dimensions) ) result = Gradient().gradientWorker(fc) self.assertEqual(expected_result, result)
def testSeal(self): field = FieldContainer(self.testData, 1, longname=self.longname, shortname=self.shortname) field.seal() self.assertNotEqual(field.id, None) self.assertNotEqual(field.hash, None) try: field.data = scipy.array([1, 2, 3]) except TypeError, e: pass
def testUnits(self): data = (np.arange(0, 256, .01)).reshape((80, 320)) image = FieldContainer(data, unit=Quantity('1 mJ')) for dim in image.dimensions: dim.unit = Quantity('1 cm') image.seal() gradient = Gradient() result = gradient.gradientWorker(image) self.assertEqual(result.dimensions, image.dimensions) self.assertEqual(result.unit, Quantity('1 mJ / cm'))
class FieldContainerTestCase(unittest.TestCase): def setUp(self): data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0]) unit = PQ('3s') error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2]) longname = u'Test: FieldContainer H5FileHandler' shortname = u'TestH5FC' attributes = {'custom1':u'testing1...', 'custom2':u'testing2...'} self.fc = FieldContainer(data, unit, error, None, None, longname, shortname, attributes) self.fc.seal()
def testDeepcopy(self): field = FieldContainer(self.testData, 1, longname=self.longname, shortname=self.shortname) copiedField = copy.deepcopy(field) self.assertEqual(field, copiedField) field.seal() copiedField.seal() self.assertEqual(field, copiedField) # equal because only real data is considered: self.assertEqual(field.hash, copiedField.hash) # unique due to timestamp in dimension ids: self.assertNotEqual(field.id, copiedField.id)
def testDateTime(self): """Test the correct saving and restoring of object arrays composed from datetime objects.""" objectArray = numpy.array([datetime.datetime.now() for i in range(10)]) objectField = FieldContainer(objectArray,longname=u"timestamp", shortname='t') objectField.seal() self.eln.createGroup(self.eln.root,'testObjectFields') saveField(self.eln,self.eln.root.testObjectFields,objectField) restoredField = loadField(self.eln,self.eln.root.testObjectFields) for i,j in zip(restoredField.data.tolist(),objectField.data.tolist()): self.assertEqual(i,j,'Expected %s but got %s!' % (j,i))
class FieldContainerTestCase(unittest.TestCase): def setUp(self): data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0]) unit = PQ('3s') error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2]) longname = u'Test: FieldContainer H5FileHandler' shortname = u'TestH5FC' attributes = {'custom1': u'testing1...', 'custom2': u'testing2...'} self.fc = FieldContainer(data, unit, error, None, None, longname, shortname, attributes) self.fc.seal()
def find(self, image, subscriber=0): newdata = self.findExtrema(image.data) longname = "FindLocalExtrema" from pyphant.core.DataContainer import FieldContainer result = FieldContainer(newdata, copy.deepcopy(image.unit), copy.deepcopy(image.error), copy.deepcopy(image.mask), copy.deepcopy(image.dimensions), longname, image.shortname, copy.deepcopy(image.attributes), False) result.seal() return result
def testUnits(self): from ImageProcessing.Gradient import Gradient from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity data = (numpy.arange(0, 256, .01)).reshape((80, 320)) image = FieldContainer(data, unit=Quantity('1 mJ')) for dim in image.dimensions: dim.unit = Quantity('1 cm') image.seal() gradient = Gradient() result = gradient.gradientWorker(image) self.assertEqual(result.dimensions, image.dimensions) self.assertEqual(result.unit, Quantity('1 mJ / cm'))
def invert(self, image, subscriber=0): max = scipy.amax(image.data) min = scipy.amin(image.data) data = max + min - image.data from pyphant.core.DataContainer import FieldContainer result = FieldContainer(data, unit=image.unit, dimensions=copy.deepcopy(image.dimensions), mask=copy.deepcopy(image.mask), error=copy.deepcopy(image.error), longname=image.longname, shortname=image.shortname) result.seal() return result
def testUnicodeFields(self): self.field.seal() unicodeArray = numpy.array([u'Hallo World!',u'Hallo Wörld!']) unicodeField = FieldContainer(unicodeArray,longname=u"blabla", shortname=self.shortname, unit = 1, attributes = self.attributes ) unicodeField.seal() self.eln.createGroup(self.eln.root,'testUnicodeFields') saveField(self.eln,self.eln.root.testUnicodeFields,unicodeField) restoredField = loadField(self.eln,self.eln.root.testUnicodeFields) self.assertEqual(restoredField,unicodeField, "Restored unicode string is %s (%s) but is expected to be %s (%s)." % (restoredField.data,restoredField.data.dtype,unicodeField.data,unicodeField.data.dtype))
def loadImageAsGreyScale(self, subscriber=0): import os import re from scipy.misc import imread import numpy from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity path = os.path.realpath(self.paramPath.value) if os.path.isfile(path): path = os.path.dirname(path) pattern = re.compile(self.paramRegex.value) filenames = filter(lambda x: pattern.match(x) is not None, os.listdir(path)) filenames.sort() filenames = [os.path.join(path, fname) for fname in filenames] print path zClip = self.getClip(self.paramZClip.value) filenames = filenames[zClip[0]:zClip[1]] assert len(filenames) >= 1 yClip = self.getClip(self.paramYClip.value) xClip = self.getClip(self.paramXClip.value) dtype = self.paramDtype.value data = [] for i, fn in enumerate(filenames): subscriber %= 1 + 99 * i / len(filenames) data.append(imread(fn, True)[yClip[0]:yClip[1], xClip[0]:xClip[1]]) data = numpy.array(data, dtype=dtype) axes = ['z', 'y', 'x'] dimensions = [ self.getDimension(a, data.shape[i]) for i, a in enumerate(axes) ] try: unit = Quantity(self.paramFieldUnit.value) except AttributeError: unit = self.paramFieldUnit.value longname = self.paramLongname.value shortname = self.paramShortname.value image = FieldContainer(data=data, dimensions=dimensions, unit=unit, longname=longname, shortname=shortname, attributes={ 'yFactor': Quantity(self.paramDy.value), 'xFactor': Quantity(self.paramDx.value) }) image.seal() subscriber %= 100 return image
def find(self, image, subscriber=0): newdata = self.findExtrema(image.data) longname = "FindLocalExtrema" from pyphant.core.DataContainer import FieldContainer result = FieldContainer( newdata, copy.deepcopy(image.unit), copy.deepcopy(image.error), copy.deepcopy(image.mask), copy.deepcopy(image.dimensions), longname, image.shortname, copy.deepcopy(image.attributes), False) result.seal() return result
def testInvert(self): from ImageProcessing.InvertWorker import InvertWorker from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity data = numpy.ones((100, 100), dtype='uint8') * 112 data[0][0] = 0 image = FieldContainer(data) for dim in image.dimensions: dim.unit = Quantity('1 cm') image.seal() invert = InvertWorker() result = invert.invert(image) self.assertEqual(result.dimensions, image.dimensions) expected = numpy.zeros((100, 100), dtype='uint8') expected[0][0] = 112 self.assertTrue((result.data == expected).all())
def loadImageAsGreyScale(self, subscriber=0): import os import re from scipy.misc import imread import numpy from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity path = os.path.realpath(self.paramPath.value) if os.path.isfile(path): path = os.path.dirname(path) pattern = re.compile(self.paramRegex.value) filenames = filter( lambda x: pattern.match(x) is not None, os.listdir(path) ) filenames.sort() filenames = [os.path.join(path, fname) for fname in filenames] print path zClip = self.getClip(self.paramZClip.value) filenames = filenames[zClip[0]:zClip[1]] assert len(filenames) >= 1 yClip = self.getClip(self.paramYClip.value) xClip = self.getClip(self.paramXClip.value) dtype = self.paramDtype.value data = [] for i, fn in enumerate(filenames): subscriber %= 1 + 99 * i / len(filenames) data.append(imread(fn, True)[yClip[0]:yClip[1], xClip[0]:xClip[1]]) data = numpy.array(data, dtype=dtype) axes = ['z', 'y', 'x'] dimensions = [ self.getDimension(a, data.shape[i]) for i, a in enumerate(axes) ] try: unit = Quantity(self.paramFieldUnit.value) except AttributeError: unit = self.paramFieldUnit.value longname = self.paramLongname.value shortname = self.paramShortname.value image = FieldContainer( data=data, dimensions=dimensions, unit=unit, longname=longname, shortname=shortname, attributes={'yFactor':Quantity(self.paramDy.value), 'xFactor':Quantity(self.paramDx.value)} ) image.seal() subscriber %= 100 return image
def testFindExtrPoint(self): from ImageProcessing.FindLocalExtrema import FindLocalExtrema from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity data = numpy.ones((100, 100), dtype='uint8') * 112 data[10][20] = 255 image = FieldContainer(data) for dim in image.dimensions: dim.unit = Quantity('2 mum') image.seal() fle = FindLocalExtrema() fle.paramExcolor.value = 1 result = fle.find(image) self.assertEqual(result.dimensions, image.dimensions) expected = numpy.zeros((100, 100), dtype='uint8') expected[10][20] = 1 self.assertTrue((result.data == expected).all())
def setUp(self): data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0]) unit = PQ('3s') error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2]) longname = u'Test: FieldContainer H5FileHandler' shortname = u'TestH5FC' attributes = {'custom1':u'testing1...', 'custom2':u'testing2...'} self.fc = FieldContainer(data, unit, error, None, None, longname, shortname, attributes) self.fc.seal() fc2 = FieldContainer(NPArray([4003.2, 5.3, 600.9]), PQ('0.2m'), None, None, None, 'FieldContainer 2', 'FC2') fc2.seal() columns = [self.fc, fc2] longname = u'Test: SampleContainer H5FileHandler' shortname = u'TestH5SC' self.sc = SampleContainer(columns, longname, shortname, attributes) self.sc.seal()
def setUp(self): data = NPArray([10.0, -103.5, 1000.43, 0.0, 10.0]) unit = PQ('3s') error = NPArray([0.1, 0.2, 4.5, 0.1, 0.2]) longname = u'Test: FieldContainer H5FileHandler' shortname = u'TestH5FC' attributes = {'custom1': u'testing1...', 'custom2': u'testing2...'} self.fc = FieldContainer(data, unit, error, None, None, longname, shortname, attributes) self.fc.seal() fc2 = FieldContainer(NPArray([4003.2, 5.3, 600.9]), PQ('0.2m'), None, None, None, 'FieldContainer 2', 'FC2') fc2.seal() columns = [self.fc, fc2] longname = u'Test: SampleContainer H5FileHandler' shortname = u'TestH5SC' self.sc = SampleContainer(columns, longname, shortname, attributes) self.sc.seal()
def add_noise(self, input_fc, subscriber=0): width = parseFCUnit(self.paramWidth.value) scale = float(width / input_fc.unit) noisy_data = input_fc.data + normal( scale=scale, size=input_fc.data.shape ) output_fc = FieldContainer( data=noisy_data, unit=deepcopy(input_fc.unit), dimensions=deepcopy(input_fc.dimensions), longname=input_fc.longname + u" with noise", shortname=input_fc.shortname, error=deepcopy(input_fc.error), mask=deepcopy(input_fc.mask), attributes=deepcopy(input_fc.attributes) ) output_fc.seal() return output_fc
def threshold(self, image, subscriber=0): from pyphant.quantities.ParseQuantities import parseQuantity w1 = parseQuantity(self.paramW1.value)[0] rho1 = parseQuantity(self.paramRho1.value)[0] rho2 = parseQuantity(self.paramRho2.value)[0] coveragePercent = weight2Coverage(w1, rho1, rho2) th = calculateThreshold(image, coveragePercent) import scipy from ImageProcessing import (FEATURE_COLOR, BACKGROUND_COLOR) import copy from pyphant.core.DataContainer import FieldContainer resultArray = scipy.where(image.data < th, FEATURE_COLOR, BACKGROUND_COLOR) result = FieldContainer(resultArray, dimensions=copy.deepcopy(image.dimensions), longname=u"Binary Image", shortname=u"B") result.seal() return result
def testETFR(self): from ImageProcessing.EdgeTouchingFeatureRemover \ import EdgeTouchingFeatureRemover from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity from ImageProcessing import BACKGROUND_COLOR, FEATURE_COLOR data = numpy.ones((10, 10), dtype='uint8') * BACKGROUND_COLOR data[5:10, 3:8] = FEATURE_COLOR data[1:4, 3:8] = FEATURE_COLOR image = FieldContainer(data) for dim in image.dimensions: dim.unit = Quantity('2 mum') image.seal() etfr = EdgeTouchingFeatureRemover() result = etfr.fillFeatures(image) self.assertEqual(result.dimensions, image.dimensions) expected = numpy.ones((10, 10), dtype='uint8') * BACKGROUND_COLOR expected[1:4, 3:8] = FEATURE_COLOR self.assertTrue((expected == result.data).all())
def gradientWorker(self, image, subscriber=0): for dim in image.dimensions: assert dim.unit == image.dimensions[0].unit, \ "Non-uniform dimensions!" newdata = gradient(image.data.astype(float)) longname = "Gradient" from pyphant.core.DataContainer import FieldContainer result = FieldContainer( newdata, image.unit / image.dimensions[0].unit, None, copy.deepcopy(image.mask), copy.deepcopy(image.dimensions), longname, image.shortname, copy.deepcopy(image.attributes), False) result.seal() return result
def testETFR(self): from ImageProcessing.EdgeTouchingFeatureRemover \ import EdgeTouchingFeatureRemover from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity from ImageProcessing import BACKGROUND_COLOR, FEATURE_COLOR data = numpy.ones((10, 10), dtype='uint8') * BACKGROUND_COLOR data[5:10,3:8] = FEATURE_COLOR data[1:4,3:8] = FEATURE_COLOR image = FieldContainer(data) for dim in image.dimensions: dim.unit = Quantity('2 mum') image.seal() etfr = EdgeTouchingFeatureRemover() result = etfr.fillFeatures(image) self.assertEqual(result.dimensions, image.dimensions) expected = numpy.ones((10, 10), dtype='uint8') * BACKGROUND_COLOR expected[1:4,3:8] = FEATURE_COLOR self.assertTrue((expected == result.data).all())
def testUltimatePoints(self): from ImageProcessing.UltimatePointsCalculator \ import UltimatePointsCalculator from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity data = numpy.zeros((10, 10), dtype='uint8') data[1][2] = 1 data[5][3] = 2 image = FieldContainer(data) for dim in image.dimensions: dim.unit = Quantity('1 mum') image.seal() upc = UltimatePointsCalculator() result = upc.findUltimatePoints(image) self.assertEqual(result['i'].unit, Quantity('1 mum')) self.assertEqual(result['j'].unit, Quantity('1 mum')) indices = zip(result['j'].data, result['i'].data) self.assertTrue((1, 2) in indices) self.assertTrue((5, 3) in indices) self.assertEqual(len(indices), 2)
def testWatershed(self): from ImageProcessing.Watershed import Watershed from pyphant.core.DataContainer import FieldContainer from pyphant.quantities import Quantity data = numpy.zeros((10, 10), dtype='uint8') data[2:8, 2:8] = 1 image = FieldContainer(data) for dim in image.dimensions: dim.unit = Quantity('1 mum') image.seal() data = numpy.zeros((10, 10), dtype='uint8') data[3][3] = 1 data[4][6] = 2 markers = FieldContainer(data) for dim in markers.dimensions: dim.unit = Quantity('1 mum') wshed = Watershed() result = wshed.wsworker(image, markers) self.assertEqual(result.dimensions, image.dimensions) from scipy.ndimage import label self.assertEqual(label(result.data)[1], 2)
def gradientWorker(self, image, subscriber=0): d0 = image.dimensions[0] dims = [d0] + [d.inUnitsOf(d0) for d in image.dimensions[1:]] data = image.data.astype(float) gradient = np.gradient(data) magnitude = np.zeros_like(data) for i, (dim, grad) in enumerate(zip(dims, gradient)): shape = [ 1, ] * len(dims) shape[i] = dim.data.shape[0] grad /= np.gradient(dim.data).reshape(shape) magnitude += grad**2 magnitude = np.sqrt(magnitude) longname = "Gradient" from pyphant.core.DataContainer import FieldContainer result = FieldContainer(magnitude, image.unit / d0.unit, None, copy.deepcopy(image.mask), copy.deepcopy(image.dimensions), longname, image.shortname, copy.deepcopy(image.attributes), False) result.seal() return result
class KnowledgeManagerTestCase(unittest.TestCase): def setUp(self): a = N.array([0, 1, 2, 3]) self._fc = FieldContainer(a) self._fc.seal() def testGetLocalFile(self): h5fileid, h5name = tempfile.mkstemp(suffix='.h5', prefix='test-') os.close(h5fileid) h5 = open_file(h5name, 'w') resultsGroup = h5.create_group("/", "results") ptp.saveResult(self._fc, h5) h5.close() km = KnowledgeManager.getInstance() from urllib import pathname2url url = pathname2url(h5name) if not url.startswith('///'): url = '//' + url url = 'file:' + url km.registerURL(url, temporary=True) km_fc = km.getDataContainer(self._fc.id) self.assertEqual(self._fc, km_fc) os.remove(h5name) def testGetHTTPFile(self): host = "pyphant.sourceforge.net" remote_dir = "" url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5" # Get remote file and load DataContainer filename, headers = urllib.urlretrieve(url) h5 = open_file(filename, 'r') for g in h5.walk_groups("/results"): if (len(g._v_attrs.TITLE)>0) \ and (r"\Psi" in g._v_attrs.shortname): http_fc = ptp.loadField(h5, g) h5.close() km = KnowledgeManager.getInstance() km.registerURL(url, temporary=True) km_fc = km.getDataContainer(http_fc.id) self.assertEqual(http_fc, km_fc) os.remove(filename) def testGetDataContainer(self): km = KnowledgeManager.getInstance() km.registerDataContainer(self._fc, temporary=True) km_fc = km.getDataContainer(self._fc.id) self.assertEqual(self._fc, km_fc) def testSCwithSCColumn(self): fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10))) fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20))) sc_child = SampleContainer(longname='sc_child', columns=[fc_child1]) sc_parent = SampleContainer(longname='sc_parent', columns=[sc_child, fc_child2]) sc_parent.seal() km = KnowledgeManager.getInstance() km.registerDataContainer(sc_parent, temporary=True) lnlist = km.search(['longname'], {'col_of': {'longname': 'sc_parent'}}) lnlist = [entry[0] for entry in lnlist] assert len(lnlist) == 2 assert 'fc_child2' in lnlist assert 'sc_child' in lnlist def testExceptions(self): km = KnowledgeManager.getInstance() #TODO: #invalid id #DataContainer not sealed #Local file not readable #Register empty hdf def testRegisterFMF(self): km = KnowledgeManager.getInstance() fileid, filename = tempfile.mkstemp(suffix='.fmf', prefix='test-') os.close(fileid) handler = open(filename, 'w') fmfstring = """; -*- fmf-version: 1.0 -*- [*reference] title: Knowledge Manager FMF Test creator: Alexander Held created: 2009-05-25 08:45:00+02:00 place: Uni Freiburg [*data definitions] voltage: V [V] current: I(V) [A] [*data] 1.0\t0.5 2.0\t1.0 3.0\t1.5 """ handler.write(fmfstring) handler.close() dc_id = km.registerFMF(filename, temporary=True) os.remove(filename) km.getDataContainer(dc_id) def testCache(self): print "Preparing FCs for cache test (cache size: %d MB)..."\ % (CACHE_MAX_SIZE / 1024 / 1024) km = KnowledgeManager.getInstance() sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)] byte_sizes = [reduce(lambda x, y: x * y, size) * 8 for size in sizes] ids = [] rand_id_pool = [] assert_dict = {} for num in xrange(2 * 20): fc = FieldContainer(N.ones((500, 500))) fc.data.flat[0] = num fc.seal() km.registerDataContainer(fc, temporary=True) rand_id_pool.append(fc.id) assert_dict[fc.id] = num for size in sizes: fc = FieldContainer(N.ones(size)) fc.seal() km.registerDataContainer(fc, temporary=True) ids.append(fc.id) for id, bytes in zip(ids, byte_sizes): uc_acc_time = 0.0 c_acc_time = 0.0 reps = 30 for rep in xrange(reps): t1 = time() km.getDataContainer(id, use_cache=False) t2 = time() uc_acc_time += t2 - t1 km._cache = [] km._cache_size = 0 for rep in xrange(reps): t1 = time() km.getDataContainer(id) t2 = time() c_acc_time += t2 - t1 uc_avr = 1000.0 * uc_acc_time / reps c_avr = 1000.0 * c_acc_time / reps print "Avr. access time for %0.2f kB sequential read: "\ "%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024, uc_avr, c_avr) km._cache = [] km._cache_size = 0 rand_ids = [] reps = 500 import pyphant.core.KnowledgeManager pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20 for run in xrange(reps): rand_ids.append(rand_id_pool[random.randint( 0, len(rand_id_pool) - 1)]) uc_acc_time = 0.0 c_acc_time = 0.0 for id in rand_ids: t1 = time() fc = km.getDataContainer(id, use_cache=False) t2 = time() uc_acc_time += t2 - t1 for id in rand_ids: t1 = time() fc = km.getDataContainer(id) t2 = time() c_acc_time += t2 - t1 assert km._cache_size >= 0 assert km._cache_size <= CACHE_MAX_SIZE assert len(km._cache) <= CACHE_MAX_NUMBER assert assert_dict[id] == fc.data.flat[0] uc_avr = 1000.0 * uc_acc_time / reps c_avr = 1000.0 * c_acc_time / reps bytes = float(500 * 500 * 8) / 1024 print "Avr. access time for %0.2f kB random read: "\ "%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr) print "------ End Cache Test ------"
def testCache(self): print "Preparing FCs for cache test (cache size: %d MB)..."\ % (CACHE_MAX_SIZE / 1024 / 1024) km = KnowledgeManager.getInstance() sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)] byte_sizes = [reduce(lambda x, y: x * y, size) * 8 for size in sizes] ids = [] rand_id_pool = [] assert_dict = {} for num in xrange(2 * 20): fc = FieldContainer(N.ones((500, 500))) fc.data.flat[0] = num fc.seal() km.registerDataContainer(fc, temporary=True) rand_id_pool.append(fc.id) assert_dict[fc.id] = num for size in sizes: fc = FieldContainer(N.ones(size)) fc.seal() km.registerDataContainer(fc, temporary=True) ids.append(fc.id) for id, bytes in zip(ids, byte_sizes): uc_acc_time = 0.0 c_acc_time = 0.0 reps = 30 for rep in xrange(reps): t1 = time() km.getDataContainer(id, use_cache=False) t2 = time() uc_acc_time += t2 - t1 km._cache = [] km._cache_size = 0 for rep in xrange(reps): t1 = time() km.getDataContainer(id) t2 = time() c_acc_time += t2 - t1 uc_avr = 1000.0 * uc_acc_time / reps c_avr = 1000.0 * c_acc_time / reps print "Avr. access time for %0.2f kB sequential read: "\ "%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024, uc_avr, c_avr) km._cache = [] km._cache_size = 0 rand_ids = [] reps = 500 import pyphant.core.KnowledgeManager pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20 for run in xrange(reps): rand_ids.append(rand_id_pool[random.randint( 0, len(rand_id_pool) - 1)]) uc_acc_time = 0.0 c_acc_time = 0.0 for id in rand_ids: t1 = time() fc = km.getDataContainer(id, use_cache=False) t2 = time() uc_acc_time += t2 - t1 for id in rand_ids: t1 = time() fc = km.getDataContainer(id) t2 = time() c_acc_time += t2 - t1 assert km._cache_size >= 0 assert km._cache_size <= CACHE_MAX_SIZE assert len(km._cache) <= CACHE_MAX_NUMBER assert assert_dict[id] == fc.data.flat[0] uc_avr = 1000.0 * uc_acc_time / reps c_avr = 1000.0 * c_acc_time / reps bytes = float(500 * 500 * 8) / 1024 print "Avr. access time for %0.2f kB random read: "\ "%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr) print "------ End Cache Test ------"
class KnowledgeManagerTestCase(unittest.TestCase): def setUp(self): a = N.array([0, 1, 2, 3]) self._fc = FieldContainer(a) self._fc.seal() def testGetLocalFile(self): h5fileid, h5name = tempfile.mkstemp(suffix='.h5',prefix='test-') os.close(h5fileid) h5 = tables.openFile(h5name,'w') resultsGroup = h5.createGroup("/", "results") ptp.saveResult(self._fc, h5) h5.close() km = KnowledgeManager.getInstance() from urllib import pathname2url url = pathname2url(h5name) if not url.startswith('///'): url = '//' + url url = 'file:' + url km.registerURL(url, temporary=True) km_fc = km.getDataContainer(self._fc.id) self.assertEqual(self._fc, km_fc) os.remove(h5name) def testGetHTTPFile(self): host = "pyphant.sourceforge.net" remote_dir = "" url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5" # Get remote file and load DataContainer filename, headers = urllib.urlretrieve(url) h5 = tables.openFile(filename, 'r') for g in h5.walkGroups("/results"): if (len(g._v_attrs.TITLE)>0) \ and (r"\Psi" in g._v_attrs.shortname): http_fc = ptp.loadField(h5,g) h5.close() km = KnowledgeManager.getInstance() km.registerURL(url, temporary=True) km_fc = km.getDataContainer(http_fc.id) self.assertEqual(http_fc, km_fc) os.remove(filename) def testGetDataContainer(self): km = KnowledgeManager.getInstance() km.registerDataContainer(self._fc, temporary=True) km_fc = km.getDataContainer(self._fc.id) self.assertEqual(self._fc, km_fc) def testSCwithSCColumn(self): fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10))) fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20))) sc_child = SampleContainer(longname='sc_child', columns=[fc_child1]) sc_parent = SampleContainer(longname='sc_parent', columns=[sc_child, fc_child2]) sc_parent.seal() km = KnowledgeManager.getInstance() km.registerDataContainer(sc_parent, temporary=True) lnlist = km.search(['longname'], {'col_of':{'longname':'sc_parent'}}) lnlist = [entry[0] for entry in lnlist] assert len(lnlist) == 2 assert 'fc_child2' in lnlist assert 'sc_child' in lnlist def testExceptions(self): km = KnowledgeManager.getInstance() #TODO: #invalid id #DataContainer not sealed #Local file not readable #Register empty hdf def testRegisterFMF(self): km = KnowledgeManager.getInstance() fileid, filename = tempfile.mkstemp(suffix='.fmf', prefix='test-') os.close(fileid) handler = open(filename, 'w') fmfstring = """; -*- fmf-version: 1.0 -*- [*reference] title: Knowledge Manager FMF Test creator: Alexander Held created: 2009-05-25 08:45:00+02:00 place: Uni Freiburg [*data definitions] voltage: V [V] current: I(V) [A] [*data] 1.0\t0.5 2.0\t1.0 3.0\t1.5 """ handler.write(fmfstring) handler.close() dc_id = km.registerFMF(filename, temporary=True) os.remove(filename) km.getDataContainer(dc_id) def testCache(self): print "Preparing FCs for cache test (cache size: %d MB)..."\ % (CACHE_MAX_SIZE / 1024 / 1024) km = KnowledgeManager.getInstance() sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)] byte_sizes = [reduce(lambda x, y:x * y, size) * 8 for size in sizes] ids = [] rand_id_pool = [] assert_dict = {} for num in xrange(2 * 20): fc = FieldContainer(N.ones((500, 500))) fc.data.flat[0] = num fc.seal() km.registerDataContainer(fc, temporary=True) rand_id_pool.append(fc.id) assert_dict[fc.id] = num for size in sizes: fc = FieldContainer(N.ones(size)) fc.seal() km.registerDataContainer(fc, temporary=True) ids.append(fc.id) for id, bytes in zip(ids, byte_sizes): uc_acc_time = 0.0 c_acc_time = 0.0 reps = 30 for rep in xrange(reps): t1 = time() km.getDataContainer(id, use_cache=False) t2 = time() uc_acc_time += t2 - t1 km._cache = [] km._cache_size = 0 for rep in xrange(reps): t1 = time() km.getDataContainer(id) t2 = time() c_acc_time += t2 - t1 uc_avr = 1000.0 * uc_acc_time / reps c_avr = 1000.0 * c_acc_time / reps print "Avr. access time for %0.2f kB sequential read: "\ "%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024, uc_avr, c_avr) km._cache = [] km._cache_size = 0 rand_ids = [] reps = 500 import pyphant.core.KnowledgeManager pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20 for run in xrange(reps): rand_ids.append(rand_id_pool[ random.randint(0, len(rand_id_pool) - 1)]) uc_acc_time = 0.0 c_acc_time = 0.0 for id in rand_ids: t1 = time() fc = km.getDataContainer(id, use_cache=False) t2 = time() uc_acc_time += t2 - t1 for id in rand_ids: t1 = time() fc = km.getDataContainer(id) t2 = time() c_acc_time += t2 - t1 assert km._cache_size >= 0 assert km._cache_size <= CACHE_MAX_SIZE assert len(km._cache) <= CACHE_MAX_NUMBER assert assert_dict[id] == fc.data.flat[0] uc_avr = 1000.0 * uc_acc_time / reps c_avr = 1000.0 * c_acc_time / reps bytes = float(500 * 500 * 8) / 1024 print "Avr. access time for %0.2f kB random read: "\ "%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr) print "------ End Cache Test ------"
def testCache(self): print "Preparing FCs for cache test (cache size: %d MB)..."\ % (CACHE_MAX_SIZE / 1024 / 1024) km = KnowledgeManager.getInstance() sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)] byte_sizes = [reduce(lambda x, y:x * y, size) * 8 for size in sizes] ids = [] rand_id_pool = [] assert_dict = {} for num in xrange(2 * 20): fc = FieldContainer(N.ones((500, 500))) fc.data.flat[0] = num fc.seal() km.registerDataContainer(fc, temporary=True) rand_id_pool.append(fc.id) assert_dict[fc.id] = num for size in sizes: fc = FieldContainer(N.ones(size)) fc.seal() km.registerDataContainer(fc, temporary=True) ids.append(fc.id) for id, bytes in zip(ids, byte_sizes): uc_acc_time = 0.0 c_acc_time = 0.0 reps = 30 for rep in xrange(reps): t1 = time() km.getDataContainer(id, use_cache=False) t2 = time() uc_acc_time += t2 - t1 km._cache = [] km._cache_size = 0 for rep in xrange(reps): t1 = time() km.getDataContainer(id) t2 = time() c_acc_time += t2 - t1 uc_avr = 1000.0 * uc_acc_time / reps c_avr = 1000.0 * c_acc_time / reps print "Avr. access time for %0.2f kB sequential read: "\ "%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024, uc_avr, c_avr) km._cache = [] km._cache_size = 0 rand_ids = [] reps = 500 import pyphant.core.KnowledgeManager pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20 for run in xrange(reps): rand_ids.append(rand_id_pool[ random.randint(0, len(rand_id_pool) - 1)]) uc_acc_time = 0.0 c_acc_time = 0.0 for id in rand_ids: t1 = time() fc = km.getDataContainer(id, use_cache=False) t2 = time() uc_acc_time += t2 - t1 for id in rand_ids: t1 = time() fc = km.getDataContainer(id) t2 = time() c_acc_time += t2 - t1 assert km._cache_size >= 0 assert km._cache_size <= CACHE_MAX_SIZE assert len(km._cache) <= CACHE_MAX_NUMBER assert assert_dict[id] == fc.data.flat[0] uc_avr = 1000.0 * uc_acc_time / reps c_avr = 1000.0 * c_acc_time / reps bytes = float(500 * 500 * 8) / 1024 print "Avr. access time for %0.2f kB random read: "\ "%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr) print "------ End Cache Test ------"