def onImportLocal(self, event):
msg = "Select HDF5 or FMF file to import DataContainer(s) from."
wc = "*.h5, *.hdf, *.hdf5, *.fmf|*.h5;*.hdf;*.hdf5;*.fmf"
dlg = wx.FileDialog(self,
message=msg,
defaultDir=os.getcwd(),
defaultFile="",
wildcard=wc,
style=wx.OPEN)
if dlg.ShowModal() == wx.ID_OK:
filename = dlg.GetPath()
url = 'file://' + os.path.realpath(filename)
km = KnowledgeManager.getInstance()
cpt2 = "Info"
msg2 = "Successfully imported DataContainer(s) from\n'%s'"\
% (filename ,)
try:
km.registerURL(url)
except Exception:
cpt2 = "Error"
msg2 = "'%s' is not a valid HDF5 or FMF file.\n"\
"(Tried to import from '%s')" % (filename, url)
finally:
dlg2 = wx.MessageDialog(self, msg2, cpt2, wx.OK)
dlg2.ShowModal()
dlg2.Destroy()
else:
dlg.Destroy()
def onImportURL(self, event):
cpt = "Import HDF5 or FMF from URL"
msg = "Enter a URL to a valid HDF5 or FMF file "\
"(e.g. http://www.example.org/data.h5).\n"\
"The file is stored permanently in your home directory in the "\
".pyphant directory\nand all DataContainers contained in that "\
"file are available by using the\nSCSource or DCSource "\
"Worker even after "\
"restarting wxPyphant.\nHTTP redirects are resolved "\
"automatically, i.e. DOIs are supported as well."
dlg = wx.TextEntryDialog(self, msg, cpt)
dlgid = dlg.ShowModal()
if dlgid != wx.ID_CANCEL:
url = dlg.GetValue()
cpt2 = "Info"
msg2 = "Successfully imported DataContainers from\n'%s'"\
% (url ,)
km = KnowledgeManager.getInstance()
try:
km.registerURL(url)
except Exception:
cpt2 = "Error"
msg2 = "'%s' is not a valid URL to a HDF5 or FMF file."\
% (url, )
finally:
dlg2 = wx.MessageDialog(self, msg2, cpt2, wx.OK)
dlg2.ShowModal()
dlg2.Destroy()
dlg.Destroy()
def __init__(self, local_km=None,
host=u'127.0.0.1', port=8080, start=False,
web_interface=False, dbase=u'default'):
"""
Arguments:
- `local_km`: Local KnowledgeManager instance to hook up to.
If set to `None`, KnowledgeManager.getInstance() is used.
- `host`: hostname to listen on
- `port`: port to listen on
- `start`: flag that indicates whether to start the server
- `web_interface`: flag that indicates whether to enable
the web interface. You can enable/disable it anytime by
setting (KN instance).web_interface.enabled to `True`/`False`.
- `dbase`: leave this to 'default', other values are allowed for
debug purposes
"""
RoutingHTTPServer.__init__(self, host, port, start)
if local_km == None:
local_km = KnowledgeManager.getInstance()
self.km = local_km
self.remotes = []
if dbase == u'default':
self._dbase = os.path.join(getPyphantPath('sqlite3'),
'kn_remotes.sqlite3')
else:
self._dbase = dbase
self._restore_remotes()
self._setup_routes()
self._tempdir = mkdtemp(prefix='HDF5Wrap')
tpl_path = pkg_resources.resource_filename('pyphant', 'web/templates')
if not tpl_path in pyphant.core.bottle.TEMPLATE_PATH:
pyphant.core.bottle.TEMPLATE_PATH.append(tpl_path)
from pyphant.core.WebInterface import WebInterface
self.web_interface = WebInterface(self, web_interface)
self.km.node = self
def refreshParams(self, subscriber=None, update=True, reset=False):
if update:
self.expectedValues = dict(
[(name, param.value) for name, param \
in self._params.iteritems() if name not in \
['name', 'reset', 'remaining']])
elif reset:
self.expectedValues = dict(
[(name, ANYSTR) for name in self._params.iterkeys() \
if name not in ['name', 'reset', 'remaining']])
kmanager = KnowledgeManager.getInstance()
for name, param in self._params.iteritems():
if name in ['name', 'reset', 'remaining']:
continue
search_dict = self.getSearchDict(name)
newEVs = [[ANYSTR]]
newEVs.extend(kmanager.search(
[self.getResKey(name)], search_dict=search_dict, distinct=True))
newEVs = [newEV[0] for newEV in newEVs]
event = VisualizerChangeValue(param, possibleValues=newEVs)
if reset:
event.value = ANYSTR
param._eventDispatcher.dispatchEvent(event)
if update:
param.possibleValues = newEVs
if name == 'id':
if self.expectedValues['id'] != ANYSTR:
self.remaining = 1
else:
self.remaining = len(newEVs) - 1
if update:
self.paramRemaining.value = self.remaining
self.possibleIds = newEVs
self.dirty = False
def onImportURL(self, event):
cpt = "Import HDF5 or FMF from URL"
msg = "Enter a URL to a valid HDF5 or FMF file "\
"(e.g. http://www.example.org/data.h5).\n"\
"The file is stored permanently in your home directory in the "\
".pyphant directory\nand all DataContainers contained in that "\
"file are available by using the\nSCSource or DCSource "\
"Worker even after "\
"restarting wxPyphant.\nHTTP redirects are resolved "\
"automatically, i.e. DOIs are supported as well."
dlg = wx.TextEntryDialog(self, msg, cpt)
dlgid = dlg.ShowModal()
if dlgid != wx.ID_CANCEL:
url = dlg.GetValue()
cpt2 = "Info"
msg2 = "Successfully imported DataContainers from\n'%s'"\
% (url ,)
km = KnowledgeManager.getInstance()
try:
km.registerURL(url)
except Exception:
cpt2 = "Error"
msg2 = "'%s' is not a valid URL to a HDF5 or FMF file."\
% (url, )
finally:
dlg2 = wx.MessageDialog(self, msg2, cpt2, wx.OK)
dlg2.ShowModal()
dlg2.Destroy()
dlg.Destroy()
def testRegisterFMF(self):
from pyphant.core.KnowledgeManager import KnowledgeManager
kmanager = KnowledgeManager.getInstance()
table_id = kmanager.registerFMF(self.filename, temporary=True)
table = kmanager.getDataContainer(table_id)
print "Testing registered SampleContainer for consistency..."
for column in ['y0', 'y1', 'y2', 'y3', 'y4', 'y5', 'y6', 'y7', 'y8']:
self.assertEqual(table[column].dimensions[0].id, table['x'].id)
def getSampleContainer(self, subscriber = 0):
if self.dirty:
self.refreshParams(update=False)
if len(self.possibleIds) == 2:
emd5 = self.possibleIds[1]
else:
emd5 = self.paramId.value
kmanager = KnowledgeManager.getInstance()
return kmanager.getDataContainer(emd5)
def testRegisterFMF(self):
from pyphant.core.KnowledgeManager import KnowledgeManager
kmanager = KnowledgeManager.getInstance()
table_id = kmanager.registerFMF(self.filename, temporary=True)
table = kmanager.getDataContainer(table_id)
print "Testing registered SampleContainer for consistency..."
for column in ["y0", "y1", "y2", "y3", "y4", "y5", "y6", "y7", "y8"]:
self.assertEqual(table[column].dimensions[0].id, table["x"].id)
def testRegisterFMF(self):
from pyphant.core.KnowledgeManager import KnowledgeManager
kmanager = KnowledgeManager.getInstance()
table_id = kmanager.registerFMF(self.filename, temporary=True)
table = kmanager.getDataContainer(table_id)
print "Testing registered SampleContainer for consistency..."
for column in ['y0', 'y1', 'y2', 'y3', 'y4', 'y5', 'y6', 'y7', 'y8']:
self.assertEqual(table[column].dimensions[0].id,
table['x'].id)
def getFieldContainer(self, subscriber=0):
if self.dirty:
self.refreshParams(update=False)
if len(self.possibleIds) == 2:
emd5 = self.possibleIds[1]
else:
emd5 = self.paramId.value
kmanager = KnowledgeManager.getInstance()
return kmanager.getDataContainer(emd5)
def main():
km = KnowledgeManager.getInstance()
sourcefile = sys.argv[1]
h5 = tables.openFile(sourcefile, 'r+')
options, args, recipe, orders = processArgs(h5)
h5.close()
import os.path
km.registerURL("file://" + os.path.realpath(sourcefile))
orderLists = []
for order in orders:
order = globOrder(order, None)
singles = dict(
[(socket, id_[0]) \
for socket, id_ in order[0].iteritems() if len(id_) == 1]
)
lists = dict(
[(socket, id_) \
for socket, id_ in order[0].iteritems() if len(id_) > 1]
)
lens = [len(l) for l in lists.itervalues()]
assert max(lens) == min(lens), "Illegal lengths."
count = lens[0]
orderList = []
for i in xrange(count):
sockMap = copy.deepcopy(singles)
for s, l in lists.iteritems():
sockMap[s] = l[i]
orderList.append((sockMap, order[1]))
orderLists.append(orderList)
orders = sum(orderLists, [])
if options.maxOrders == None:
count = len(orders)
else:
count = options.maxOrders
i = 0
orderLists = []
while i + count < len(orders):
orderLists.append(orders[i: i + count])
i += count
orderLists.append(orders[i:])
for i, orderList in enumerate(orderLists):
filename = os.path.basename(sourcefile)[:-3] + '_%i.h5' % (i, )
with H5FileHandler(filename, 'w') as handler:
handler.saveRecipe(recipe)
h5 = tables.openFile(filename, 'r+')
for o in orderList:
PyTablesPersister.saveExecutionOrder(h5, o)
for data in o[0].values():
dc = km.getDataContainer(data)
PyTablesPersister.saveResult(dc, h5)
h5.close()
def testSCwithSCColumn(self):
fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10)))
fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20)))
sc_child = SampleContainer(longname='sc_child', columns=[fc_child1])
sc_parent = SampleContainer(longname='sc_parent', columns=[sc_child,
fc_child2])
sc_parent.seal()
km = KnowledgeManager.getInstance()
km.registerDataContainer(sc_parent, temporary=True)
lnlist = km.search(['longname'], {'col_of':{'longname':'sc_parent'}})
lnlist = [entry[0] for entry in lnlist]
assert len(lnlist) == 2
assert 'fc_child2' in lnlist
assert 'sc_child' in lnlist
def testSCwithSCColumn(self):
fc_child1 = FieldContainer(longname='fc_child1', data=N.ones((10, 10)))
fc_child2 = FieldContainer(longname='fc_child2', data=N.ones((20, 20)))
sc_child = SampleContainer(longname='sc_child', columns=[fc_child1])
sc_parent = SampleContainer(longname='sc_parent',
columns=[sc_child, fc_child2])
sc_parent.seal()
km = KnowledgeManager.getInstance()
km.registerDataContainer(sc_parent, temporary=True)
lnlist = km.search(['longname'], {'col_of': {'longname': 'sc_parent'}})
lnlist = [entry[0] for entry in lnlist]
assert len(lnlist) == 2
assert 'fc_child2' in lnlist
assert 'sc_child' in lnlist
def globOrder(order, h5):
km = KnowledgeManager.getInstance()
sockMap = {}
for sSpec in order[0].iteritems():
if sSpec[1].startswith('emd5://'):
sockMap[sSpec[0]] = [sSpec[1]]
else:
ids = []
for f in glob.iglob(sSpec[1]):
dc = f2dc(f)
if h5 != None:
PyTablesPersister.saveResult(dc, h5)
km.registerDataContainer(dc)
ids.append(dc.id)
sockMap[sSpec[0]] = ids
return (sockMap, order[1])
def testGetLocalFile(self):
h5fileid, h5name = tempfile.mkstemp(suffix='.h5',prefix='test-')
os.close(h5fileid)
h5 = tables.openFile(h5name,'w')
resultsGroup = h5.createGroup("/", "results")
ptp.saveResult(self._fc, h5)
h5.close()
km = KnowledgeManager.getInstance()
from urllib import pathname2url
url = pathname2url(h5name)
if not url.startswith('///'):
url = '//' + url
url = 'file:' + url
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
os.remove(h5name)
def batch(recipe, input, plug, longname, dobatch=True, temporary=False):
"""
Runs the same recipe multiple times for different input data.
The return value is either a SampleContainer similar to input
with 'emd5' column replaced by results or the resulting
DataContainer from plug, if dobatch is set to False.
recipe -- CompositeWorker instance
input -- SampleContainer with 'emd5' column or any DataContainer if
dobatch is set to False
plug -- plug contained in recipe to get output from
(there has to be exactly one open socket in recipe
ascending from plug)
longname -- longname of resulting SampleContainer, works only for
dobatch == True
dobatch -- if set to False, input is treated as a single data source
temporary -- whether to register results temporarily, only applies when
dobatch is set to True
"""
socket = recipe.getOpenSocketsForPlug(plug)[0]
from tools import Emd5Src
DummyWorker = Emd5Src.Emd5Src()
socket.insert(DummyWorker.getPlugs()[0])
DummyWorker.paramSelectby.value = u"enter emd5"
from pyphant.core.KnowledgeManager import KnowledgeManager
km = KnowledgeManager.getInstance()
if dobatch:
import copy
output = copy.deepcopy(input)
index = 0
for emd5 in input["emd5"].data:
DummyWorker.paramEnteremd5.value = emd5
resultDC = plug.getResult()
km.registerDataContainer(resultDC, temporary=temporary)
output["emd5"].data[index] = resultDC.id
index += 1
output.longname = longname
output.seal()
else:
km.registerDataContainer(input)
DummyWorker.paramEnteremd5.value = input.id
output = plug.getResult()
socket.pullPlug()
return output
def testGetHTTPFile(self):
host = "pyphant.sourceforge.net"
remote_dir = ""
url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5"
# Get remote file and load DataContainer
filename, headers = urllib.urlretrieve(url)
h5 = tables.openFile(filename, 'r')
for g in h5.walkGroups("/results"):
if (len(g._v_attrs.TITLE)>0) \
and (r"\Psi" in g._v_attrs.shortname):
http_fc = ptp.loadField(h5,g)
h5.close()
km = KnowledgeManager.getInstance()
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(http_fc.id)
self.assertEqual(http_fc, km_fc)
os.remove(filename)
def testGetLocalFile(self):
h5fileid, h5name = tempfile.mkstemp(suffix='.h5', prefix='test-')
os.close(h5fileid)
h5 = open_file(h5name, 'w')
resultsGroup = h5.create_group("/", "results")
ptp.saveResult(self._fc, h5)
h5.close()
km = KnowledgeManager.getInstance()
from urllib import pathname2url
url = pathname2url(h5name)
if not url.startswith('///'):
url = '//' + url
url = 'file:' + url
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
os.remove(h5name)
def testGetHTTPFile(self):
host = "pyphant.sourceforge.net"
remote_dir = ""
url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5"
# Get remote file and load DataContainer
filename, headers = urllib.urlretrieve(url)
h5 = open_file(filename, 'r')
for g in h5.walk_groups("/results"):
if (len(g._v_attrs.TITLE)>0) \
and (r"\Psi" in g._v_attrs.shortname):
http_fc = ptp.loadField(h5, g)
h5.close()
km = KnowledgeManager.getInstance()
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(http_fc.id)
self.assertEqual(http_fc, km_fc)
os.remove(filename)
def batch(recipe, input, plug, longname, dobatch=True, temporary=False):
"""
Runs the same recipe multiple times for different input data.
The return value is either a SampleContainer similar to input
with 'emd5' column replaced by results or the resulting
DataContainer from plug, if dobatch is set to False.
recipe -- CompositeWorker instance
input -- SampleContainer with 'emd5' column or any DataContainer if
dobatch is set to False
plug -- plug contained in recipe to get output from
(there has to be exactly one open socket in recipe
ascending from plug)
longname -- longname of resulting SampleContainer, works only for
dobatch == True
dobatch -- if set to False, input is treated as a single data source
temporary -- whether to register results temporarily, only applies when
dobatch is set to True
"""
socket = recipe.getOpenSocketsForPlug(plug)[0]
from pyphant.core.Emd5Src import Emd5Src
DummyWorker = Emd5Src()
socket.insert(DummyWorker.getPlugs()[0])
DummyWorker.paramSelectby.value = u"enter emd5"
from pyphant.core.KnowledgeManager import KnowledgeManager
km = KnowledgeManager.getInstance()
if dobatch:
import copy
output = copy.deepcopy(input)
index = 0
for emd5 in input['emd5'].data:
DummyWorker.paramEnteremd5.value = emd5
resultDC = plug.getResult()
km.registerDataContainer(resultDC, temporary=temporary)
output['emd5'].data[index] = resultDC.id
index += 1
output.longname = longname
output.seal()
else:
km.registerDataContainer(input)
DummyWorker.paramEnteremd5.value = input.id
output = plug.getResult()
socket.pullPlug()
return output
def __init__(self,
local_km=None,
host=u'127.0.0.1',
port=8080,
start=False,
web_interface=False,
dbase=u'default'):
"""
Arguments:
- `local_km`: Local KnowledgeManager instance to hook up to.
If set to `None`, KnowledgeManager.getInstance() is used.
- `host`: hostname to listen on
- `port`: port to listen on
- `start`: flag that indicates whether to start the server
- `web_interface`: flag that indicates whether to enable
the web interface. You can enable/disable it anytime by
setting (KN instance).web_interface.enabled to `True`/`False`.
- `dbase`: leave this to 'default', other values are allowed for
debug purposes
"""
RoutingHTTPServer.__init__(self, host, port, start)
if local_km == None:
local_km = KnowledgeManager.getInstance()
self.km = local_km
self.remotes = []
if dbase == u'default':
self._dbase = os.path.join(getPyphantPath('sqlite3'),
'kn_remotes.sqlite3')
else:
self._dbase = dbase
self._restore_remotes()
self._setup_routes()
self._tempdir = mkdtemp(prefix='HDF5Wrap')
tpl_path = os.path.join(pyphant_source_path[0], 'web', 'templates')
if not tpl_path in pyphant.core.bottle.TEMPLATE_PATH:
pyphant.core.bottle.TEMPLATE_PATH.append(tpl_path)
from pyphant.core.WebInterface import WebInterface
self.web_interface = WebInterface(self, web_interface)
self.km.node = self
def testRegisterFMF(self):
km = KnowledgeManager.getInstance()
fileid, filename = tempfile.mkstemp(suffix='.fmf', prefix='test-')
os.close(fileid)
handler = open(filename, 'w')
fmfstring = """; -*- fmf-version: 1.0 -*-
[*reference]
title: Knowledge Manager FMF Test
creator: Alexander Held
created: 2009-05-25 08:45:00+02:00
place: Uni Freiburg
[*data definitions]
voltage: V [V]
current: I(V) [A]
[*data]
1.0\t0.5
2.0\t1.0
3.0\t1.5
"""
handler.write(fmfstring)
handler.close()
dc_id = km.registerFMF(filename, temporary=True)
os.remove(filename)
km.getDataContainer(dc_id)
def testRegisterFMF(self):
km = KnowledgeManager.getInstance()
fileid, filename = tempfile.mkstemp(suffix='.fmf', prefix='test-')
os.close(fileid)
handler = open(filename, 'w')
fmfstring = """; -*- fmf-version: 1.0 -*-
[*reference]
title: Knowledge Manager FMF Test
creator: Alexander Held
created: 2009-05-25 08:45:00+02:00
place: Uni Freiburg
[*data definitions]
voltage: V [V]
current: I(V) [A]
[*data]
1.0\t0.5
2.0\t1.0
3.0\t1.5
"""
handler.write(fmfstring)
handler.close()
dc_id = km.registerFMF(filename, temporary=True)
os.remove(filename)
km.getDataContainer(dc_id)
def onImportLocal(self, event):
msg = "Select HDF5 or FMF file to import DataContainer(s) from."
wc = "*.h5, *.hdf, *.hdf5, *.fmf|*.h5;*.hdf;*.hdf5;*.fmf"
dlg = wx.FileDialog(self, message=msg, defaultDir=os.getcwd(),
defaultFile="", wildcard=wc, style=wx.OPEN)
if dlg.ShowModal() == wx.ID_OK:
filename = dlg.GetPath()
url = 'file://' + os.path.realpath(filename)
km = KnowledgeManager.getInstance()
cpt2 = "Info"
msg2 = "Successfully imported DataContainer(s) from\n'%s'"\
% (filename ,)
try:
km.registerURL(url)
except Exception:
cpt2 = "Error"
msg2 = "'%s' is not a valid HDF5 or FMF file.\n"\
"(Tried to import from '%s')" % (filename, url)
finally:
dlg2 = wx.MessageDialog(self, msg2, cpt2, wx.OK)
dlg2.ShowModal()
dlg2.Destroy()
else:
dlg.Destroy()
return (options, args, recipe,
[(options.socketMap, [p.id for p in options.requestedResults])])
from pyphant.core.Helpers import loadImageAsGreyScale
def f2dc(f):
res = loadImageAsGreyScale(f)
res.seal()
return res
import glob
from pyphant.core.KnowledgeManager import KnowledgeManager
km = KnowledgeManager.getInstance()
def globOrder(order, h5):
sockMap = {}
for sSpec in order[0].iteritems():
if sSpec[1].startswith('emd5://'):
sockMap[sSpec[0]] = [sSpec[1]]
else:
ids = []
for f in glob.iglob(sSpec[1]):
dc = f2dc(f)
if h5 != None:
PyTablesPersister.saveResult(dc, h5)
km.registerDataContainer(dc)
ids.append(dc.id)
def __init__(self, DataContainer, show=True):
if show:
kmanager = KnowledgeManager.getInstance()
kmanager.registerDataContainer(DataContainer)
from ImageProcessing.ImageLoaderWorker import ImageLoaderWorker
def f2dc(f):
il = ImageLoaderWorker()
il.paramFilename.value = f
res = il.plugLoadImageAsGreyScale.getResult()
res.seal()
return res
import glob
from pyphant.core.KnowledgeManager import KnowledgeManager
km = KnowledgeManager.getInstance()
def globOrder(order, h5):
sockMap = {}
for sSpec in order[0].iteritems():
if sSpec[1].startswith('emd5://'):
sockMap[sSpec[0]] = [sSpec[1]]
else:
ids = []
for f in glob.iglob(sSpec[1]):
dc = f2dc(f)
if h5 != None:
PyTablesPersister.saveResult(dc, h5)
km.registerDataContainer(dc)
ids.append(dc.id)
def testExceptions(self):
km = KnowledgeManager.getInstance()
def testGetDataContainer(self):
km = KnowledgeManager.getInstance()
km.registerDataContainer(self._fc, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
def testCache(self):
print "Preparing FCs for cache test (cache size: %d MB)..."\
% (CACHE_MAX_SIZE / 1024 / 1024)
km = KnowledgeManager.getInstance()
sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)]
byte_sizes = [reduce(lambda x, y:x * y, size) * 8 for size in sizes]
ids = []
rand_id_pool = []
assert_dict = {}
for num in xrange(2 * 20):
fc = FieldContainer(N.ones((500, 500)))
fc.data.flat[0] = num
fc.seal()
km.registerDataContainer(fc, temporary=True)
rand_id_pool.append(fc.id)
assert_dict[fc.id] = num
for size in sizes:
fc = FieldContainer(N.ones(size))
fc.seal()
km.registerDataContainer(fc, temporary=True)
ids.append(fc.id)
for id, bytes in zip(ids, byte_sizes):
uc_acc_time = 0.0
c_acc_time = 0.0
reps = 30
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
km._cache = []
km._cache_size = 0
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
print "Avr. access time for %0.2f kB sequential read: "\
"%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024,
uc_avr, c_avr)
km._cache = []
km._cache_size = 0
rand_ids = []
reps = 500
import pyphant.core.KnowledgeManager
pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20
for run in xrange(reps):
rand_ids.append(rand_id_pool[
random.randint(0, len(rand_id_pool) - 1)])
uc_acc_time = 0.0
c_acc_time = 0.0
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
assert km._cache_size >= 0
assert km._cache_size <= CACHE_MAX_SIZE
assert len(km._cache) <= CACHE_MAX_NUMBER
assert assert_dict[id] == fc.data.flat[0]
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
bytes = float(500 * 500 * 8) / 1024
print "Avr. access time for %0.2f kB random read: "\
"%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr)
print "------ End Cache Test ------"
def testCache(self):
print "Preparing FCs for cache test (cache size: %d MB)..."\
% (CACHE_MAX_SIZE / 1024 / 1024)
km = KnowledgeManager.getInstance()
sizes = [(20, ), (10, 10, 10), (200, 200), (700, 700), (2000, 2000)]
byte_sizes = [reduce(lambda x, y: x * y, size) * 8 for size in sizes]
ids = []
rand_id_pool = []
assert_dict = {}
for num in xrange(2 * 20):
fc = FieldContainer(N.ones((500, 500)))
fc.data.flat[0] = num
fc.seal()
km.registerDataContainer(fc, temporary=True)
rand_id_pool.append(fc.id)
assert_dict[fc.id] = num
for size in sizes:
fc = FieldContainer(N.ones(size))
fc.seal()
km.registerDataContainer(fc, temporary=True)
ids.append(fc.id)
for id, bytes in zip(ids, byte_sizes):
uc_acc_time = 0.0
c_acc_time = 0.0
reps = 30
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
km._cache = []
km._cache_size = 0
for rep in xrange(reps):
t1 = time()
km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
print "Avr. access time for %0.2f kB sequential read: "\
"%0.3f ms unchached, %0.3f ms cached" % (float(bytes) / 1024,
uc_avr, c_avr)
km._cache = []
km._cache_size = 0
rand_ids = []
reps = 500
import pyphant.core.KnowledgeManager
pyphant.core.KnowledgeManager.CACHE_MAX_NUMBER = 20
for run in xrange(reps):
rand_ids.append(rand_id_pool[random.randint(
0,
len(rand_id_pool) - 1)])
uc_acc_time = 0.0
c_acc_time = 0.0
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id, use_cache=False)
t2 = time()
uc_acc_time += t2 - t1
for id in rand_ids:
t1 = time()
fc = km.getDataContainer(id)
t2 = time()
c_acc_time += t2 - t1
assert km._cache_size >= 0
assert km._cache_size <= CACHE_MAX_SIZE
assert len(km._cache) <= CACHE_MAX_NUMBER
assert assert_dict[id] == fc.data.flat[0]
uc_avr = 1000.0 * uc_acc_time / reps
c_avr = 1000.0 * c_acc_time / reps
bytes = float(500 * 500 * 8) / 1024
print "Avr. access time for %0.2f kB random read: "\
"%0.3f ms unchached, %0.3f ms cached" % (bytes, uc_avr, c_avr)
print "------ End Cache Test ------"
def testExceptions(self):
km = KnowledgeManager.getInstance()
def testGetDataContainer(self):
km = KnowledgeManager.getInstance()
km.registerDataContainer(self._fc, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
def getDataContainer(self, subscriber=0):
emd5 = self.paramEmd5.value
kmanager = KnowledgeManager.getInstance()
return kmanager.getDataContainer(emd5)
def getDataContainer(self, subscriber=0):
emd5 = self.paramEmd5.value
kmanager = KnowledgeManager.getInstance()
return kmanager.getDataContainer(emd5)
