def main():
import sys
filename = sys.argv[1]
from pyphant.core import KnowledgeManager
km = KnowledgeManager.KnowledgeManager.getInstance()
import os.path
km.registerURL("file://" + os.path.realpath(filename))
import tables
h5 = tables.openFile(filename, 'r+')
from pyphant.core import PyTablesPersister
recipe = PyTablesPersister.loadRecipe(h5)
executionOrders = PyTablesPersister.loadExecutionOrders(h5)
h5.close()
from pyphant.core.Emd5Src import Emd5Src
for order in executionOrders:
for socket, emd5 in order[0].iteritems():
sSpec = socket.split('.')
w = recipe.getWorker(sSpec[0])
s = getattr(w, sSpec[-1])
src = Emd5Src(recipe)
src.paramEmd5.value = emd5
if s.isFull():
s.pullPlug()
s.insert(src.plugGetDataContainer)
pSpec = order[1][0].split('.')
d = recipe.getWorker(pSpec[0])
plug = getattr(d, pSpec[1])
res = plug.getResult()
res.seal()
h5 = tables.openFile(filename, 'r+')
PyTablesPersister.saveResult(res, h5)
h5.close()
def main():
km = KnowledgeManager.getInstance()
sourcefile = sys.argv[1]
h5 = tables.openFile(sourcefile, 'r+')
options, args, recipe, orders = processArgs(h5)
h5.close()
import os.path
km.registerURL("file://" + os.path.realpath(sourcefile))
orderLists = []
for order in orders:
order = globOrder(order, None)
singles = dict(
[(socket, id_[0]) \
for socket, id_ in order[0].iteritems() if len(id_) == 1]
)
lists = dict(
[(socket, id_) \
for socket, id_ in order[0].iteritems() if len(id_) > 1]
)
lens = [len(l) for l in lists.itervalues()]
assert max(lens) == min(lens), "Illegal lengths."
count = lens[0]
orderList = []
for i in xrange(count):
sockMap = copy.deepcopy(singles)
for s, l in lists.iteritems():
sockMap[s] = l[i]
orderList.append((sockMap, order[1]))
orderLists.append(orderList)
orders = sum(orderLists, [])
if options.maxOrders == None:
count = len(orders)
else:
count = options.maxOrders
i = 0
orderLists = []
while i + count < len(orders):
orderLists.append(orders[i: i + count])
i += count
orderLists.append(orders[i:])
for i, orderList in enumerate(orderLists):
filename = os.path.basename(sourcefile)[:-3] + '_%i.h5' % (i, )
with H5FileHandler(filename, 'w') as handler:
handler.saveRecipe(recipe)
h5 = tables.openFile(filename, 'r+')
for o in orderList:
PyTablesPersister.saveExecutionOrder(h5, o)
for data in o[0].values():
dc = km.getDataContainer(data)
PyTablesPersister.saveResult(dc, h5)
h5.close()
def globOrder(order, h5):
sockMap = {}
for sSpec in order[0].iteritems():
if sSpec[1].startswith('emd5://'):
sockMap[sSpec[0]] = [sSpec[1]]
else:
ids = []
for f in glob.iglob(sSpec[1]):
dc = f2dc(f)
if h5 != None:
PyTablesPersister.saveResult(dc, h5)
km.registerDataContainer(dc)
ids.append(dc.id)
sockMap[sSpec[0]] = ids
return (sockMap, order[1])
def globOrder(order, h5):
sockMap = {}
for sSpec in order[0].iteritems():
if sSpec[1].startswith('emd5://'):
sockMap[sSpec[0]] = [sSpec[1]]
else:
ids = []
for f in glob.iglob(sSpec[1]):
dc = f2dc(f)
if h5 != None:
PyTablesPersister.saveResult(dc, h5)
km.registerDataContainer(dc)
ids.append(dc.id)
sockMap[sSpec[0]] = ids
return (sockMap, order[1])
def loadSample(self, resNode):
"""
Loads a SampleContainer from the given node and returns it as an
instance. This method is intended for internal use only.
resNode -- node at which the SampleContainer is located in the file.
"""
return PyTablesPersister.loadSample(self.handle, resNode)
def loadSample(self, resNode):
"""
Loads a SampleContainer from the given node and returns it as an
instance. This method is intended for internal use only.
resNode -- node at which the SampleContainer is located in the file.
"""
return PyTablesPersister.loadSample(self.handle, resNode)
def testGetLocalFile(self):
h5fileid, h5name = tempfile.mkstemp(suffix='.h5',prefix='test-')
os.close(h5fileid)
h5 = tables.openFile(h5name,'w')
resultsGroup = h5.createGroup("/", "results")
ptp.saveResult(self._fc, h5)
h5.close()
km = KnowledgeManager.getInstance()
from urllib import pathname2url
url = pathname2url(h5name)
if not url.startswith('///'):
url = '//' + url
url = 'file:' + url
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
os.remove(h5name)
def saveRecipe(self, recipe, saveResults=True):
"""
Saves a recipe
recipe -- CompositeWorker to be saved
saveResults -- Whether to save results of the workers
"""
return PyTablesPersister.saveRecipe(self.handle, recipe, saveResults)
def saveRecipe(self, recipe, saveResults=True):
"""
Saves a recipe
recipe -- CompositeWorker to be saved
saveResults -- Whether to save results of the workers
"""
return PyTablesPersister.saveRecipe(self.handle, recipe, saveResults)
def testGetLocalFile(self):
h5fileid, h5name = tempfile.mkstemp(suffix='.h5', prefix='test-')
os.close(h5fileid)
h5 = open_file(h5name, 'w')
resultsGroup = h5.create_group("/", "results")
ptp.saveResult(self._fc, h5)
h5.close()
km = KnowledgeManager.getInstance()
from urllib import pathname2url
url = pathname2url(h5name)
if not url.startswith('///'):
url = '//' + url
url = 'file:' + url
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
os.remove(h5name)
def saveField(self, resultGroup, result):
"""
Saves a FieldContainer instance to the given node. This method is
intended for internal use only.
resultGroup -- node at which the FieldContainer should be saved
in the file.
result -- FieldContainer instance to be saved
"""
return PyTablesPersister.saveField(self.handle, resultGroup, result)
def processArgs(h5):
argv = sys.argv[1:]
if '-h' in argv or '--help' in argv:
help = True
else:
help = False
args = filter(lambda a: not a in ['-h', '--help'], argv)
parser = optparse.OptionParser()
recipeFile = argv[0]
recipe = PyTablesPersister.loadRecipe(h5)
plugs = recipe.getAllPlugs()
for p in plugs:
parser.add_option(
"",
"--%s" % p.id,
help="Request result of plug %s" % p.id,
action="append_const",
const=p,
dest="requestedResults")
(options, args) = parser.parse_args(filter(lambda o: 'plug' in o, argv))
if hasattr(options,
"requestedResults") and options.requestedResults and len(
options.requestedResults) > 0:
openSockets = sum([
recipe.getOpenSocketsForPlug(p) for p in options.requestedResults
], [])
parser.values.socketMap = {}
def optCallback(opt, opt_str, value, parser, socket):
try:
parser.values.socketMap[socket.id] = value
except:
parser.values.socketMap = {}
parser.values.socketMap[socket.id] = value
for s in openSockets:
parser.add_option(
"",
"--%s" % s.id,
help="Feed socket %s with data from argument." % s.id,
action="callback",
callback=optCallback,
nargs=1,
type='string',
callback_args=(s, ))
if help:
args.append('-h')
parser.add_option(
"-n",
"--max-orders-per-file",
help="Maximal number of orders per file",
type="int",
dest="maxOrders")
(options, args) = parser.parse_args(argv)
# Order ::= (socketMap, resultPlug)
return (options, args, recipe,
[(options.socketMap, [p.id for p in options.requestedResults])])
def saveField(self, resultGroup, result):
"""
Saves a FieldContainer instance to the given node. This method is
intended for internal use only.
resultGroup -- node at which the FieldContainer should be saved
in the file.
result -- FieldContainer instance to be saved
"""
return PyTablesPersister.saveField(self.handle, resultGroup, result)
def processArgs(h5):
argv = sys.argv[1:]
if '-h' in argv or '--help' in argv:
help = True
else:
help = False
args = filter(lambda a: not a in ['-h', '--help'], argv)
parser = optparse.OptionParser()
recipeFile = argv[0]
recipe = PyTablesPersister.loadRecipe(h5)
plugs = recipe.getAllPlugs()
for p in plugs:
parser.add_option("",
"--%s" % p.id,
help="Request result of plug %s" % p.id,
action="append_const",
const=p,
dest="requestedResults")
(options, args) = parser.parse_args(filter(lambda o: 'plug' in o, argv))
if hasattr(options,
"requestedResults") and options.requestedResults and len(
options.requestedResults) > 0:
openSockets = sum([
recipe.getOpenSocketsForPlug(p) for p in options.requestedResults
], [])
parser.values.socketMap = {}
def optCallback(opt, opt_str, value, parser, socket):
try:
parser.values.socketMap[socket.id] = value
except:
parser.values.socketMap = {}
parser.values.socketMap[socket.id] = value
for s in openSockets:
parser.add_option("",
"--%s" % s.id,
help="Feed socket %s with data from argument." %
s.id,
action="callback",
callback=optCallback,
nargs=1,
type='string',
callback_args=(s, ))
if help:
args.append('-h')
parser.add_option("-n",
"--max-orders-per-file",
help="Maximal number of orders per file",
type="int",
dest="maxOrders")
(options, args) = parser.parse_args(argv)
# Order ::= (socketMap, resultPlug)
return (options, args, recipe,
[(options.socketMap, [p.id for p in options.requestedResults])])
def testGetHTTPFile(self):
host = "pyphant.sourceforge.net"
remote_dir = ""
url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5"
# Get remote file and load DataContainer
filename, headers = urllib.urlretrieve(url)
h5 = tables.openFile(filename, 'r')
for g in h5.walkGroups("/results"):
if (len(g._v_attrs.TITLE)>0) \
and (r"\Psi" in g._v_attrs.shortname):
http_fc = ptp.loadField(h5,g)
h5.close()
km = KnowledgeManager.getInstance()
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(http_fc.id)
self.assertEqual(http_fc, km_fc)
os.remove(filename)
def testGetHTTPFile(self):
host = "pyphant.sourceforge.net"
remote_dir = ""
url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5"
# Get remote file and load DataContainer
filename, headers = urllib.urlretrieve(url)
h5 = open_file(filename, 'r')
for g in h5.walk_groups("/results"):
if (len(g._v_attrs.TITLE)>0) \
and (r"\Psi" in g._v_attrs.shortname):
http_fc = ptp.loadField(h5, g)
h5.close()
km = KnowledgeManager.getInstance()
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(http_fc.id)
self.assertEqual(http_fc, km_fc)
os.remove(filename)
def loadRecipe(self):
return PyTablesPersister.loadRecipe(self.handle)
def loadRecipe(self):
return PyTablesPersister.loadRecipe(self.handle)