def main():
import sys
filename = sys.argv[1]
from pyphant.core import KnowledgeManager
km = KnowledgeManager.KnowledgeManager.getInstance()
import os.path
km.registerURL("file://" + os.path.realpath(filename))
import tables
h5 = tables.openFile(filename, 'r+')
from pyphant.core import PyTablesPersister
recipe = PyTablesPersister.loadRecipe(h5)
executionOrders = PyTablesPersister.loadExecutionOrders(h5)
h5.close()
from pyphant.core.Emd5Src import Emd5Src
for order in executionOrders:
for socket, emd5 in order[0].iteritems():
sSpec = socket.split('.')
w = recipe.getWorker(sSpec[0])
s = getattr(w, sSpec[-1])
src = Emd5Src(recipe)
src.paramEmd5.value = emd5
if s.isFull():
s.pullPlug()
s.insert(src.plugGetDataContainer)
pSpec = order[1][0].split('.')
d = recipe.getWorker(pSpec[0])
plug = getattr(d, pSpec[1])
res = plug.getResult()
res.seal()
h5 = tables.openFile(filename, 'r+')
PyTablesPersister.saveResult(res, h5)
h5.close()
def main():
km = KnowledgeManager.getInstance()
sourcefile = sys.argv[1]
h5 = tables.openFile(sourcefile, 'r+')
options, args, recipe, orders = processArgs(h5)
h5.close()
import os.path
km.registerURL("file://" + os.path.realpath(sourcefile))
orderLists = []
for order in orders:
order = globOrder(order, None)
singles = dict(
[(socket, id_[0]) \
for socket, id_ in order[0].iteritems() if len(id_) == 1]
)
lists = dict(
[(socket, id_) \
for socket, id_ in order[0].iteritems() if len(id_) > 1]
)
lens = [len(l) for l in lists.itervalues()]
assert max(lens) == min(lens), "Illegal lengths."
count = lens[0]
orderList = []
for i in xrange(count):
sockMap = copy.deepcopy(singles)
for s, l in lists.iteritems():
sockMap[s] = l[i]
orderList.append((sockMap, order[1]))
orderLists.append(orderList)
orders = sum(orderLists, [])
if options.maxOrders == None:
count = len(orders)
else:
count = options.maxOrders
i = 0
orderLists = []
while i + count < len(orders):
orderLists.append(orders[i: i + count])
i += count
orderLists.append(orders[i:])
for i, orderList in enumerate(orderLists):
filename = os.path.basename(sourcefile)[:-3] + '_%i.h5' % (i, )
with H5FileHandler(filename, 'w') as handler:
handler.saveRecipe(recipe)
h5 = tables.openFile(filename, 'r+')
for o in orderList:
PyTablesPersister.saveExecutionOrder(h5, o)
for data in o[0].values():
dc = km.getDataContainer(data)
PyTablesPersister.saveResult(dc, h5)
h5.close()
def globOrder(order, h5):
sockMap = {}
for sSpec in order[0].iteritems():
if sSpec[1].startswith('emd5://'):
sockMap[sSpec[0]] = [sSpec[1]]
else:
ids = []
for f in glob.iglob(sSpec[1]):
dc = f2dc(f)
if h5 != None:
PyTablesPersister.saveResult(dc, h5)
km.registerDataContainer(dc)
ids.append(dc.id)
sockMap[sSpec[0]] = ids
return (sockMap, order[1])
def globOrder(order, h5):
sockMap = {}
for sSpec in order[0].iteritems():
if sSpec[1].startswith('emd5://'):
sockMap[sSpec[0]] = [sSpec[1]]
else:
ids = []
for f in glob.iglob(sSpec[1]):
dc = f2dc(f)
if h5 != None:
PyTablesPersister.saveResult(dc, h5)
km.registerDataContainer(dc)
ids.append(dc.id)
sockMap[sSpec[0]] = ids
return (sockMap, order[1])
def testGetLocalFile(self):
h5fileid, h5name = tempfile.mkstemp(suffix='.h5', prefix='test-')
os.close(h5fileid)
h5 = open_file(h5name, 'w')
resultsGroup = h5.create_group("/", "results")
ptp.saveResult(self._fc, h5)
h5.close()
km = KnowledgeManager.getInstance()
from urllib import pathname2url
url = pathname2url(h5name)
if not url.startswith('///'):
url = '//' + url
url = 'file:' + url
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
os.remove(h5name)
def testGetLocalFile(self):
h5fileid, h5name = tempfile.mkstemp(suffix='.h5',prefix='test-')
os.close(h5fileid)
h5 = tables.openFile(h5name,'w')
resultsGroup = h5.createGroup("/", "results")
ptp.saveResult(self._fc, h5)
h5.close()
km = KnowledgeManager.getInstance()
from urllib import pathname2url
url = pathname2url(h5name)
if not url.startswith('///'):
url = '//' + url
url = 'file:' + url
km.registerURL(url, temporary=True)
km_fc = km.getDataContainer(self._fc.id)
self.assertEqual(self._fc, km_fc)
os.remove(h5name)
