def main(): import sys filename = sys.argv[1] from pyphant.core import KnowledgeManager km = KnowledgeManager.KnowledgeManager.getInstance() import os.path km.registerURL("file://" + os.path.realpath(filename)) import tables h5 = tables.openFile(filename, 'r+') from pyphant.core import PyTablesPersister recipe = PyTablesPersister.loadRecipe(h5) executionOrders = PyTablesPersister.loadExecutionOrders(h5) h5.close() from pyphant.core.Emd5Src import Emd5Src for order in executionOrders: for socket, emd5 in order[0].iteritems(): sSpec = socket.split('.') w = recipe.getWorker(sSpec[0]) s = getattr(w, sSpec[-1]) src = Emd5Src(recipe) src.paramEmd5.value = emd5 if s.isFull(): s.pullPlug() s.insert(src.plugGetDataContainer) pSpec = order[1][0].split('.') d = recipe.getWorker(pSpec[0]) plug = getattr(d, pSpec[1]) res = plug.getResult() res.seal() h5 = tables.openFile(filename, 'r+') PyTablesPersister.saveResult(res, h5) h5.close()
def main(): km = KnowledgeManager.getInstance() sourcefile = sys.argv[1] h5 = tables.openFile(sourcefile, 'r+') options, args, recipe, orders = processArgs(h5) h5.close() import os.path km.registerURL("file://" + os.path.realpath(sourcefile)) orderLists = [] for order in orders: order = globOrder(order, None) singles = dict( [(socket, id_[0]) \ for socket, id_ in order[0].iteritems() if len(id_) == 1] ) lists = dict( [(socket, id_) \ for socket, id_ in order[0].iteritems() if len(id_) > 1] ) lens = [len(l) for l in lists.itervalues()] assert max(lens) == min(lens), "Illegal lengths." count = lens[0] orderList = [] for i in xrange(count): sockMap = copy.deepcopy(singles) for s, l in lists.iteritems(): sockMap[s] = l[i] orderList.append((sockMap, order[1])) orderLists.append(orderList) orders = sum(orderLists, []) if options.maxOrders == None: count = len(orders) else: count = options.maxOrders i = 0 orderLists = [] while i + count < len(orders): orderLists.append(orders[i: i + count]) i += count orderLists.append(orders[i:]) for i, orderList in enumerate(orderLists): filename = os.path.basename(sourcefile)[:-3] + '_%i.h5' % (i, ) with H5FileHandler(filename, 'w') as handler: handler.saveRecipe(recipe) h5 = tables.openFile(filename, 'r+') for o in orderList: PyTablesPersister.saveExecutionOrder(h5, o) for data in o[0].values(): dc = km.getDataContainer(data) PyTablesPersister.saveResult(dc, h5) h5.close()
def globOrder(order, h5): sockMap = {} for sSpec in order[0].iteritems(): if sSpec[1].startswith('emd5://'): sockMap[sSpec[0]] = [sSpec[1]] else: ids = [] for f in glob.iglob(sSpec[1]): dc = f2dc(f) if h5 != None: PyTablesPersister.saveResult(dc, h5) km.registerDataContainer(dc) ids.append(dc.id) sockMap[sSpec[0]] = ids return (sockMap, order[1])
def globOrder(order, h5): sockMap = {} for sSpec in order[0].iteritems(): if sSpec[1].startswith('emd5://'): sockMap[sSpec[0]] = [sSpec[1]] else: ids = [] for f in glob.iglob(sSpec[1]): dc = f2dc(f) if h5 != None: PyTablesPersister.saveResult(dc, h5) km.registerDataContainer(dc) ids.append(dc.id) sockMap[sSpec[0]] = ids return (sockMap, order[1])
def testGetLocalFile(self): h5fileid, h5name = tempfile.mkstemp(suffix='.h5', prefix='test-') os.close(h5fileid) h5 = open_file(h5name, 'w') resultsGroup = h5.create_group("/", "results") ptp.saveResult(self._fc, h5) h5.close() km = KnowledgeManager.getInstance() from urllib import pathname2url url = pathname2url(h5name) if not url.startswith('///'): url = '//' + url url = 'file:' + url km.registerURL(url, temporary=True) km_fc = km.getDataContainer(self._fc.id) self.assertEqual(self._fc, km_fc) os.remove(h5name)
def testGetLocalFile(self): h5fileid, h5name = tempfile.mkstemp(suffix='.h5',prefix='test-') os.close(h5fileid) h5 = tables.openFile(h5name,'w') resultsGroup = h5.createGroup("/", "results") ptp.saveResult(self._fc, h5) h5.close() km = KnowledgeManager.getInstance() from urllib import pathname2url url = pathname2url(h5name) if not url.startswith('///'): url = '//' + url url = 'file:' + url km.registerURL(url, temporary=True) km_fc = km.getDataContainer(self._fc.id) self.assertEqual(self._fc, km_fc) os.remove(h5name)