Example #1
0
def main():
    import sys
    filename = sys.argv[1]
    from pyphant.core import KnowledgeManager
    km = KnowledgeManager.KnowledgeManager.getInstance()
    import os.path
    km.registerURL("file://" + os.path.realpath(filename))
    import tables
    h5 = tables.openFile(filename, 'r+')
    from pyphant.core import PyTablesPersister
    recipe = PyTablesPersister.loadRecipe(h5)
    executionOrders = PyTablesPersister.loadExecutionOrders(h5)
    h5.close()
    from pyphant.core.Emd5Src import Emd5Src
    for order in executionOrders:
        for socket, emd5 in order[0].iteritems():
            sSpec = socket.split('.')
            w = recipe.getWorker(sSpec[0])
            s = getattr(w, sSpec[-1])
            src = Emd5Src(recipe)
            src.paramEmd5.value = emd5
            if s.isFull():
                s.pullPlug()
            s.insert(src.plugGetDataContainer)
        pSpec = order[1][0].split('.')
        d = recipe.getWorker(pSpec[0])
        plug = getattr(d, pSpec[1])
        res = plug.getResult()
        res.seal()
        h5 = tables.openFile(filename, 'r+')
        PyTablesPersister.saveResult(res, h5)
        h5.close()
Example #2
0
def main():
    km = KnowledgeManager.getInstance()
    sourcefile = sys.argv[1]
    h5 = tables.openFile(sourcefile, 'r+')
    options, args, recipe, orders = processArgs(h5)
    h5.close()
    import os.path
    km.registerURL("file://" + os.path.realpath(sourcefile))
    orderLists = []
    for order in orders:
        order = globOrder(order, None)
        singles = dict(
            [(socket, id_[0]) \
                 for socket, id_ in order[0].iteritems() if len(id_) == 1]
            )
        lists = dict(
            [(socket, id_) \
                 for socket, id_ in order[0].iteritems() if len(id_) > 1]
            )
        lens = [len(l) for l in lists.itervalues()]
        assert max(lens) == min(lens), "Illegal lengths."
        count = lens[0]
        orderList = []
        for i in xrange(count):
            sockMap = copy.deepcopy(singles)
            for s, l in lists.iteritems():
                sockMap[s] = l[i]
            orderList.append((sockMap, order[1]))
        orderLists.append(orderList)
    orders = sum(orderLists, [])
    if options.maxOrders == None:
        count = len(orders)
    else:
        count = options.maxOrders
    i = 0
    orderLists = []
    while i + count < len(orders):
        orderLists.append(orders[i: i + count])
        i += count
    orderLists.append(orders[i:])
    for i, orderList in enumerate(orderLists):
        filename = os.path.basename(sourcefile)[:-3] + '_%i.h5' % (i, )
        with H5FileHandler(filename, 'w') as handler:
            handler.saveRecipe(recipe)
        h5 = tables.openFile(filename, 'r+')
        for o in orderList:
            PyTablesPersister.saveExecutionOrder(h5, o)
            for data in o[0].values():
                dc = km.getDataContainer(data)
                PyTablesPersister.saveResult(dc, h5)
        h5.close()
Example #3
0
def globOrder(order, h5):
    sockMap = {}
    for sSpec in order[0].iteritems():
        if sSpec[1].startswith('emd5://'):
            sockMap[sSpec[0]] = [sSpec[1]]
        else:
            ids = []
            for f in glob.iglob(sSpec[1]):
                dc = f2dc(f)
                if h5 != None:
                    PyTablesPersister.saveResult(dc, h5)
                km.registerDataContainer(dc)
                ids.append(dc.id)
            sockMap[sSpec[0]] = ids
    return (sockMap, order[1])
Example #4
0
def globOrder(order, h5):
    sockMap = {}
    for sSpec in order[0].iteritems():
        if sSpec[1].startswith('emd5://'):
            sockMap[sSpec[0]] = [sSpec[1]]
        else:
            ids = []
            for f in glob.iglob(sSpec[1]):
                dc = f2dc(f)
                if h5 != None:
                    PyTablesPersister.saveResult(dc, h5)
                km.registerDataContainer(dc)
                ids.append(dc.id)
            sockMap[sSpec[0]] = ids
    return (sockMap, order[1])
Example #5
0
 def loadSample(self, resNode):
     """
     Loads a SampleContainer from the given node and returns it as an
     instance. This method is intended for internal use only.
     resNode -- node at which the SampleContainer is located in the file.
     """
     return PyTablesPersister.loadSample(self.handle, resNode)
Example #6
0
 def loadSample(self, resNode):
     """
     Loads a SampleContainer from the given node and returns it as an
     instance. This method is intended for internal use only.
     resNode -- node at which the SampleContainer is located in the file.
     """
     return PyTablesPersister.loadSample(self.handle, resNode)
Example #7
0
 def testGetLocalFile(self):
     h5fileid, h5name = tempfile.mkstemp(suffix='.h5',prefix='test-')
     os.close(h5fileid)
     h5 = tables.openFile(h5name,'w')
     resultsGroup = h5.createGroup("/", "results")
     ptp.saveResult(self._fc, h5)
     h5.close()
     km = KnowledgeManager.getInstance()
     from urllib import pathname2url
     url = pathname2url(h5name)
     if not url.startswith('///'):
         url = '//' + url
     url = 'file:' + url
     km.registerURL(url, temporary=True)
     km_fc = km.getDataContainer(self._fc.id)
     self.assertEqual(self._fc, km_fc)
     os.remove(h5name)
Example #8
0
    def saveRecipe(self, recipe, saveResults=True):
        """
        Saves a recipe

        recipe -- CompositeWorker to be saved
        saveResults -- Whether to save results of the workers
        """
        return PyTablesPersister.saveRecipe(self.handle, recipe, saveResults)
Example #9
0
    def saveRecipe(self, recipe, saveResults=True):
        """
        Saves a recipe

        recipe -- CompositeWorker to be saved
        saveResults -- Whether to save results of the workers
        """
        return PyTablesPersister.saveRecipe(self.handle, recipe, saveResults)
Example #10
0
 def testGetLocalFile(self):
     h5fileid, h5name = tempfile.mkstemp(suffix='.h5', prefix='test-')
     os.close(h5fileid)
     h5 = open_file(h5name, 'w')
     resultsGroup = h5.create_group("/", "results")
     ptp.saveResult(self._fc, h5)
     h5.close()
     km = KnowledgeManager.getInstance()
     from urllib import pathname2url
     url = pathname2url(h5name)
     if not url.startswith('///'):
         url = '//' + url
     url = 'file:' + url
     km.registerURL(url, temporary=True)
     km_fc = km.getDataContainer(self._fc.id)
     self.assertEqual(self._fc, km_fc)
     os.remove(h5name)
Example #11
0
 def saveField(self, resultGroup, result):
     """
     Saves a FieldContainer instance to the given node. This method is
     intended for internal use only.
     resultGroup -- node at which the FieldContainer should be saved
                    in the file.
     result -- FieldContainer instance to be saved
     """
     return PyTablesPersister.saveField(self.handle, resultGroup, result)
Example #12
0
def processArgs(h5):
    argv = sys.argv[1:]
    if '-h' in argv or '--help' in argv:
        help = True
    else:
        help = False
    args = filter(lambda a: not a in ['-h', '--help'], argv)
    parser = optparse.OptionParser()
    recipeFile = argv[0]
    recipe = PyTablesPersister.loadRecipe(h5)
    plugs = recipe.getAllPlugs()
    for p in plugs:
        parser.add_option(
            "",
            "--%s" % p.id,
            help="Request result of plug %s" % p.id,
            action="append_const",
            const=p,
            dest="requestedResults")
    (options, args) = parser.parse_args(filter(lambda o: 'plug' in o, argv))
    if hasattr(options,
               "requestedResults") and options.requestedResults and len(
                   options.requestedResults) > 0:
        openSockets = sum([
            recipe.getOpenSocketsForPlug(p) for p in options.requestedResults
        ], [])
        parser.values.socketMap = {}

        def optCallback(opt, opt_str, value, parser, socket):
            try:
                parser.values.socketMap[socket.id] = value
            except:
                parser.values.socketMap = {}
                parser.values.socketMap[socket.id] = value

        for s in openSockets:
            parser.add_option(
                "",
                "--%s" % s.id,
                help="Feed socket %s with data from argument." % s.id,
                action="callback",
                callback=optCallback,
                nargs=1,
                type='string',
                callback_args=(s, ))
    if help:
        args.append('-h')
    parser.add_option(
        "-n",
        "--max-orders-per-file",
        help="Maximal number of orders per file",
        type="int",
        dest="maxOrders")
    (options, args) = parser.parse_args(argv)
    # Order ::= (socketMap, resultPlug)
    return (options, args, recipe,
            [(options.socketMap, [p.id for p in options.requestedResults])])
Example #13
0
 def saveField(self, resultGroup, result):
     """
     Saves a FieldContainer instance to the given node. This method is
     intended for internal use only.
     resultGroup -- node at which the FieldContainer should be saved
                    in the file.
     result -- FieldContainer instance to be saved
     """
     return PyTablesPersister.saveField(self.handle, resultGroup, result)
Example #14
0
def processArgs(h5):
    argv = sys.argv[1:]
    if '-h' in argv or '--help' in argv:
        help = True
    else:
        help = False
    args = filter(lambda a: not a in ['-h', '--help'], argv)
    parser = optparse.OptionParser()
    recipeFile = argv[0]
    recipe = PyTablesPersister.loadRecipe(h5)
    plugs = recipe.getAllPlugs()
    for p in plugs:
        parser.add_option("",
                          "--%s" % p.id,
                          help="Request result of plug %s" % p.id,
                          action="append_const",
                          const=p,
                          dest="requestedResults")
    (options, args) = parser.parse_args(filter(lambda o: 'plug' in o, argv))
    if hasattr(options,
               "requestedResults") and options.requestedResults and len(
                   options.requestedResults) > 0:
        openSockets = sum([
            recipe.getOpenSocketsForPlug(p) for p in options.requestedResults
        ], [])
        parser.values.socketMap = {}

        def optCallback(opt, opt_str, value, parser, socket):
            try:
                parser.values.socketMap[socket.id] = value
            except:
                parser.values.socketMap = {}
                parser.values.socketMap[socket.id] = value

        for s in openSockets:
            parser.add_option("",
                              "--%s" % s.id,
                              help="Feed socket %s with data from argument." %
                              s.id,
                              action="callback",
                              callback=optCallback,
                              nargs=1,
                              type='string',
                              callback_args=(s, ))
    if help:
        args.append('-h')
    parser.add_option("-n",
                      "--max-orders-per-file",
                      help="Maximal number of orders per file",
                      type="int",
                      dest="maxOrders")
    (options, args) = parser.parse_args(argv)
    # Order ::= (socketMap, resultPlug)
    return (options, args, recipe,
            [(options.socketMap, [p.id for p in options.requestedResults])])
Example #15
0
 def testGetHTTPFile(self):
     host = "pyphant.sourceforge.net"
     remote_dir = ""
     url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5"
     # Get remote file and load DataContainer
     filename, headers = urllib.urlretrieve(url)
     h5 = tables.openFile(filename, 'r')
     for g in h5.walkGroups("/results"):
         if (len(g._v_attrs.TITLE)>0) \
                 and (r"\Psi" in g._v_attrs.shortname):
             http_fc = ptp.loadField(h5,g)
     h5.close()
     km = KnowledgeManager.getInstance()
     km.registerURL(url, temporary=True)
     km_fc = km.getDataContainer(http_fc.id)
     self.assertEqual(http_fc, km_fc)
     os.remove(filename)
Example #16
0
 def testGetHTTPFile(self):
     host = "pyphant.sourceforge.net"
     remote_dir = ""
     url = "http://" + host + remote_dir + "/knowledgemanager-http-test.h5"
     # Get remote file and load DataContainer
     filename, headers = urllib.urlretrieve(url)
     h5 = open_file(filename, 'r')
     for g in h5.walk_groups("/results"):
         if (len(g._v_attrs.TITLE)>0) \
                 and (r"\Psi" in g._v_attrs.shortname):
             http_fc = ptp.loadField(h5, g)
     h5.close()
     km = KnowledgeManager.getInstance()
     km.registerURL(url, temporary=True)
     km_fc = km.getDataContainer(http_fc.id)
     self.assertEqual(http_fc, km_fc)
     os.remove(filename)
Example #17
0
 def loadRecipe(self):
     return PyTablesPersister.loadRecipe(self.handle)
Example #18
0
 def loadRecipe(self):
     return PyTablesPersister.loadRecipe(self.handle)