Exemplo n.º 1
0
    def test_many_relationships(self):
        """
        A test in which a drop is related to many other drops that live in a
        separate DM (and thus requires many Pyro connections).

        Drop A is accessed by many applications (B1, B2, .., BN), which should
        not exhaust resources on DM #1 (in particular, the pyro thread pool).
        We collapse all into C so we can monitor only its status to know that
        the execution is over.

        DM #1                     DM #2
        =======    ====================
        |     |    | |--> B1 --|      |
        |     |    | |--> B2 --|      |
        | A --|----|-|--> B3 --|--> C |
        |     |    | |.........|      |
        |     |    | |--> BN --|      |
        =======    ====================
        """
        dm1 = NodeManager(useDLM=False)
        dm2 = NodeManager(useDLM=False)

        sessionId = 's1'
        N = 100
        g1 = [{"oid":"A", "type":"plain", "storage": "memory"}]
        g2 = [{"oid":"C", "type":"plain", "storage": "memory"}]
        for i in xrange(N):
            b_oid = "B%d" % (i,)
            # SleepAndCopyApp effectively opens the input drop
            g2.append({"oid":b_oid, "type":"app", "app":"test.graphsRepository.SleepAndCopyApp", "outputs":["C"], "sleepTime": 0})

        uris1 = dm1.quickDeploy(sessionId, g1)
        uris2 = dm2.quickDeploy(sessionId, g2)
        self.assertEquals(1,   len(uris1))
        self.assertEquals(1+N, len(uris2))

        # We externally wire the Proxy objects to establish the inter-DM
        # relationships. Make sure we release the proxies
        with Pyro4.Proxy(uris1['A']) as a:
            for i in xrange(N):
                with Pyro4.Proxy(uris2['B%d' % (i,)]) as b:
                    b.addInput(a, False)
                    a.addConsumer(b, False)

        # Run! The sole fact that this doesn't throw exceptions is already
        # a good proof that everything is working as expected
        c = Pyro4.Proxy(uris2['C'])
        with droputils.EvtConsumerProxyCtx(self, c, 5):
            a.write('a')
            a.setCompleted()

        dm1.destroySession(sessionId)
        dm2.destroySession(sessionId)
Exemplo n.º 2
0
    def test_runGraphOneDOPerDOM(self):
        """
        A test that creates three DROPs in two different DMs, wire two of
        them together externally (i.e., using their proxies), and runs the graph.
        For this the graphs that are fed into the DMs must *not* express the
        inter-DM relationships. The graph looks like:

        DM #1      DM #2
        =======    =============
        | A --|----|-> B --> C |
        =======    =============
        """
        dm1 = NodeManager(useDLM=False)
        dm2 = NodeManager(useDLM=False)

        sessionId = 's1'
        g1 = [{"oid":"A", "type":"plain", "storage": "memory"}]
        g2 = [{"oid":"B", "type":"app", "app":"dfms.apps.crc.CRCApp"},
              {"oid":"C", "type":"plain", "storage": "memory", "producers":["B"]}]

        uris1 = dm1.quickDeploy(sessionId, g1)
        uris2 = dm2.quickDeploy(sessionId, g2)
        self.assertEquals(1, len(uris1))
        self.assertEquals(2, len(uris2))

        # We externally wire the Proxy objects now
        a = Pyro4.Proxy(uris1['A'])
        b = Pyro4.Proxy(uris2['B'])
        c = Pyro4.Proxy(uris2['C'])
        a.addConsumer(b)

        # Run! We wait until c is completed
        with droputils.EvtConsumerProxyCtx(self, c, 1):
            a.write('a')
            a.setCompleted()

        for drop in a, b, c:
            self.assertEquals(DROPStates.COMPLETED, drop.status)
        self.assertEquals(a.checksum, int(droputils.allDropContents(c)))

        for dropProxy in a,b,c:
            dropProxy._pyroRelease()

        dm1.destroySession(sessionId)
        dm2.destroySession(sessionId)
Exemplo n.º 3
0
    def test_runWithFourDMs(self):
        """
        A test that creates several DROPs in two different DMs and  runs
        the graph. The graph looks like this

                      DM #2
                     +--------------------------+
                     |        |--> C --|        |
                 +---|--> B --|--> D --|--> F --|--|
                 |   |        |--> E --|        |  |
        DM #1    |   +--------------------------+  |   DM #4
        +-----+  |                                 |  +---------------------+
        |     |  |                                 |--|--> L --|            |
        | A --|--+                                    |        |--> N --> O |
        |     |  |                                 |--|--> M --|            |
        +-----+  |    DM #3                        |  +---------------------+
                 |   +--------------------------+  |
                 |   |        |--> H --|        |  |
                 +---|--> G --|--> I --|--> K --|--|
                     |        |--> J --|        |
                     +--------------------------+

        B, F, G, K and N are AppDOs; the rest are plain in-memory DROPs
        """

        dm1 = NodeManager(useDLM=False)
        dm2 = NodeManager(useDLM=False)
        dm3 = NodeManager(useDLM=False)
        dm4 = NodeManager(useDLM=False)

        sessionId = 's1'
        g1 = [memory('A', expectedSize=1)]
        g2 = [sleepAndCopy('B', outputs=['C','D','E'], sleepTime=0),
              memory('C'),
              memory('D'),
              memory('E'),
              sleepAndCopy('F', inputs=['C','D','E'], sleepTime=0)]
        g3 = [sleepAndCopy('G', outputs=['H','I','J'], sleepTime=0),
              memory('H'),
              memory('I'),
              memory('J'),
              sleepAndCopy('K', inputs=['H','I','J'], sleepTime=0)]
        g4 = [memory('L'),
              memory('M'),
              sleepAndCopy('N', inputs=['L','M'], outputs=['O'], sleepTime=0),
              memory('O')]

        uris1 = dm1.quickDeploy(sessionId, g1)
        uris2 = dm2.quickDeploy(sessionId, g2)
        uris3 = dm3.quickDeploy(sessionId, g3)
        uris4 = dm4.quickDeploy(sessionId, g4)
        self.assertEquals(1, len(uris1))
        self.assertEquals(5, len(uris2))
        self.assertEquals(5, len(uris3))
        self.assertEquals(4, len(uris4))
        allUris = {}
        allUris.update(uris1)
        allUris.update(uris2)
        allUris.update(uris3)
        allUris.update(uris4)

        # We externally wire the Proxy objects to establish the inter-DM
        # relationships. Intra-DM relationships are already established
        proxies = {}
        for uid,uri in allUris.viewitems():
            proxies[uid] = Pyro4.Proxy(uri)

        a = proxies['A']
        b = proxies['B']
        f = proxies['F']
        g = proxies['G']
        k = proxies['K']
        l = proxies['L']
        m = proxies['M']
        o = proxies['O']

        a.addConsumer(b)
        a.addConsumer(g)
        f.addOutput(l)
        k.addOutput(m)

        # Run! This should trigger the full execution of the graph
        with droputils.EvtConsumerProxyCtx(self, o, 1):
            a.write('a')

        for dropProxy in proxies.viewvalues():
            self.assertEquals(DROPStates.COMPLETED, dropProxy.status, "Status of '%s' is not COMPLETED: %d" % (dropProxy.uid, dropProxy.status))
            dropProxy._pyroRelease()

        for dm in [dm1, dm2, dm3, dm4]:
            dm.destroySession(sessionId)
Exemplo n.º 4
0
    def test_runGraphSeveralDropsPerDM(self):
        """
        A test that creates several DROPs in two different DMs and  runs
        the graph. The graph looks like this

        DM #1                  DM #2
        ===================    ================
        | A --> C --> D --|----|-|            |
        |                 |    | |--> E --> F |
        | B --------------|----|-|            |
        ===================    ================

        :see: `self.test_runGraphSingleDOPerDOM`
        """
        dm1 = NodeManager(useDLM=False)
        dm2 = NodeManager(useDLM=False)

        sessionId = 's1'
        g1 = [{"oid":"A", "type":"plain", "storage": "memory", "consumers":["C"]},
               {"oid":"B", "type":"plain", "storage": "memory"},
               {"oid":"C", "type":"app", "app":"dfms.apps.crc.CRCApp"},
               {"oid":"D", "type":"plain", "storage": "memory", "producers": ["C"]}]
        g2 = [{"oid":"E", "type":"app", "app":"test.test_drop.SumupContainerChecksum"},
               {"oid":"F", "type":"plain", "storage": "memory", "producers":["E"]}]

        uris1 = dm1.quickDeploy(sessionId, g1)
        uris2 = dm2.quickDeploy(sessionId, g2)
        self.assertEquals(4, len(uris1))
        self.assertEquals(2, len(uris2))

        # We externally wire the Proxy objects to establish the inter-DM
        # relationships
        a = Pyro4.Proxy(uris1['A'])
        b = Pyro4.Proxy(uris1['B'])
        c = Pyro4.Proxy(uris1['C'])
        d = Pyro4.Proxy(uris1['D'])
        e = Pyro4.Proxy(uris2['E'])
        f = Pyro4.Proxy(uris2['F'])
        for drop,uid in [(a,'A'),(b,'B'),(c,'C'),(d,'D'),(e,'E'),(f,'F')]:
            self.assertEquals(uid, drop.uid, "Proxy is not the DROP we think should be (assumed: %s/ actual: %s)" % (uid, drop.uid))
        e.addInput(d)
        e.addInput(b)

        # Run! The sole fact that this doesn't throw exceptions is already
        # a good proof that everything is working as expected
        with droputils.EvtConsumerProxyCtx(self, f, 5):
            a.write('a')
            a.setCompleted()
            b.write('a')
            b.setCompleted()

        for drop in a,b,c,d,e,f:
            self.assertEquals(DROPStates.COMPLETED, drop.status, "DROP %s is not COMPLETED" % (drop.uid))

        self.assertEquals(a.checksum, int(droputils.allDropContents(d)))
        self.assertEquals(b.checksum + d.checksum, int(droputils.allDropContents(f)))

        for dropProxy in a,b,c,d,e,f:
            dropProxy._pyroRelease()

        dm1.destroySession(sessionId)
        dm2.destroySession(sessionId)