예제 #1
0
파일: test_drop.py 프로젝트: koniolog/dfms
    def test_app_multiple_outputs(self):
        """
        A small method that tests that the AppDROPs writing to two
        different DROPs outputs works

        The graph constructed by this example looks as follow:

                              |--> E
        A --> B --> C --> D --|
                              |--> F

        Here B and D are an AppDROPs, with D writing to two DROPs
        outputs (E and F) and reading from C. C, in turn, is written by B, which
        in turns reads the data from A
        """

        # This is used as "B"
        class NumberWriterApp(BarrierAppDROP):
            def run(self):
                inputDrop = self.inputs[0]
                output = self.outputs[0]
                howMany = int(droputils.allDropContents(inputDrop))
                for i in range(howMany):
                    output.write(str(i) + " ")

        # This is used as "D"
        class OddAndEvenContainerApp(BarrierAppDROP):
            def run(self):
                inputDrop = self.inputs[0]
                outputs = self.outputs

                numbers = droputils.allDropContents(inputDrop).strip().split()
                for n in numbers:
                    outputs[int(n) % 2].write(n + six.b(" "))

        # Create DROPs
        a =     InMemoryDROP('oid:A', 'uid:A')
        b =        NumberWriterApp('oid:B', 'uid:B')
        c =     InMemoryDROP('oid:A', 'uid:A')
        d = OddAndEvenContainerApp('oid:D', 'uid:D')
        e =     InMemoryDROP('oid:E', 'uid:E')
        f =     InMemoryDROP('oid:F', 'uid:F')

        # Wire them together
        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
        d.addOutput(e)
        d.addOutput(f)

        # Start the execution
        with DROPWaiterCtx(self, [e,f]):
            a.write('20')
            a.setCompleted()

        # Check the final results are correct
        for drop in [a,b,c,d,e]:
            self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
        self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip())
        self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
예제 #2
0
    def test_app_multiple_outputs(self):
        """
        A small method that tests that the AppDROPs writing to two
        different DROPs outputs works

        The graph constructed by this example looks as follow:

                              |--> E
        A --> B --> C --> D --|
                              |--> F

        Here B and D are an AppDROPs, with D writing to two DROPs
        outputs (E and F) and reading from C. C, in turn, is written by B, which
        in turns reads the data from A
        """

        # This is used as "B"
        class NumberWriterApp(BarrierAppDROP):
            def run(self):
                inputDrop = self.inputs[0]
                output = self.outputs[0]
                howMany = int(droputils.allDropContents(inputDrop))
                for i in xrange(howMany):
                    output.write(str(i) + " ")

        # This is used as "D"
        class OddAndEvenContainerApp(BarrierAppDROP):
            def run(self):
                inputDrop = self.inputs[0]
                outputs = self.outputs

                numbers = droputils.allDropContents(inputDrop).strip().split()
                for n in numbers:
                    outputs[int(n) % 2].write(n + " ")

        # Create DROPs
        a = InMemoryDROP("oid:A", "uid:A")
        b = NumberWriterApp("oid:B", "uid:B")
        c = InMemoryDROP("oid:A", "uid:A")
        d = OddAndEvenContainerApp("oid:D", "uid:D")
        e = InMemoryDROP("oid:E", "uid:E")
        f = InMemoryDROP("oid:F", "uid:F")

        # Wire them together
        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
        d.addOutput(e)
        d.addOutput(f)

        # Start the execution
        with DROPWaiterCtx(self, [e, f]):
            a.write("20")
            a.setCompleted()

        # Check the final results are correct
        for drop in [a, b, c, d, e]:
            self.assertEquals(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
        self.assertEquals("0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip())
        self.assertEquals("1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
예제 #3
0
 def assertMsgIsCorrect(msg, command):
     a = DockerApp('a', 'a', image='ubuntu:14.04', command=command)
     b = FileDROP('b','b')
     a.addOutput(b)
     with DROPWaiterCtx(self, b, 100):
         a.execute()
     self.assertEquals(msg, droputils.allDropContents(b))
예제 #4
0
파일: test_drop.py 프로젝트: koniolog/dfms
 def run(self):
     drop = self.inputs[0]
     output = self.outputs[0]
     sortedLines = BytesIO(droputils.allDropContents(drop)).readlines()
     sortedLines.sort()
     for line in sortedLines:
         output.write(line)
예제 #5
0
파일: test_drop.py 프로젝트: koniolog/dfms
 def run(self):
     drop = self.inputs[0]
     output = self.outputs[0]
     allLines = BytesIO(droputils.allDropContents(drop)).readlines()
     for line in allLines:
         if self._substring in line:
             output.write(line)
예제 #6
0
    def test_speadApp(self):

        port = 1111
        itemId = 0x2000

        thread_pool = spead2.ThreadPool()
        self._stream = spead2.send.UdpStream(
            thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7))

        a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId)
        b = InMemoryDROP('b', 'b')
        a.addOutput(b)

        size = 1024
        threading.Thread(target=lambda: a.execute()).start()
        time.sleep(1)
        msg = os.urandom(size)
        with DROPWaiterCtx(self, b, timeout=1):
            ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48))
            item = ig.add_item(itemId,
                               'main_data',
                               'a char array',
                               shape=(size, ),
                               format=[('c', 8)])
            item.value = msg
            self._stream.send_heap(ig.get_heap())
            self._stream.send_heap(ig.get_end())

        for drop in a, b:
            self.assertEqual(DROPStates.COMPLETED, drop.status)

        self.assertEqual(size, b.size)
        self.assertEqual(msg, droputils.allDropContents(b))
예제 #7
0
 def run(self):
     drop = self.inputs[0]
     output = self.outputs[0]
     allLines = StringIO(droputils.allDropContents(drop)).readlines()
     for line in allLines:
         if self._substring in line:
             output.write(line)
예제 #8
0
    def test_speadApp(self):

        port = 1111
        itemId = 0x2000

        thread_pool = spead2.ThreadPool()
        self._stream = spead2.send.UdpStream(thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7))

        a = SpeadReceiverApp('a','a',port=port, itemId=itemId)
        b = InMemoryDROP('b','b')
        a.addOutput(b)

        size = 1024
        threading.Thread(target=lambda: a.execute()).start()
        time.sleep(1)
        msg = os.urandom(size)
        with DROPWaiterCtx(self, b, timeout=1):
            ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48))
            item = ig.add_item(itemId, 'main_data', 'a char array', shape=(size,), format=[('c',8)])
            item.value = msg
            self._stream.send_heap(ig.get_heap())
            self._stream.send_heap(ig.get_end())

        for drop in a,b:
            self.assertEquals(DROPStates.COMPLETED, drop.status)

        self.assertEquals(size, b.size)
        self.assertEquals(msg, droputils.allDropContents(b))
예제 #9
0
 def assert_message_is_correct(message, command):
     a = BashShellApp('a', 'a', command=command)
     b = FileDROP('b', 'b')
     a.addOutput(b)
     with DROPWaiterCtx(self, b, 100):
         a.execute()
     self.assertEquals(message, droputils.allDropContents(b))
예제 #10
0
            def run(self):
                inputDrop = self.inputs[0]
                outputs = self.outputs

                numbers = droputils.allDropContents(inputDrop).strip().split()
                for n in numbers:
                    outputs[int(n) % 2].write(n + " ")
예제 #11
0
 def run(self):
     drop = self.inputs[0]
     output = self.outputs[0]
     sortedLines = StringIO(droputils.allDropContents(drop)).readlines()
     sortedLines.sort()
     for line in sortedLines:
         output.write(line)
예제 #12
0
 def assertMsgIsCorrect(msg, command):
     a = DockerApp('a', 'a', image='ubuntu:14.04', command=command)
     b = FileDROP('b', 'b')
     a.addOutput(b)
     with DROPWaiterCtx(self, b, 100):
         a.execute()
     self.assertEqual(six.b(msg), droputils.allDropContents(b))
예제 #13
0
파일: test_bash.py 프로젝트: koniolog/dfms
 def assert_message_is_correct(message, command):
     a = BashShellApp('a', 'a', command=command)
     b = FileDROP('b', 'b')
     a.addOutput(b)
     with DROPWaiterCtx(self, b, 100):
         a.async_execute()
     self.assertEqual(six.b(message), droputils.allDropContents(b))
예제 #14
0
파일: test_drop.py 프로젝트: koniolog/dfms
    def test_dropWroteFromOutside(self):
        """
        A different scenario to those tested above, in which the data
        represented by the DROP isn't actually written *through* the
        DROP. Still, the DROP needs to be moved to COMPLETED once
        the data is written, and reading from it should still yield a correct
        result
        """

        # Write, but not through the DROP
        a = FileDROP('A', 'A')
        filename = a.path
        msg = 'a message'
        with open(filename, 'w') as f:
            f.write(msg)
        a.setCompleted()

        # Read from the DROP
        self.assertEqual(six.b(msg), droputils.allDropContents(a))
        self.assertIsNone(a.checksum)
        self.assertIsNone(a.size)

        # We can manually set the size because the DROP wasn't able to calculate
        # it itself; if we couldn't an exception would be thrown
        a.size = len(msg)
예제 #15
0
파일: test_drop.py 프로젝트: koniolog/dfms
            def run(self):
                inputDrop = self.inputs[0]
                outputs = self.outputs

                numbers = droputils.allDropContents(inputDrop).strip().split()
                for n in numbers:
                    outputs[int(n) % 2].write(n + six.b(" "))
예제 #16
0
    def test_dropWroteFromOutside(self):
        """
        A different scenario to those tested above, in which the data
        represented by the DROP isn't actually written *through* the
        DROP. Still, the DROP needs to be moved to COMPLETED once
        the data is written, and reading from it should still yield a correct
        result
        """

        # Write, but not through the DROP
        a = FileDROP("A", "A")
        filename = a.path
        msg = "a message"
        with open(filename, "w") as f:
            f.write(msg)
        a.setCompleted()

        # Read from the DROP
        self.assertEquals(msg, droputils.allDropContents(a))
        self.assertIsNone(a.checksum)
        self.assertIsNone(a.size)

        # We can manually set the size because the DROP wasn't able to calculate
        # it itself; if we couldn't an exception would be thrown
        a.size = len(msg)
예제 #17
0
    def test_objectAsNormalAndStreamingInput(self):
        """
        A test that checks that a DROP can act as normal and streaming input of
        different AppDROPs at the same time. We use the following graph:

        A --|--> B --> D
            |--> C --> E

        Here B uses A as a streaming input, while C uses it as a normal
        input
        """

        class LastCharWriterApp(AppDROP):
            def initialize(self, **kwargs):
                super(LastCharWriterApp, self).initialize(**kwargs)
                self._lastChar = None

            def dataWritten(self, uid, data):
                self.execStatus = AppDROPStates.RUNNING
                outputDrop = self.outputs[0]
                self._lastChar = data[-1]
                outputDrop.write(self._lastChar)

            def dropCompleted(self, uid, status):
                self.execStatus = AppDROPStates.FINISHED
                self._notifyAppIsFinished()

        a = InMemoryDROP("a", "a")
        b = LastCharWriterApp("b", "b")
        c = SumupContainerChecksum("c", "c")
        d = InMemoryDROP("d", "d")
        e = InMemoryDROP("e", "e")
        a.addStreamingConsumer(b)
        a.addConsumer(c)
        b.addOutput(d)
        c.addOutput(e)

        # Consumer cannot be normal and streaming at the same time
        self.assertRaises(Exception, lambda: a.addConsumer(b))
        self.assertRaises(Exception, lambda: a.addStreamingConsumer(c))

        # Write a little, then check the consumers
        def checkDropStates(aStatus, dStatus, eStatus, lastChar):
            self.assertEquals(aStatus, a.status)
            self.assertEquals(dStatus, d.status)
            self.assertEquals(eStatus, e.status)
            self.assertEquals(lastChar, b._lastChar)

        checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
        a.write("abcde")
        checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e")
        a.write("fghij")
        checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j")
        a.write("k")
        with DROPWaiterCtx(self, [d, e]):
            a.setCompleted()
        checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k")

        self.assertEquals("ejk", droputils.allDropContents(d))
예제 #18
0
파일: test_drop.py 프로젝트: koniolog/dfms
    def test_objectAsNormalAndStreamingInput(self):
        """
        A test that checks that a DROP can act as normal and streaming input of
        different AppDROPs at the same time. We use the following graph:

        A --|--> B --> D
            |--> C --> E

        Here B uses A as a streaming input, while C uses it as a normal
        input
        """

        class LastCharWriterApp(AppDROP):
            def initialize(self, **kwargs):
                super(LastCharWriterApp, self).initialize(**kwargs)
                self._lastByte = None
            def dataWritten(self, uid, data):
                self.execStatus = AppDROPStates.RUNNING
                outputDrop = self.outputs[0]
                self._lastByte = six.indexbytes(data, -1)
                outputDrop.write(self._lastByte)
            def dropCompleted(self, uid, status):
                self.execStatus = AppDROPStates.FINISHED
                self._notifyAppIsFinished()

        a = InMemoryDROP('a', 'a')
        b = LastCharWriterApp('b', 'b')
        c = SumupContainerChecksum('c', 'c')
        d = InMemoryDROP('d', 'd')
        e = InMemoryDROP('e', 'e')
        a.addStreamingConsumer(b)
        a.addConsumer(c)
        b.addOutput(d)
        c.addOutput(e)

        # Consumer cannot be normal and streaming at the same time
        self.assertRaises(Exception, a.addConsumer, b)
        self.assertRaises(Exception, a.addStreamingConsumer, c)

        # Write a little, then check the consumers
        def checkDropStates(aStatus, dStatus, eStatus, lastByte):
            self.assertEqual(aStatus, a.status)
            self.assertEqual(dStatus, d.status)
            self.assertEqual(eStatus, e.status)
            if lastByte is not None:
                self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte))

        checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
        a.write('abcde')
        checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e')
        a.write('fghij')
        checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j')
        a.write('k')
        with DROPWaiterCtx(self, [d,e]):
            a.setCompleted()
        checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k')

        self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
예제 #19
0
 def test_NullDROP(self):
     """
     Check that the NullDROP is usable for testing
     """
     a = NullDROP("A", "A", expectedSize=5)
     a.write("1234")
     a.write("5")
     allContents = droputils.allDropContents(a)
     self.assertFalse(allContents)
예제 #20
0
 def _ngas_and_fs_io(self, command):
     a = NgasDROP('a', 'a') # not a filesystem-related DROP, we can reference its URL in the command-line
     b = DockerApp('b', 'b', image="ubuntu:14.04", command=command)
     c = FileDROP('c', 'c')
     b.addInput(a)
     b.addOutput(c)
     with DROPWaiterCtx(self, b, 100):
         a.setCompleted()
     self.assertEquals(a.dataURL, droputils.allDropContents(c))
예제 #21
0
파일: test_drop.py 프로젝트: koniolog/dfms
 def test_NullDROP(self):
     """
     Check that the NullDROP is usable for testing
     """
     a = NullDROP('A', 'A', expectedSize=5)
     a.write("1234")
     a.write("5")
     allContents = droputils.allDropContents(a)
     self.assertFalse(allContents)
예제 #22
0
 def _ngas_and_fs_io(self, command):
     a = NgasDROP(
         'a', 'a'
     )  # not a filesystem-related DROP, we can reference its URL in the command-line
     b = DockerApp('b', 'b', image="ubuntu:14.04", command=command)
     c = FileDROP('c', 'c')
     b.addInput(a)
     b.addOutput(c)
     with DROPWaiterCtx(self, b, 100):
         a.setCompleted()
     self.assertEqual(six.b(a.dataURL), droputils.allDropContents(c))
예제 #23
0
파일: test_drop.py 프로젝트: koniolog/dfms
 def run(self):
     drop = self.inputs[0]
     output = self.outputs[0]
     allbytes = droputils.allDropContents(drop)
     buf = bytearray()
     for c in allbytes:
         if c == six.b(' ') or c == six.b('\n'):
             output.write(buf[::-1])
             output.write(c)
             buf = bytearray()
         else:
             buf.append(c)
예제 #24
0
 def run(self):
     drop = self.inputs[0]
     output = self.outputs[0]
     allLines = StringIO(droputils.allDropContents(drop)).readlines()
     for line in allLines:
         buf = ""
         for c in line:
             if c == " " or c == "\n":
                 output.write(buf[::-1])
                 output.write(c)
                 buf = ""
             else:
                 buf += c
예제 #25
0
파일: test_bash.py 프로젝트: koniolog/dfms
    def test_two_simultaneous_pipes(self):
        """
        A more complicated test where three bash applications run at the same
        time. The first streams its output to the second one, while the second
        one streams *its* output to the third one.

        -------------     --------------     -------------     --------------     -------------     ----------
        | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F |
        |   echo    |     | "/pipe1"   |     |    dc     |     | "/pipe2"   |     |   sort    |     |        |
        -----*-------     --------------     ----*--*-----     --------------     -----*-------     ----------
             |                                   |  |                                  |
             \-------------|named-pipe|----------\  \-----------|named-pipe|-----------/

        BashApp A writes "5 4 3 2 1" (each on a new line), which is read
        by "cat" (BashApp C). The printed results (a copy of the original) are
        streamed through D and read by "sort" (BashApp E), which writes the
        output to F.
        """

        output_fname = tempfile.mktemp()

        a = StreamingOutputBashApp('a',
                                   'a',
                                   command=r"echo -en '5\n4\n3\n2\n1'")
        b = InMemoryDROP('b', 'b')
        c = StreamingInputOutputBashApp('c', 'c', command="cat")
        d = InMemoryDROP('d', 'd')
        e = StreamingInputBashApp('e', 'e', command="sort -n > %o0")
        f = FileDROP('f', 'f', filepath=output_fname)

        a.addOutput(b)
        b.addStreamingConsumer(c)
        c.addOutput(d)
        d.addStreamingConsumer(e)
        e.addOutput(f)

        # Let's fire the app
        with DROPWaiterCtx(self, f, 2):
            a.async_execute()

        # The application executed, finished, and its output was recorded
        for drop in (a, b, c, d, e, f):
            self.assertEqual(DROPStates.COMPLETED, drop.status)
        self.assertEqual([1, 2, 3, 4, 5], [
            int(x)
            for x in droputils.allDropContents(f).strip().split(six.b('\n'))
        ])

        # Clean up and go
        os.remove(output_fname)
예제 #26
0
    def test_socket_listener(self):
        '''
        A simple test to check that SocketListenerApps are indeed working as
        expected; that is, they write the data they receive into their output,
        and finish when the connection is closed from the client side

        The data flow diagram looks like this:

        A --> B --> C --> D
        '''

        host = 'localhost'
        port = 9933
        data = os.urandom(1025)

        a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
        b = InMemoryDROP('oid:B', 'uid:B')
        c = SumupContainerChecksum('oid:C', 'uid:C')
        d = InMemoryDROP('oid:D', 'uid:D')
        a.addOutput(b)
        b.addConsumer(c)
        c.addOutput(d)

        # Create the socket, write, and close the connection, allowing
        # A to move to COMPLETED
        with DROPWaiterCtx(self, d, 3): # That's plenty of time
            a.async_execute()
            utils.write_to(host, port, data, 1)

        for drop in [a,b,c,d]:
            self.assertEqual(DROPStates.COMPLETED, drop.status)

        # Our expectations are fulfilled!
        bContents = droputils.allDropContents(b)
        dContents = int(droputils.allDropContents(d))
        self.assertEqual(data, bContents)
        self.assertEqual(crc32(data, 0), dContents)
예제 #27
0
    def test_socket_listener(self):
        '''
        A simple test to check that SocketListenerApps are indeed working as
        expected; that is, they write the data they receive into their output,
        and finish when the connection is closed from the client side

        The data flow diagram looks like this:

        A --> B --> C --> D
        '''

        host = 'localhost'
        port = 9933
        data = 'shine on you crazy diamond'

        a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
        b = InMemoryDROP('oid:B', 'uid:B')
        c = SumupContainerChecksum('oid:C', 'uid:C')
        d = InMemoryDROP('oid:D', 'uid:D')
        a.addOutput(b)
        b.addConsumer(c)
        c.addOutput(d)

        # Create the socket, write, and close the connection, allowing
        # A to move to COMPLETED
        with DROPWaiterCtx(self, d, 3): # That's plenty of time
            threading.Thread(target=lambda a: a.execute(), args=(a,)).start()
            utils.writeToRemotePort(host, port, data, 1)

        for drop in [a,b,c,d]:
            self.assertEquals(DROPStates.COMPLETED, drop.status)

        # Our expectations are fulfilled!
        bContents = droputils.allDropContents(b)
        dContents = int(droputils.allDropContents(d))
        self.assertEquals(data, bContents)
        self.assertEquals(crc32(data, 0), dContents)
예제 #28
0
    def test_clientServer(self):
        """
        A client-server duo. The server outputs the data it receives to its
        output DROP, which in turn is the data held in its input DROP. The graph
        looks like this:

        A --|--> B(client) --|--> D
            |--> C(server) --|

        C is a server application which B connects to. Therefore C must be
        started before B, so B knows C's IP address and connects successfully.
        Although the real writing is done by C, B in this example is also
        treated as a publisher of D. This way D waits for both applications to
        finish before proceeding.
        """
        try:
            AutoVersionClient().close()
        except DockerException:
            warnings.warn(
                "Cannot contact the Docker daemon, skipping docker tests")
            return

        a = FileDROP('a', 'a')
        b = DockerApp('b',
                      'b',
                      image='ubuntu:14.04',
                      command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
        c = DockerApp('c',
                      'c',
                      image='ubuntu:14.04',
                      command='nc -l 8000 > %o0')
        d = FileDROP('d', 'd')

        b.addInput(a)
        b.addOutput(d)
        c.addInput(a)
        c.addOutput(d)

        # Let 'b' handle its interest in c
        b.handleInterest(c)

        data = os.urandom(10)
        with DROPWaiterCtx(self, d, 100):
            a.write(data)
            a.setCompleted()

        self.assertEqual(data, droputils.allDropContents(d))
예제 #29
0
    def test_runGraphOneDOPerDOM(self):
        """
        A test that creates three DROPs in two different DMs and runs the graph.
        For this the graphs that are fed into the DMs must *not* express the
        inter-DM relationships, although they are still passed down
        separately. The graph looks like:

        DM #1      DM #2
        =======    =============
        | A --|----|-> B --> C |
        =======    =============
        """
        dm1, dm2 = [self._start_dm() for _ in range(2)]

        sessionId = 's1'
        g1 = [{"oid": "A", "type": "plain", "storage": "memory"}]
        g2 = [{
            "oid": "B",
            "type": "app",
            "app": "dfms.apps.crc.CRCApp"
        }, {
            "oid": "C",
            "type": "plain",
            "storage": "memory",
            "producers": ["B"]
        }]

        rels = [DROPRel('B', DROPLinkType.CONSUMER, 'A')]
        quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
        quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})
        self.assertEqual(1, len(dm1._sessions[sessionId].drops))
        self.assertEqual(2, len(dm2._sessions[sessionId].drops))

        # Run! We wait until c is completed
        a = dm1._sessions[sessionId].drops['A']
        b, c = [dm2._sessions[sessionId].drops[x] for x in ('B', 'C')]
        with droputils.DROPWaiterCtx(self, c, 1):
            a.write('a')
            a.setCompleted()

        for drop in a, b, c:
            self.assertEqual(DROPStates.COMPLETED, drop.status)
        self.assertEqual(a.checksum, int(droputils.allDropContents(c)))

        dm1.destroySession(sessionId)
        dm2.destroySession(sessionId)
예제 #30
0
파일: test_bash.py 프로젝트: koniolog/dfms
    def test_single_pipe(self):
        """
        A simple test where two bash apps are connected to each other in a
        streaming fashion. The data flows through a pipe which is created by
        the framework. The data drop in between acts only as a intermediator
        to establish the underlying communication channel.

        -------------     --------------     -------------     ----------
        | BashApp A | --> | InMemory B | --> | BashApp C | --> | File D |
        |   echo    |     | "/a/pipe"  |     |    dc     |     |        |
        -----*-------     --------------     ------*------     ----------
             |                                     |
             \-------------|named-pipe|------------/

        BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat
        and redirected to D.
        """

        output_fname = tempfile.mktemp()

        a = StreamingOutputBashApp('a',
                                   'a',
                                   command=r"echo -en '5\n4\n3\n2\n1'")
        b = InMemoryDROP('b', 'b')
        c = StreamingInputBashApp('c', 'c', command="cat > %o0")
        d = FileDROP('d', 'd', filepath=output_fname)

        a.addOutput(b)
        c.addStreamingInput(b)
        c.addOutput(d)

        # Let's fire the app
        with DROPWaiterCtx(self, d, 2):
            a.async_execute()

        # The application executed, finished, and its output was recorded
        for drop in (a, b, c, d):
            self.assertEqual(DROPStates.COMPLETED, drop.status,
                             "Drop %r not COMPLETED: %d" % (drop, drop.status))
        self.assertEqual(
            [5, 4, 3, 2, 1],
            [int(x) for x in droputils.allDropContents(d).split(six.b('\n'))])

        # Clean up and go
        os.remove(output_fname)
예제 #31
0
    def test_runGraphOneDOPerDOM(self):
        """
        A test that creates three DROPs in two different DMs, wire two of
        them together externally (i.e., using their proxies), and runs the graph.
        For this the graphs that are fed into the DMs must *not* express the
        inter-DM relationships. The graph looks like:

        DM #1      DM #2
        =======    =============
        | A --|----|-> B --> C |
        =======    =============
        """
        dm1 = NodeManager(useDLM=False)
        dm2 = NodeManager(useDLM=False)

        sessionId = 's1'
        g1 = [{"oid":"A", "type":"plain", "storage": "memory"}]
        g2 = [{"oid":"B", "type":"app", "app":"dfms.apps.crc.CRCApp"},
              {"oid":"C", "type":"plain", "storage": "memory", "producers":["B"]}]

        uris1 = dm1.quickDeploy(sessionId, g1)
        uris2 = dm2.quickDeploy(sessionId, g2)
        self.assertEquals(1, len(uris1))
        self.assertEquals(2, len(uris2))

        # We externally wire the Proxy objects now
        a = Pyro4.Proxy(uris1['A'])
        b = Pyro4.Proxy(uris2['B'])
        c = Pyro4.Proxy(uris2['C'])
        a.addConsumer(b)

        # Run! We wait until c is completed
        with droputils.EvtConsumerProxyCtx(self, c, 1):
            a.write('a')
            a.setCompleted()

        for drop in a, b, c:
            self.assertEquals(DROPStates.COMPLETED, drop.status)
        self.assertEquals(a.checksum, int(droputils.allDropContents(c)))

        for dropProxy in a,b,c:
            dropProxy._pyroRelease()

        dm1.destroySession(sessionId)
        dm2.destroySession(sessionId)
예제 #32
0
    def test_echo(self):
        a = FileDROP('a', 'a')
        b = BashShellApp('b', 'b', command='cp %i0 %o0')
        c = FileDROP('c', 'c')

        b.addInput(a)
        b.addOutput(c)

        # Random data so we always check different contents
        data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
        with DROPWaiterCtx(self, c, 100):
            a.write(data)
            a.setCompleted()

        self.assertEquals(data, droputils.allDropContents(c))

        # We own the file, not root
        uid = os.getuid()
        self.assertEquals(uid, os.stat(c.path).st_uid)
예제 #33
0
파일: test_bash.py 프로젝트: koniolog/dfms
    def test_echo(self):
        a = FileDROP('a', 'a')
        b = BashShellApp('b', 'b', command='cp %i0 %o0')
        c = FileDROP('c', 'c')

        b.addInput(a)
        b.addOutput(c)

        # Random data so we always check different contents
        data = os.urandom(10)
        with DROPWaiterCtx(self, c, 100):
            a.write(data)
            a.setCompleted()

        self.assertEqual(data, droputils.allDropContents(c))

        # We own the file, not root
        uid = os.getuid()
        self.assertEqual(uid, os.stat(c.path).st_uid)
예제 #34
0
    def test_clientServer(self):
        """
        A client-server duo. The server outputs the data it receives to its
        output DROP, which in turn is the data held in its input DROP. The graph
        looks like this:

        A --|--> B(client) --|--> D
            |--> C(server) --|

        C is a server application which B connects to. Therefore C must be
        started before B, so B knows C's IP address and connects successfully.
        Although the real writing is done by C, B in this example is also
        treated as a publisher of D. This way D waits for both applications to
        finish before proceeding.
        """
        try:
            AutoVersionClient()
        except DockerException:
            warnings.warn("Cannot contact the Docker daemon, skipping docker tests")
            return

        a = FileDROP('a', 'a')
        b = DockerApp('b', 'b', image='ubuntu:14.04', command='cat %i0 > /dev/tcp/%containerIp[c]%/8000')
        c = DockerApp('c', 'c', image='ubuntu:14.04', command='nc -l 8000 > %o0')
        d = FileDROP('d', 'd')

        b.addInput(a)
        b.addOutput(d)
        c.addInput(a)
        c.addOutput(d)

        # Let 'b' handle its interest in c
        b.handleInterest(c)

        data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
        with DROPWaiterCtx(self, d, 100):
            a.write(data)
            a.setCompleted()

        self.assertEquals(data, droputils.allDropContents(d))
예제 #35
0
    def _test_write_withDropType(self, dropType):
        """
        Test an AbstractDROP and a simple AppDROP (for checksum calculation)
        """
        a = dropType("oid:A", "uid:A", expectedSize=self._test_drop_sz * ONE_MB)
        b = SumupContainerChecksum("oid:B", "uid:B")
        c = InMemoryDROP("oid:C", "uid:C")
        b.addInput(a)
        b.addOutput(c)

        test_crc = 0
        with DROPWaiterCtx(self, c):
            for _ in range(self._test_num_blocks):
                a.write(self._test_block)
                test_crc = crc32(self._test_block, test_crc)

        # Read the checksum from c
        cChecksum = int(droputils.allDropContents(c))

        self.assertNotEquals(a.checksum, 0)
        self.assertEquals(a.checksum, test_crc)
        self.assertEquals(cChecksum, test_crc)
예제 #36
0
파일: test_drop.py 프로젝트: koniolog/dfms
    def _test_write_withDropType(self, dropType):
        """
        Test an AbstractDROP and a simple AppDROP (for checksum calculation)
        """
        a = dropType('oid:A', 'uid:A', expectedSize = self._test_drop_sz * ONE_MB)
        b = SumupContainerChecksum('oid:B', 'uid:B')
        c = InMemoryDROP('oid:C', 'uid:C')
        b.addInput(a)
        b.addOutput(c)

        test_crc = 0
        with DROPWaiterCtx(self, c):
            for _ in range(self._test_num_blocks):
                a.write(self._test_block)
                test_crc = crc32(self._test_block, test_crc)

        # Read the checksum from c
        cChecksum = int(droputils.allDropContents(c))

        self.assertNotEqual(a.checksum, 0)
        self.assertEqual(a.checksum, test_crc)
        self.assertEqual(cChecksum, test_crc)
예제 #37
0
    def test_simpleCopy(self):
        """
        Simple test for a dockerized application. It copies the contents of one
        file into another via the command-line cp utility. It then checks that
        the contents of the target DROP are correct, and that the target file is
        actually owned by our process.

        The test will not run if a docker daemon cannot be contacted though;
        this is to avoid failures in machines that don't have a docker service
        running.
        """

        try:
            AutoVersionClient().close()
        except DockerException:
            warnings.warn(
                "Cannot contact the Docker daemon, skipping docker tests")
            return

        a = FileDROP('a', 'a')
        b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
        c = FileDROP('c', 'c')

        b.addInput(a)
        b.addOutput(c)

        # Random data so we always check different contents
        data = os.urandom(10)
        with DROPWaiterCtx(self, c, 100):
            a.write(data)
            a.setCompleted()

        self.assertEqual(data, droputils.allDropContents(c))

        # We own the file, not root
        uid = os.getuid()
        self.assertEqual(uid, os.stat(c.path).st_uid)
예제 #38
0
    def test_simpleCopy(self):
        """
        Simple test for a dockerized application. It copies the contents of one
        file into another via the command-line cp utility. It then checks that
        the contents of the target DROP are correct, and that the target file is
        actually owned by our process.

        The test will not run if a docker daemon cannot be contacted though;
        this is to avoid failures in machines that don't have a docker service
        running.
        """

        try:
            AutoVersionClient()
        except DockerException:
            warnings.warn("Cannot contact the Docker daemon, skipping docker tests")
            return

        a = FileDROP('a', 'a')
        b = DockerApp('b', 'b', image='ubuntu:14.04', command='cp %i0 %o0')
        c = FileDROP('c', 'c')

        b.addInput(a)
        b.addOutput(c)

        # Random data so we always check different contents
        data = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
        with DROPWaiterCtx(self, c, 100):
            a.write(data)
            a.setCompleted()

        self.assertEquals(data, droputils.allDropContents(c))

        # We own the file, not root
        uid = os.getuid()
        self.assertEquals(uid, os.stat(c.path).st_uid)
예제 #39
0
    def test_simple_chain(self):
        """
        Simple test that creates a pipeline-like chain of commands.
        In this case we simulate a pipeline that does this, holding
        each intermediate result in memory:

        cat someFile | grep 'a' | sort | rev
        """

        class GrepResult(BarrierAppDROP):
            def initialize(self, **kwargs):
                super(GrepResult, self).initialize(**kwargs)
                self._substring = kwargs["substring"]

            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                allLines = StringIO(droputils.allDropContents(drop)).readlines()
                for line in allLines:
                    if self._substring in line:
                        output.write(line)

        class SortResult(BarrierAppDROP):
            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                sortedLines = StringIO(droputils.allDropContents(drop)).readlines()
                sortedLines.sort()
                for line in sortedLines:
                    output.write(line)

        class RevResult(BarrierAppDROP):
            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                allLines = StringIO(droputils.allDropContents(drop)).readlines()
                for line in allLines:
                    buf = ""
                    for c in line:
                        if c == " " or c == "\n":
                            output.write(buf[::-1])
                            output.write(c)
                            buf = ""
                        else:
                            buf += c

        a = InMemoryDROP("oid:A", "uid:A")
        b = GrepResult("oid:B", "uid:B", substring="a")
        c = InMemoryDROP("oid:C", "uid:C")
        d = SortResult("oid:D", "uid:D")
        e = InMemoryDROP("oid:E", "uid:E")
        f = RevResult("oid:F", "oid:F")
        g = InMemoryDROP("oid:G", "uid:G")

        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
        d.addOutput(e)
        e.addConsumer(f)
        f.addOutput(g)

        # Initial write
        contents = "first line\nwe have an a here\nand another one\nnoone knows me"
        cResExpected = "we have an a here\nand another one\n"
        eResExpected = "and another one\nwe have an a here\n"
        gResExpected = "dna rehtona eno\new evah na a ereh\n"

        with DROPWaiterCtx(self, g):
            a.write(contents)
            a.setCompleted()

        # Get intermediate and final results and compare
        actualRes = []
        for i in [c, e, g]:
            actualRes.append(droputils.allDropContents(i))
        map(lambda x, y: self.assertEquals(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
예제 #40
0
    def test_join(self):
        """
        Using the container data object to implement a join/barrier dataflow.

        A1, A2 and A3 are FileDROPs
        B1, B2 and B3 are SumupContainerChecksum
        C1, C2 and C3 are InMemoryDROPs
        D is a SumupContainerChecksum
        E is a InMemoryDROP

        --> A1 --> B1 --> C1 --|
        --> A2 --> B2 --> C2 --|--> D --> E
        --> A3 --> B3 --> C3 --|

        Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
        after which "C" will transition to COMPLETE. Once all "C"s have moved to
        COMPLETED "D"'s execution will also be triggered, and finally E will
        hold the sum of B1, B2 and B3's checksums
        """

        filelen = self._test_drop_sz * ONE_MB
        # create file data objects
        a1 = FileDROP("oid:A1", "uid:A1", expectedSize=filelen)
        a2 = FileDROP("oid:A2", "uid:A2", expectedSize=filelen)
        a3 = FileDROP("oid:A3", "uid:A3", expectedSize=filelen)

        # CRC Result DROPs, storing the result in memory
        b1 = SumupContainerChecksum("oid:B1", "uid:B1")
        b2 = SumupContainerChecksum("oid:B2", "uid:B2")
        b3 = SumupContainerChecksum("oid:B3", "uid:B3")
        c1 = InMemoryDROP("oid:C1", "uid:C1")
        c2 = InMemoryDROP("oid:C2", "uid:C2")
        c3 = InMemoryDROP("oid:C3", "uid:C3")

        # The final DROP that sums up the CRCs from the container DROP
        d = SumupContainerChecksum("oid:D", "uid:D")
        e = InMemoryDROP("oid:E", "uid:E")

        # Wire together
        dropAList = [a1, a2, a3]
        dropBList = [b1, b2, b3]
        dropCList = [c1, c2, c3]
        for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList):
            dropA.addConsumer(dropB)
        for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList):
            dropB.addOutput(dropC)
        for dropC in dropCList:
            dropC.addConsumer(d)
        d.addOutput(e)

        # Write data into the initial "A" DROPs, which should trigger
        # the whole chain explained above
        with DROPWaiterCtx(self, e):
            for dropA in dropAList:  # this should be parallel for
                for _ in range(self._test_num_blocks):
                    dropA.write(self._test_block)

        # All DROPs are completed now that the chain executed correctly
        for drop in dropAList + dropBList + dropCList:
            self.assertEqual(drop.status, DROPStates.COMPLETED)

        # The results we want to compare
        sum_crc = c1.checksum + c2.checksum + c3.checksum
        dropEData = int(droputils.allDropContents(e))

        self.assertNotEquals(sum_crc, 0)
        self.assertEquals(sum_crc, dropEData)
예제 #41
0
파일: test_drop.py 프로젝트: koniolog/dfms
    def test_simple_chain(self):
        '''
        Simple test that creates a pipeline-like chain of commands.
        In this case we simulate a pipeline that does this, holding
        each intermediate result in memory:

        cat someFile | grep 'a' | sort | rev
        '''

        class GrepResult(BarrierAppDROP):
            def initialize(self, **kwargs):
                super(GrepResult, self).initialize(**kwargs)
                self._substring = six.b(kwargs['substring'])

            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                allLines = BytesIO(droputils.allDropContents(drop)).readlines()
                for line in allLines:
                    if self._substring in line:
                        output.write(line)

        class SortResult(BarrierAppDROP):
            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                sortedLines = BytesIO(droputils.allDropContents(drop)).readlines()
                sortedLines.sort()
                for line in sortedLines:
                    output.write(line)

        class RevResult(BarrierAppDROP):
            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                allbytes = droputils.allDropContents(drop)
                buf = bytearray()
                for c in allbytes:
                    if c == six.b(' ') or c == six.b('\n'):
                        output.write(buf[::-1])
                        output.write(c)
                        buf = bytearray()
                    else:
                        buf.append(c)

        a = InMemoryDROP('oid:A', 'uid:A')
        b = GrepResult('oid:B', 'uid:B', substring="a")
        c = InMemoryDROP('oid:C', 'uid:C')
        d = SortResult('oid:D', 'uid:D')
        e = InMemoryDROP('oid:E', 'uid:E')
        f = RevResult('oid:F', 'oid:F')
        g = InMemoryDROP('oid:G', 'uid:G')

        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
        d.addOutput(e)
        e.addConsumer(f)
        f.addOutput(g)

        # Initial write
        contents = "first line\nwe have an a here\nand another one\nnoone knows me"
        cResExpected = "we have an a here\nand another one\n"
        eResExpected = "and another one\nwe have an a here\n"
        gResExpected = "dna rehtona eno\new evah na a ereh\n"

        with DROPWaiterCtx(self, g):
            a.write(contents)
            a.setCompleted()

        # Get intermediate and final results and compare
        actualRes   = []
        for i in [c, e, g]:
            actualRes.append(droputils.allDropContents(i))
        map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
예제 #42
0
파일: test_drop.py 프로젝트: koniolog/dfms
    def branch_failure(self, tooManyFailures):
        """
        Using the container data object to implement a join/barrier dataflow.

        A1, A2 and A3 are FileDROPs
        B1, B2 and B3 are SumupContainerChecksum
        C1, C2 and C3 are InMemoryDROPs
        D is a SumupContainerChecksum
        E is a InMemoryDROP

        --> A1 --> B1 --> C1 --|
        --> A2 --> B2 --> C2 --|--> D --> E
        --> A3 --> B3 --> C3 --|

        Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
        after which "C" will transition to COMPLETE. Once all "C"s have moved to
        COMPLETED "D"'s execution will also be triggered, and finally E will
        hold the sum of B1, B2 and B3's checksums
        """

        #create file data objects
        a1 = InMemoryDROP('oid:A1', 'uid:A1')
        a2 = InMemoryDROP('oid:A2', 'uid:A2')
        a3 = InMemoryDROP('oid:A3', 'uid:A3')

        # CRC Result DROPs, storing the result in memory
        b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
        b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
        b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
        c1 = InMemoryDROP('oid:C1', 'uid:C1')
        c2 = InMemoryDROP('oid:C2', 'uid:C2')
        c3 = InMemoryDROP('oid:C3', 'uid:C3')

        # The final DROP that sums up the CRCs from the container DROP
        d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33)
        e = InMemoryDROP('oid:E', 'uid:E')

        # Wire together
        dropAList = [a1,a2,a3]
        dropBList = [b1,b2,b3]
        dropCList = [c1,c2,c3]
        for dropA,dropB in zip(dropAList, dropBList):
            dropA.addConsumer(dropB)
        for dropB,dropC in zip(dropBList, dropCList):
            dropB.addOutput(dropC)
        for dropC in dropCList:
            dropC.addConsumer(d)
        d.addOutput(e)

        # Write data into the initial "A" DROPs, which should trigger
        # the whole chain explained above
        with DROPWaiterCtx(self, e):
            #for dropA in dropAList: # this should be parallel for
            a1.write(' '); a1.setCompleted()
            if tooManyFailures:
                a2.setError()
            else:
                a2.write(' '); a2.setCompleted()
            a3.setError()

        if tooManyFailures:
            completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
            errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
        else:
            completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
            errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]

        for drop in completedDrops:
            self.assertEqual(drop.status, DROPStates.COMPLETED)
        for drop in errorDrops:
            self.assertEqual(drop.status, DROPStates.ERROR)

        # The results we want to compare
        # (only in case that at least two branches executed)
        if not tooManyFailures:
            sum_crc = c1.checksum + c2.checksum
            dropEData = int(droputils.allDropContents(e))

            self.assertNotEqual(sum_crc, 0)
            self.assertEqual(sum_crc, dropEData)
예제 #43
0
파일: test_drop.py 프로젝트: koniolog/dfms
    def test_join(self):
        """
        Using the container data object to implement a join/barrier dataflow.

        A1, A2 and A3 are FileDROPs
        B1, B2 and B3 are SumupContainerChecksum
        C1, C2 and C3 are InMemoryDROPs
        D is a SumupContainerChecksum
        E is a InMemoryDROP

        --> A1 --> B1 --> C1 --|
        --> A2 --> B2 --> C2 --|--> D --> E
        --> A3 --> B3 --> C3 --|

        Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
        after which "C" will transition to COMPLETE. Once all "C"s have moved to
        COMPLETED "D"'s execution will also be triggered, and finally E will
        hold the sum of B1, B2 and B3's checksums
        """

        filelen = self._test_drop_sz * ONE_MB
        #create file data objects
        a1 = FileDROP('oid:A1', 'uid:A1', expectedSize=filelen)
        a2 = FileDROP('oid:A2', 'uid:A2', expectedSize=filelen)
        a3 = FileDROP('oid:A3', 'uid:A3', expectedSize=filelen)

        # CRC Result DROPs, storing the result in memory
        b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
        b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
        b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
        c1 = InMemoryDROP('oid:C1', 'uid:C1')
        c2 = InMemoryDROP('oid:C2', 'uid:C2')
        c3 = InMemoryDROP('oid:C3', 'uid:C3')

        # The final DROP that sums up the CRCs from the container DROP
        d = SumupContainerChecksum('oid:D', 'uid:D')
        e = InMemoryDROP('oid:E', 'uid:E')

        # Wire together
        dropAList = [a1,a2,a3]
        dropBList = [b1,b2,b3]
        dropCList = [c1,c2,c3]
        for dropA,dropB in map(lambda a,b: (a,b), dropAList, dropBList):
            dropA.addConsumer(dropB)
        for dropB,dropC in map(lambda b,c: (b,c), dropBList, dropCList):
            dropB.addOutput(dropC)
        for dropC in dropCList:
            dropC.addConsumer(d)
        d.addOutput(e)

        # Write data into the initial "A" DROPs, which should trigger
        # the whole chain explained above
        with DROPWaiterCtx(self, e):
            for dropA in dropAList: # this should be parallel for
                for _ in range(self._test_num_blocks):
                    dropA.write(self._test_block)

        # All DROPs are completed now that the chain executed correctly
        for drop in dropAList + dropBList + dropCList:
            self.assertEqual(drop.status, DROPStates.COMPLETED)

        # The results we want to compare
        sum_crc = c1.checksum + c2.checksum + c3.checksum
        dropEData = int(droputils.allDropContents(e))

        self.assertNotEqual(sum_crc, 0)
        self.assertEqual(sum_crc, dropEData)
예제 #44
0
파일: test_drop.py 프로젝트: koniolog/dfms
 def run(self):
     inputDrop = self.inputs[0]
     output = self.outputs[0]
     howMany = int(droputils.allDropContents(inputDrop))
     for i in range(howMany):
         output.write(str(i) + " ")
예제 #45
0
 def run(self):
     inputDrop = self.inputs[0]
     output = self.outputs[0]
     howMany = int(droputils.allDropContents(inputDrop))
     for i in xrange(howMany):
         output.write(str(i) + " ")
예제 #46
0
    def test_runGraphSeveralDropsPerDM(self):
        """
        A test that creates several DROPs in two different DMs and  runs
        the graph. The graph looks like this

        DM #1                  DM #2
        ===================    ================
        | A --> C --> D --|----|-|            |
        |                 |    | |--> E --> F |
        | B --------------|----|-|            |
        ===================    ================

        :see: `self.test_runGraphSingleDOPerDOM`
        """
        dm1, dm2 = [self._start_dm() for _ in range(2)]

        sessionId = 's1'
        g1 = [{
            "oid": "A",
            "type": "plain",
            "storage": "memory",
            "consumers": ["C"]
        }, {
            "oid": "B",
            "type": "plain",
            "storage": "memory"
        }, {
            "oid": "C",
            "type": "app",
            "app": "dfms.apps.crc.CRCApp"
        }, {
            "oid": "D",
            "type": "plain",
            "storage": "memory",
            "producers": ["C"]
        }]
        g2 = [{
            "oid": "E",
            "type": "app",
            "app": "test.test_drop.SumupContainerChecksum"
        }, {
            "oid": "F",
            "type": "plain",
            "storage": "memory",
            "producers": ["E"]
        }]

        rels = [
            DROPRel('D', DROPLinkType.INPUT, 'E'),
            DROPRel('B', DROPLinkType.INPUT, 'E')
        ]
        quickDeploy(dm1, sessionId, g1, {nm_conninfo(1): rels})
        quickDeploy(dm2, sessionId, g2, {nm_conninfo(0): rels})

        self.assertEqual(4, len(dm1._sessions[sessionId].drops))
        self.assertEqual(2, len(dm2._sessions[sessionId].drops))

        # Run! The sole fact that this doesn't throw exceptions is already
        # a good proof that everything is working as expected
        a, b, c, d = [
            dm1._sessions[sessionId].drops[x] for x in ('A', 'B', 'C', 'D')
        ]
        e, f = [dm2._sessions[sessionId].drops[x] for x in ('E', 'F')]
        with droputils.DROPWaiterCtx(self, f, 5):
            a.write('a')
            a.setCompleted()
            b.write('a')
            b.setCompleted()

        for drop in a, b, c, d, e, f:
            self.assertEqual(DROPStates.COMPLETED, drop.status,
                             "DROP %s is not COMPLETED" % (drop.uid))

        self.assertEqual(a.checksum, int(droputils.allDropContents(d)))
        self.assertEqual(b.checksum + d.checksum,
                         int(droputils.allDropContents(f)))

        dm1.destroySession(sessionId)
        dm2.destroySession(sessionId)
예제 #47
0
    def test_runGraphSeveralDropsPerDM(self):
        """
        A test that creates several DROPs in two different DMs and  runs
        the graph. The graph looks like this

        DM #1                  DM #2
        ===================    ================
        | A --> C --> D --|----|-|            |
        |                 |    | |--> E --> F |
        | B --------------|----|-|            |
        ===================    ================

        :see: `self.test_runGraphSingleDOPerDOM`
        """
        dm1 = NodeManager(useDLM=False)
        dm2 = NodeManager(useDLM=False)

        sessionId = 's1'
        g1 = [{"oid":"A", "type":"plain", "storage": "memory", "consumers":["C"]},
               {"oid":"B", "type":"plain", "storage": "memory"},
               {"oid":"C", "type":"app", "app":"dfms.apps.crc.CRCApp"},
               {"oid":"D", "type":"plain", "storage": "memory", "producers": ["C"]}]
        g2 = [{"oid":"E", "type":"app", "app":"test.test_drop.SumupContainerChecksum"},
               {"oid":"F", "type":"plain", "storage": "memory", "producers":["E"]}]

        uris1 = dm1.quickDeploy(sessionId, g1)
        uris2 = dm2.quickDeploy(sessionId, g2)
        self.assertEquals(4, len(uris1))
        self.assertEquals(2, len(uris2))

        # We externally wire the Proxy objects to establish the inter-DM
        # relationships
        a = Pyro4.Proxy(uris1['A'])
        b = Pyro4.Proxy(uris1['B'])
        c = Pyro4.Proxy(uris1['C'])
        d = Pyro4.Proxy(uris1['D'])
        e = Pyro4.Proxy(uris2['E'])
        f = Pyro4.Proxy(uris2['F'])
        for drop,uid in [(a,'A'),(b,'B'),(c,'C'),(d,'D'),(e,'E'),(f,'F')]:
            self.assertEquals(uid, drop.uid, "Proxy is not the DROP we think should be (assumed: %s/ actual: %s)" % (uid, drop.uid))
        e.addInput(d)
        e.addInput(b)

        # Run! The sole fact that this doesn't throw exceptions is already
        # a good proof that everything is working as expected
        with droputils.EvtConsumerProxyCtx(self, f, 5):
            a.write('a')
            a.setCompleted()
            b.write('a')
            b.setCompleted()

        for drop in a,b,c,d,e,f:
            self.assertEquals(DROPStates.COMPLETED, drop.status, "DROP %s is not COMPLETED" % (drop.uid))

        self.assertEquals(a.checksum, int(droputils.allDropContents(d)))
        self.assertEquals(b.checksum + d.checksum, int(droputils.allDropContents(f)))

        for dropProxy in a,b,c,d,e,f:
            dropProxy._pyroRelease()

        dm1.destroySession(sessionId)
        dm2.destroySession(sessionId)
예제 #48
0
    def branch_failure(self, tooManyFailures):
        """
        Using the container data object to implement a join/barrier dataflow.

        A1, A2 and A3 are FileDROPs
        B1, B2 and B3 are SumupContainerChecksum
        C1, C2 and C3 are InMemoryDROPs
        D is a SumupContainerChecksum
        E is a InMemoryDROP

        --> A1 --> B1 --> C1 --|
        --> A2 --> B2 --> C2 --|--> D --> E
        --> A3 --> B3 --> C3 --|

        Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
        after which "C" will transition to COMPLETE. Once all "C"s have moved to
        COMPLETED "D"'s execution will also be triggered, and finally E will
        hold the sum of B1, B2 and B3's checksums
        """

        # create file data objects
        a1 = InMemoryDROP("oid:A1", "uid:A1")
        a2 = InMemoryDROP("oid:A2", "uid:A2")
        a3 = InMemoryDROP("oid:A3", "uid:A3")

        # CRC Result DROPs, storing the result in memory
        b1 = SumupContainerChecksum("oid:B1", "uid:B1")
        b2 = SumupContainerChecksum("oid:B2", "uid:B2")
        b3 = SumupContainerChecksum("oid:B3", "uid:B3")
        c1 = InMemoryDROP("oid:C1", "uid:C1")
        c2 = InMemoryDROP("oid:C2", "uid:C2")
        c3 = InMemoryDROP("oid:C3", "uid:C3")

        # The final DROP that sums up the CRCs from the container DROP
        d = SumupContainerChecksum("oid:D", "uid:D", input_error_threshold=33)
        e = InMemoryDROP("oid:E", "uid:E")

        # Wire together
        dropAList = [a1, a2, a3]
        dropBList = [b1, b2, b3]
        dropCList = [c1, c2, c3]
        for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList):
            dropA.addConsumer(dropB)
        for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList):
            dropB.addOutput(dropC)
        for dropC in dropCList:
            dropC.addConsumer(d)
        d.addOutput(e)

        # Write data into the initial "A" DROPs, which should trigger
        # the whole chain explained above
        with DROPWaiterCtx(self, e):
            # for dropA in dropAList: # this should be parallel for
            a1.write(" ")
            a1.setCompleted()
            if tooManyFailures:
                a2.setError()
            else:
                a2.write(" ")
                a2.setCompleted()
            a3.setError()

        if tooManyFailures:
            completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
            errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
        else:
            completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
            errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]

        for drop in completedDrops:
            self.assertEquals(drop.status, DROPStates.COMPLETED)
        for drop in errorDrops:
            self.assertEquals(drop.status, DROPStates.ERROR)

        # The results we want to compare
        # (only in case that at least two branches executed)
        if not tooManyFailures:
            sum_crc = c1.checksum + c2.checksum
            dropEData = int(droputils.allDropContents(e))

            self.assertNotEquals(sum_crc, 0)
            self.assertEquals(sum_crc, dropEData)