Ejemplo n.º 1
0
    def test_eager_inputFired_app(self):
        """
        Tests that InputFiredApps works as expected
        """

        # No n_effective_inputs given
        self.assertRaises(ValueError, InputFiredAppDROP, "a", "a")
        # Invalid values
        self.assertRaises(ValueError, InputFiredAppDROP, "a", "a", n_effective_inputs=-2)
        self.assertRaises(ValueError, InputFiredAppDROP, "a", "a", n_effective_inputs=0)

        # More effective inputs than inputs
        a = InMemoryDROP("b", "b")
        b = InputFiredAppDROP("a", "a", n_effective_inputs=2)
        b.addInput(a)
        self.assertRaises(Exception, a.setCompleted)

        # 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the
        # app has run
        a, b, c, d = [InMemoryDROP(str(i), str(i)) for i in xrange(4)]
        e = InputFiredAppDROP("e", "e", n_effective_inputs=2)
        map(lambda x: e.addInput(x), [a, b, c, d])

        with DROPWaiterCtx(self, e, 5):
            a.setCompleted()
            b.setCompleted()

        self.assertEquals(AppDROPStates.FINISHED, e.execStatus)
        self.assertEquals(DROPStates.COMPLETED, a.status)
        self.assertEquals(DROPStates.COMPLETED, b.status)
        self.assertEquals(DROPStates.INITIALIZED, c.status)
        self.assertEquals(DROPStates.INITIALIZED, d.status)
Ejemplo n.º 2
0
    def test_eager_inputFired_app(self):
        """
        Tests that InputFiredApps works as expected
        """

        # No n_effective_inputs given
        self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a')
        # Invalid values
        self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=-2)
        self.assertRaises(InvalidDropException, InputFiredAppDROP, 'a', 'a', n_effective_inputs=0)

        # More effective inputs than inputs
        a = InMemoryDROP('b', 'b')
        b = InputFiredAppDROP('a', 'a', n_effective_inputs=2)
        b.addInput(a)
        self.assertRaises(Exception, a.setCompleted)

        # 2 effective inputs, 4 outputs. Trigger 2 inputs and make sure the
        # app has run
        a,b,c,d = [InMemoryDROP(str(i), str(i)) for i in range(4)]
        e = InputFiredAppDROP('e', 'e', n_effective_inputs=2)
        for x in a,b,c,d:
            e.addInput(x)

        with DROPWaiterCtx(self, e, 5):
            a.setCompleted()
            b.setCompleted()

        self.assertEqual(AppDROPStates.FINISHED, e.execStatus)
        self.assertEqual(DROPStates.COMPLETED, a.status)
        self.assertEqual(DROPStates.COMPLETED, b.status)
        self.assertEqual(DROPStates.INITIALIZED, c.status)
        self.assertEqual(DROPStates.INITIALIZED, d.status)
Ejemplo n.º 3
0
    def test_app_multiple_outputs(self):
        """
        A small method that tests that the AppDROPs writing to two
        different DROPs outputs works

        The graph constructed by this example looks as follow:

                              |--> E
        A --> B --> C --> D --|
                              |--> F

        Here B and D are an AppDROPs, with D writing to two DROPs
        outputs (E and F) and reading from C. C, in turn, is written by B, which
        in turns reads the data from A
        """

        # This is used as "B"
        class NumberWriterApp(BarrierAppDROP):
            def run(self):
                inputDrop = self.inputs[0]
                output = self.outputs[0]
                howMany = int(droputils.allDropContents(inputDrop))
                for i in xrange(howMany):
                    output.write(str(i) + " ")

        # This is used as "D"
        class OddAndEvenContainerApp(BarrierAppDROP):
            def run(self):
                inputDrop = self.inputs[0]
                outputs = self.outputs

                numbers = droputils.allDropContents(inputDrop).strip().split()
                for n in numbers:
                    outputs[int(n) % 2].write(n + " ")

        # Create DROPs
        a = InMemoryDROP("oid:A", "uid:A")
        b = NumberWriterApp("oid:B", "uid:B")
        c = InMemoryDROP("oid:A", "uid:A")
        d = OddAndEvenContainerApp("oid:D", "uid:D")
        e = InMemoryDROP("oid:E", "uid:E")
        f = InMemoryDROP("oid:F", "uid:F")

        # Wire them together
        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
        d.addOutput(e)
        d.addOutput(f)

        # Start the execution
        with DROPWaiterCtx(self, [e, f]):
            a.write("20")
            a.setCompleted()

        # Check the final results are correct
        for drop in [a, b, c, d, e]:
            self.assertEquals(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
        self.assertEquals("0 2 4 6 8 10 12 14 16 18", droputils.allDropContents(e).strip())
        self.assertEquals("1 3 5 7 9 11 13 15 17 19", droputils.allDropContents(f).strip())
Ejemplo n.º 4
0
    def _test_graphExecutionDriver(self, mode):
        """
        A small test to check that DROPs executions can be driven externally if
        required, and not always internally by themselves
        """
        a = InMemoryDROP('a', 'a', executionMode=mode, expectedSize=1)
        b = SumupContainerChecksum('b', 'b')
        c = InMemoryDROP('c', 'c')
        a.addConsumer(b)
        c.addProducer(b)

        # Write and check
        dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
        with DROPWaiterCtx(self, dropsToWaitFor):
            a.write('1')

        if mode == ExecutionMode.EXTERNAL:
            # b hasn't been triggered
            self.assertEqual(c.status, DROPStates.INITIALIZED)
            self.assertEqual(b.status, DROPStates.INITIALIZED)
            self.assertEqual(b.execStatus, AppDROPStates.NOT_RUN)
            # Now let b consume a
            with DROPWaiterCtx(self, [c]):
                b.dropCompleted('a', DROPStates.COMPLETED)
            self.assertEqual(c.status, DROPStates.COMPLETED)
        elif mode == ExecutionMode.DROP:
            # b is already done
            self.assertEqual(c.status, DROPStates.COMPLETED)
Ejemplo n.º 5
0
    def test_addDropInstances(self):

        a1 = InMemoryDROP('a', 'a1')
        a2 = InMemoryDROP('a', 'a2')
        registry = RDBMSRegistry('sqlite3', DBFILE)
        registry.addDrop(a1)

        uids = registry.getDropUids(a1)
        self.assertEqual(1, len(uids))
        self.assertEqual('a1', uids[0])

        registry.addDropInstance(a2)
        uids = registry.getDropUids(a1)
        uids.sort()
        self.assertEqual(2, len(uids))
        self.assertEqual('a1', uids[0])
        self.assertEqual('a2', uids[1])

        # Check accessing the database separately
        conn = sqlite3.connect(DBFILE)  # @UndefinedVariable
        cur = conn.cursor()
        cur.execute("SELECT uid FROM dfms_dropinstance WHERE oid = 'a'")
        rows = cur.fetchall()
        self.assertEqual(2, len(rows))
        uids = [r[0] for r in rows]
        uids.sort()
        self.assertEqual(2, len(uids))
        self.assertEqual('a1', uids[0])
        self.assertEqual('a2', uids[1])
        cur.close()
        conn.close()
Ejemplo n.º 6
0
def container_pg():
    '''
    Creates the following graph:

                |--> B --> D --|
     SL --> A --|              |--> F --> G --> H --> I
                |--> C --> E --|
    '''
    sl= SocketListenerApp('SL', 'SL', lifespan=lifespan)
    a = InMemoryDROP('A', 'A', lifespan=lifespan)
    b = SleepAndCopyApp('B', 'B', lifespan=lifespan)
    c = SleepAndCopyApp('C', 'C', lifespan=lifespan)
    d = InMemoryDROP('D', 'D', lifespan=lifespan)
    e = InMemoryDROP('E', 'E', lifespan=lifespan)
    f = SimpleBarrierApp('F', 'F', lifespan=lifespan)
    g = ContainerDROP('G', 'G', lifespan=lifespan)
    h = SleepAndCopyApp('H', 'H', lifespan=lifespan)
    i = InMemoryDROP('I', 'I', lifespan=lifespan)

    # Wire together
    sl.addOutput(a)
    a.addConsumer(b)
    a.addConsumer(c)
    b.addOutput(d)
    c.addOutput(e)
    d.addConsumer(f)
    e.addConsumer(f)
    f.addOutput(g)
    g.addChild(d)
    g.addChild(e)
    g.addConsumer(h)
    h.addOutput(i)

    return sl
Ejemplo n.º 7
0
    def test_invalid(self):

        # Shouldn't allow inputs
        a = SocketListenerApp('a', 'a', port=1)
        a.addOutput(InMemoryDROP('c', 'c'))
        self.assertRaises(Exception, a.addInput, InMemoryDROP('b', 'b'))
        self.assertRaises(Exception, a.addStreamingInput, InMemoryDROP('b', 'b'))

        # Shouldn't be able to open ports <= 1024
        a.execute()
        self.assertEqual(a.status, DROPStates.ERROR)
Ejemplo n.º 8
0
    def _test_graphExecutionDriver(self, mode):
        """
        A small test to check that DROPs executions can be driven externally if
        required, and not always internally by themselves
        """
        a = InMemoryDROP("a", "a", executionMode=mode, expectedSize=1)
        b = SumupContainerChecksum("b", "b")
        c = InMemoryDROP("c", "c")
        a.addConsumer(b)
        c.addProducer(b)

        # Write and check
        dropsToWaitFor = [] if mode == ExecutionMode.EXTERNAL else [c]
        with DROPWaiterCtx(self, dropsToWaitFor):
            a.write("1")

        if mode == ExecutionMode.EXTERNAL:
            # b hasn't been triggered
            self.assertEquals(c.status, DROPStates.INITIALIZED)
            self.assertEquals(b.status, DROPStates.INITIALIZED)
            self.assertEquals(b.execStatus, AppDROPStates.NOT_RUN)
            # Now let b consume a
            with DROPWaiterCtx(self, [c]):
                b.dropCompleted("a", DROPStates.COMPLETED)
            self.assertEquals(c.status, DROPStates.COMPLETED)
        elif mode == ExecutionMode.DROP:
            # b is already done
            self.assertEquals(c.status, DROPStates.COMPLETED)
Ejemplo n.º 9
0
def test_pg():
    """
    A very simple graph that looks like this:

    A --> B --> C --> D
    """
    a =  SocketListenerApp('A', 'A', lifespan=lifespan)
    b = InMemoryDROP('B', 'B', lifespan=lifespan)
    c =    SleepAndCopyApp('C', 'C', lifespan=lifespan)
    d = InMemoryDROP('D', 'D', lifespan=lifespan)

    a.addOutput(b)
    b.addConsumer(c)
    c.addOutput(d)

    return a
Ejemplo n.º 10
0
    def test_speadApp(self):

        port = 1111
        itemId = 0x2000

        thread_pool = spead2.ThreadPool()
        self._stream = spead2.send.UdpStream(
            thread_pool, "localhost", port, spead2.send.StreamConfig(rate=1e7))

        a = SpeadReceiverApp('a', 'a', port=port, itemId=itemId)
        b = InMemoryDROP('b', 'b')
        a.addOutput(b)

        size = 1024
        threading.Thread(target=lambda: a.execute()).start()
        time.sleep(1)
        msg = os.urandom(size)
        with DROPWaiterCtx(self, b, timeout=1):
            ig = spead2.send.ItemGroup(flavour=spead2.Flavour(4, 64, 48))
            item = ig.add_item(itemId,
                               'main_data',
                               'a char array',
                               shape=(size, ),
                               format=[('c', 8)])
            item.value = msg
            self._stream.send_heap(ig.get_heap())
            self._stream.send_heap(ig.get_end())

        for drop in a, b:
            self.assertEqual(DROPStates.COMPLETED, drop.status)

        self.assertEqual(size, b.size)
        self.assertEqual(msg, droputils.allDropContents(b))
Ejemplo n.º 11
0
    def test_multipleProducers(self):
        """
        A test that checks that multiple-producers correctly drive the state of
        their shared output
        """
        class App(BarrierAppDROP): pass

        a,b,c,d,e = [App(chr(ord('A') + i), chr(ord('A') + i)) for i in range(5)]
        f = InMemoryDROP('F', 'F')
        for drop in a,b,c,d,e:
            drop.addOutput(f)

        self.assertEqual(DROPStates.INITIALIZED, f.status)
        for drop in a,b,c,d,e:
            self.assertEqual(AppDROPStates.NOT_RUN, drop.execStatus)

        # Run the first 4 ones, F should still be in INITIALIZED
        for drop in a,b,c,d:
            drop.execute()
        self.assertEqual(DROPStates.INITIALIZED, f.status)
        self.assertEqual(AppDROPStates.NOT_RUN, e.execStatus)
        for drop in a,b,c,d:
            self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)

        # Run the final one, now F should be COMPLETED
        e.execute()
        self.assertEqual(DROPStates.COMPLETED, f.status)
        for drop in a,b,c,d,e:
            self.assertEqual(AppDROPStates.FINISHED, drop.execStatus)
Ejemplo n.º 12
0
 def test_errorState(self):
     a = InMemoryDROP('a', 'a')
     b = SumupContainerChecksum('b', 'b')
     c = InMemoryDROP('c', 'c')
     c.addProducer(b)
     b.addInput(a)
     a.setError()
     self.assertEqual(DROPStates.ERROR, a.status)
     self.assertEqual(DROPStates.ERROR, b.status)
     self.assertEqual(DROPStates.ERROR, c.status)
Ejemplo n.º 13
0
    def test_dropAccess(self):

        a1 = InMemoryDROP('a', 'a1')
        registry = RDBMSRegistry('sqlite3', DBFILE)
        registry.addDrop(a1)

        self.assertEqual(-1, registry.getLastAccess('a'))
        registry.recordNewAccess('a')

        self.assertNotEqual(-1, registry.getLastAccess('a'))
Ejemplo n.º 14
0
    def test_two_simultaneous_pipes(self):
        """
        A more complicated test where three bash applications run at the same
        time. The first streams its output to the second one, while the second
        one streams *its* output to the third one.

        -------------     --------------     -------------     --------------     -------------     ----------
        | BashApp A | --> | InMemory B | --> | BashApp C | --> | InMemory D | --> | BashApp E | --> | File F |
        |   echo    |     | "/pipe1"   |     |    dc     |     | "/pipe2"   |     |   sort    |     |        |
        -----*-------     --------------     ----*--*-----     --------------     -----*-------     ----------
             |                                   |  |                                  |
             \-------------|named-pipe|----------\  \-----------|named-pipe|-----------/

        BashApp A writes "5 4 3 2 1" (each on a new line), which is read
        by "cat" (BashApp C). The printed results (a copy of the original) are
        streamed through D and read by "sort" (BashApp E), which writes the
        output to F.
        """

        output_fname = tempfile.mktemp()

        a = StreamingOutputBashApp('a',
                                   'a',
                                   command=r"echo -en '5\n4\n3\n2\n1'")
        b = InMemoryDROP('b', 'b')
        c = StreamingInputOutputBashApp('c', 'c', command="cat")
        d = InMemoryDROP('d', 'd')
        e = StreamingInputBashApp('e', 'e', command="sort -n > %o0")
        f = FileDROP('f', 'f', filepath=output_fname)

        a.addOutput(b)
        b.addStreamingConsumer(c)
        c.addOutput(d)
        d.addStreamingConsumer(e)
        e.addOutput(f)

        # Let's fire the app
        with DROPWaiterCtx(self, f, 2):
            a.async_execute()

        # The application executed, finished, and its output was recorded
        for drop in (a, b, c, d, e, f):
            self.assertEqual(DROPStates.COMPLETED, drop.status)
        self.assertEqual([1, 2, 3, 4, 5], [
            int(x)
            for x in droputils.allDropContents(f).strip().split(six.b('\n'))
        ])

        # Clean up and go
        os.remove(output_fname)
Ejemplo n.º 15
0
    def test_addDrop(self):
        a = InMemoryDROP('a', 'a')
        registry = RDBMSRegistry('sqlite3', DBFILE)
        registry.addDrop(a)

        conn = sqlite3.connect(DBFILE)  # @UndefinedVariable
        cur = conn.cursor()
        cur.execute('SELECT oid FROM dfms_drop')
        r = cur.fetchone()
        self.assertEqual(1, len(r))
        self.assertEqual('a', r[0])
        cur.close()
        conn.close()
Ejemplo n.º 16
0
    def test_socket_listener(self):
        '''
        A simple test to check that SocketListenerApps are indeed working as
        expected; that is, they write the data they receive into their output,
        and finish when the connection is closed from the client side

        The data flow diagram looks like this:

        A --> B --> C --> D
        '''

        host = 'localhost'
        port = 9933
        data = 'shine on you crazy diamond'

        a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
        b = InMemoryDROP('oid:B', 'uid:B')
        c = SumupContainerChecksum('oid:C', 'uid:C')
        d = InMemoryDROP('oid:D', 'uid:D')
        a.addOutput(b)
        b.addConsumer(c)
        c.addOutput(d)

        # Create the socket, write, and close the connection, allowing
        # A to move to COMPLETED
        with DROPWaiterCtx(self, d, 3): # That's plenty of time
            threading.Thread(target=lambda a: a.execute(), args=(a,)).start()
            utils.writeToRemotePort(host, port, data, 1)

        for drop in [a,b,c,d]:
            self.assertEquals(DROPStates.COMPLETED, drop.status)

        # Our expectations are fulfilled!
        bContents = droputils.allDropContents(b)
        dContents = int(droputils.allDropContents(d))
        self.assertEquals(data, bContents)
        self.assertEquals(crc32(data, 0), dContents)
Ejemplo n.º 17
0
    def test_socket_listener(self):
        '''
        A simple test to check that SocketListenerApps are indeed working as
        expected; that is, they write the data they receive into their output,
        and finish when the connection is closed from the client side

        The data flow diagram looks like this:

        A --> B --> C --> D
        '''

        host = 'localhost'
        port = 9933
        data = os.urandom(1025)

        a = SocketListenerApp('oid:A', 'uid:A', host=host, port=port)
        b = InMemoryDROP('oid:B', 'uid:B')
        c = SumupContainerChecksum('oid:C', 'uid:C')
        d = InMemoryDROP('oid:D', 'uid:D')
        a.addOutput(b)
        b.addConsumer(c)
        c.addOutput(d)

        # Create the socket, write, and close the connection, allowing
        # A to move to COMPLETED
        with DROPWaiterCtx(self, d, 3): # That's plenty of time
            a.async_execute()
            utils.write_to(host, port, data, 1)

        for drop in [a,b,c,d]:
            self.assertEqual(DROPStates.COMPLETED, drop.status)

        # Our expectations are fulfilled!
        bContents = droputils.allDropContents(b)
        dContents = int(droputils.allDropContents(d))
        self.assertEqual(data, bContents)
        self.assertEqual(crc32(data, 0), dContents)
Ejemplo n.º 18
0
 def test_errorState(self):
     a = InMemoryDROP("a", "a")
     b = SumupContainerChecksum("b", "b")
     c = InMemoryDROP("c", "c")
     c.addProducer(b)
     b.addInput(a)
     a.setError()
     self.assertEquals(DROPStates.ERROR, a.status)
     self.assertEquals(DROPStates.ERROR, b.status)
     self.assertEquals(DROPStates.ERROR, c.status)
Ejemplo n.º 19
0
def _testGraph(execMode):
    """
    A test graph that looks like this:

                 |--> B1 --> C1 --|
                 |--> B2 --> C2 --|
    SL_A --> A --|--> B3 --> C3 --| --> D --> E
                 |--> .. --> .. --|
                 |--> BN --> CN --|

    B and C DROPs are InMemorySleepAndCopyApp DROPs (see above). D is simply a
    container. A is a socket listener, so we can actually write to it externally
    and watch the progress of the luigi tasks. We give DROPs a long lifespan;
    otherwise they will expire and luigi will see it as a failed task (which is
    actually right!)

    If execMode is given we use that in all DROPs. If it's None we use a mixture
    of DROP/EXTERNAL execution modes.
    """

    aMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
    bMode = execMode if execMode is not None else ExecutionMode.DROP
    cMode = execMode if execMode is not None else ExecutionMode.DROP
    dMode = execMode if execMode is not None else ExecutionMode.EXTERNAL
    eMode = execMode if execMode is not None else ExecutionMode.EXTERNAL

    sl_a =      SocketListenerApp('oid:SL_A', 'uid:SL_A', executionMode=aMode, lifespan=lifespan)
    a    =     InMemoryDROP('oid:A', 'uid:A', executionMode=aMode, lifespan=lifespan)
    d    = SumupContainerChecksum('oid:D', 'uid:D', executionMode=dMode, lifespan=lifespan)
    e    =     InMemoryDROP('oid:E', 'uid:E', executionMode=eMode, lifespan=lifespan)

    sl_a.addOutput(a)
    e.addProducer(d)
    for i in xrange(random.SystemRandom().randint(10, 20)):
        b =    SleepAndCopyApp('oid:B%d' % (i), 'uid:B%d' % (i), executionMode=bMode, lifespan=lifespan)
        c = InMemoryDROP('oid:C%d' % (i), 'uid:C%d' % (i), executionMode=cMode, lifespan=lifespan)
        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
    return sl_a
Ejemplo n.º 20
0
    def test_single_pipe(self):
        """
        A simple test where two bash apps are connected to each other in a
        streaming fashion. The data flows through a pipe which is created by
        the framework. The data drop in between acts only as a intermediator
        to establish the underlying communication channel.

        -------------     --------------     -------------     ----------
        | BashApp A | --> | InMemory B | --> | BashApp C | --> | File D |
        |   echo    |     | "/a/pipe"  |     |    dc     |     |        |
        -----*-------     --------------     ------*------     ----------
             |                                     |
             \-------------|named-pipe|------------/

        BashApp A writes "5 4 3 2 1" (each on a new line), which is read by cat
        and redirected to D.
        """

        output_fname = tempfile.mktemp()

        a = StreamingOutputBashApp('a',
                                   'a',
                                   command=r"echo -en '5\n4\n3\n2\n1'")
        b = InMemoryDROP('b', 'b')
        c = StreamingInputBashApp('c', 'c', command="cat > %o0")
        d = FileDROP('d', 'd', filepath=output_fname)

        a.addOutput(b)
        c.addStreamingInput(b)
        c.addOutput(d)

        # Let's fire the app
        with DROPWaiterCtx(self, d, 2):
            a.async_execute()

        # The application executed, finished, and its output was recorded
        for drop in (a, b, c, d):
            self.assertEqual(DROPStates.COMPLETED, drop.status,
                             "Drop %r not COMPLETED: %d" % (drop, drop.status))
        self.assertEqual(
            [5, 4, 3, 2, 1],
            [int(x) for x in droputils.allDropContents(d).split(six.b('\n'))])

        # Clean up and go
        os.remove(output_fname)
Ejemplo n.º 21
0
    def test_directoryContainer(self):
        """
        A small, simple test for the DirectoryContainer DROP that checks it allows
        only valid children to be added
        """

        # Prepare our playground
        cwd = os.getcwd()
        os.chdir('/tmp')
        dirname  = ".hidden"
        dirname2 = ".hidden/inside"
        if not os.path.exists(dirname2):
            os.makedirs(dirname2)

        # DROPs involved
        a = FileDROP('a', 'a', dirname=dirname)
        b = FileDROP('b', 'b', dirname=dirname)
        c = FileDROP('c', 'c', dirname=dirname2)
        d = FileDROP('d', 'd', dirname=dirname2)
        cont1 = DirectoryContainer('e', 'e', dirname=dirname)
        cont2 = DirectoryContainer('f', 'f', dirname=dirname2)

        # Paths are absolutely reported
        self.assertEqual(os.path.realpath('/tmp/.hidden'), os.path.realpath(cont1.path))
        self.assertEqual(os.path.realpath('/tmp/.hidden/inside'), os.path.realpath(cont2.path))

        # Certain children-to-be are rejected
        self.assertRaises(TypeError, cont1.addChild, NullDROP('g', 'g'))
        self.assertRaises(TypeError, cont1.addChild, InMemoryDROP('h', 'h'))
        self.assertRaises(TypeError, cont1.addChild, ContainerDROP('i', 'i'))
        self.assertRaises(Exception, cont1.addChild, c)
        self.assertRaises(Exception, cont1.addChild, d)
        self.assertRaises(Exception, cont2.addChild, a)
        self.assertRaises(Exception, cont2.addChild, b)

        # These children are correct
        cont1.addChild(a)
        cont1.addChild(b)
        cont2.addChild(c)
        cont2.addChild(d)

        # Revert to previous state
        shutil.rmtree(dirname, True)
        os.chdir(cwd)
Ejemplo n.º 22
0
    def _test_write_withDropType(self, dropType):
        """
        Test an AbstractDROP and a simple AppDROP (for checksum calculation)
        """
        a = dropType('oid:A', 'uid:A', expectedSize = self._test_drop_sz * ONE_MB)
        b = SumupContainerChecksum('oid:B', 'uid:B')
        c = InMemoryDROP('oid:C', 'uid:C')
        b.addInput(a)
        b.addOutput(c)

        test_crc = 0
        with DROPWaiterCtx(self, c):
            for _ in range(self._test_num_blocks):
                a.write(self._test_block)
                test_crc = crc32(self._test_block, test_crc)

        # Read the checksum from c
        cChecksum = int(droputils.allDropContents(c))

        self.assertNotEqual(a.checksum, 0)
        self.assertEqual(a.checksum, test_crc)
        self.assertEqual(cChecksum, test_crc)
Ejemplo n.º 23
0
    def test_sql_create(self):
        sqlite01 = get_oid('sqlite')
        sqlite_drop = FileDROP(
            sqlite01,
            get_uuid(),
            precious=False,
            dirname=os.path.join(TestSqlite._temp, sqlite01),
            check_exists=False,
        )
        initialize_sqlite = InitializeSqliteApp(
            get_oid('app'),
            get_uuid(),
            user='******',
        )
        sqlite_in_memory = InMemoryDROP(
            get_oid('memory'),
            get_uuid(),
            precious=False,
        )
        initialize_sqlite.addInput(sqlite_drop)
        initialize_sqlite.addOutput(sqlite_in_memory)

        with droputils.DROPWaiterCtx(self, sqlite_in_memory, 50000):
            sqlite_drop.setCompleted()
Ejemplo n.º 24
0
    def test_app_multiple_outputs(self):
        """
        A small method that tests that the AppDROPs writing to two
        different DROPs outputs works

        The graph constructed by this example looks as follow:

                              |--> E
        A --> B --> C --> D --|
                              |--> F

        Here B and D are an AppDROPs, with D writing to two DROPs
        outputs (E and F) and reading from C. C, in turn, is written by B, which
        in turns reads the data from A
        """

        # This is used as "B"
        class NumberWriterApp(BarrierAppDROP):
            def run(self):
                inputDrop = self.inputs[0]
                output = self.outputs[0]
                howMany = int(droputils.allDropContents(inputDrop))
                for i in range(howMany):
                    output.write(str(i) + " ")

        # This is used as "D"
        class OddAndEvenContainerApp(BarrierAppDROP):
            def run(self):
                inputDrop = self.inputs[0]
                outputs = self.outputs

                numbers = droputils.allDropContents(inputDrop).strip().split()
                for n in numbers:
                    outputs[int(n) % 2].write(n + six.b(" "))

        # Create DROPs
        a =     InMemoryDROP('oid:A', 'uid:A')
        b =        NumberWriterApp('oid:B', 'uid:B')
        c =     InMemoryDROP('oid:A', 'uid:A')
        d = OddAndEvenContainerApp('oid:D', 'uid:D')
        e =     InMemoryDROP('oid:E', 'uid:E')
        f =     InMemoryDROP('oid:F', 'uid:F')

        # Wire them together
        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
        d.addOutput(e)
        d.addOutput(f)

        # Start the execution
        with DROPWaiterCtx(self, [e,f]):
            a.write('20')
            a.setCompleted()

        # Check the final results are correct
        for drop in [a,b,c,d,e]:
            self.assertEqual(drop.status, DROPStates.COMPLETED, "%r is not yet COMPLETED" % (drop))
        self.assertEqual(six.b("0 2 4 6 8 10 12 14 16 18"), droputils.allDropContents(e).strip())
        self.assertEqual(six.b("1 3 5 7 9 11 13 15 17 19"), droputils.allDropContents(f).strip())
Ejemplo n.º 25
0
    def test_objectAsNormalAndStreamingInput(self):
        """
        A test that checks that a DROP can act as normal and streaming input of
        different AppDROPs at the same time. We use the following graph:

        A --|--> B --> D
            |--> C --> E

        Here B uses A as a streaming input, while C uses it as a normal
        input
        """

        class LastCharWriterApp(AppDROP):
            def initialize(self, **kwargs):
                super(LastCharWriterApp, self).initialize(**kwargs)
                self._lastByte = None
            def dataWritten(self, uid, data):
                self.execStatus = AppDROPStates.RUNNING
                outputDrop = self.outputs[0]
                self._lastByte = six.indexbytes(data, -1)
                outputDrop.write(self._lastByte)
            def dropCompleted(self, uid, status):
                self.execStatus = AppDROPStates.FINISHED
                self._notifyAppIsFinished()

        a = InMemoryDROP('a', 'a')
        b = LastCharWriterApp('b', 'b')
        c = SumupContainerChecksum('c', 'c')
        d = InMemoryDROP('d', 'd')
        e = InMemoryDROP('e', 'e')
        a.addStreamingConsumer(b)
        a.addConsumer(c)
        b.addOutput(d)
        c.addOutput(e)

        # Consumer cannot be normal and streaming at the same time
        self.assertRaises(Exception, a.addConsumer, b)
        self.assertRaises(Exception, a.addStreamingConsumer, c)

        # Write a little, then check the consumers
        def checkDropStates(aStatus, dStatus, eStatus, lastByte):
            self.assertEqual(aStatus, a.status)
            self.assertEqual(dStatus, d.status)
            self.assertEqual(eStatus, e.status)
            if lastByte is not None:
                self.assertEqual(six.b(lastByte), six.int2byte(b._lastByte))

        checkDropStates(DROPStates.INITIALIZED , DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
        a.write('abcde')
        checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'e')
        a.write('fghij')
        checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, 'j')
        a.write('k')
        with DROPWaiterCtx(self, [d,e]):
            a.setCompleted()
        checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, 'k')

        self.assertEqual(six.b('ejk'), droputils.allDropContents(d))
Ejemplo n.º 26
0
    def test_stateMachine(self):
        """
        A simple test to check that some transitions are invalid
        """

        # Nice and easy
        drop = InMemoryDROP('a', 'a')
        self.assertEqual(drop.status, DROPStates.INITIALIZED)
        drop.write('a')
        self.assertEqual(drop.status, DROPStates.WRITING)
        drop.setCompleted()
        self.assertEqual(drop.status, DROPStates.COMPLETED)

        # Try to overwrite the DROP's checksum and size
        self.assertRaises(Exception, lambda: setattr(drop, 'checksum', 0))
        self.assertRaises(Exception, lambda: setattr(drop, 'size', 0))

        # Try to write on a DROP that is already COMPLETED
        self.assertRaises(Exception, drop.write, '')

        # Invalid reading on a DROP that isn't COMPLETED yet
        drop = InMemoryDROP('a', 'a')
        self.assertRaises(Exception, drop.open)
        self.assertRaises(Exception, drop.read, 1)
        self.assertRaises(Exception, drop.close, 1)

        # Invalid file descriptors used to read/close
        drop.setCompleted()
        fd = drop.open()
        otherFd = random.SystemRandom().randint(0, 1000)
        self.assertNotEqual(fd, otherFd)
        self.assertRaises(Exception, drop.read, otherFd)
        self.assertRaises(Exception, drop.close, otherFd)
        # but using the correct one should be OK
        drop.read(fd)
        self.assertTrue(drop.isBeingRead())
        drop.close(fd)

        # Expire it, then try to set it as COMPLETED again
        drop.status = DROPStates.EXPIRED
        self.assertRaises(Exception, drop.setCompleted)
Ejemplo n.º 27
0
    def test_simple_chain(self):
        '''
        Simple test that creates a pipeline-like chain of commands.
        In this case we simulate a pipeline that does this, holding
        each intermediate result in memory:

        cat someFile | grep 'a' | sort | rev
        '''

        class GrepResult(BarrierAppDROP):
            def initialize(self, **kwargs):
                super(GrepResult, self).initialize(**kwargs)
                self._substring = six.b(kwargs['substring'])

            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                allLines = BytesIO(droputils.allDropContents(drop)).readlines()
                for line in allLines:
                    if self._substring in line:
                        output.write(line)

        class SortResult(BarrierAppDROP):
            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                sortedLines = BytesIO(droputils.allDropContents(drop)).readlines()
                sortedLines.sort()
                for line in sortedLines:
                    output.write(line)

        class RevResult(BarrierAppDROP):
            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                allbytes = droputils.allDropContents(drop)
                buf = bytearray()
                for c in allbytes:
                    if c == six.b(' ') or c == six.b('\n'):
                        output.write(buf[::-1])
                        output.write(c)
                        buf = bytearray()
                    else:
                        buf.append(c)

        a = InMemoryDROP('oid:A', 'uid:A')
        b = GrepResult('oid:B', 'uid:B', substring="a")
        c = InMemoryDROP('oid:C', 'uid:C')
        d = SortResult('oid:D', 'uid:D')
        e = InMemoryDROP('oid:E', 'uid:E')
        f = RevResult('oid:F', 'oid:F')
        g = InMemoryDROP('oid:G', 'uid:G')

        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
        d.addOutput(e)
        e.addConsumer(f)
        f.addOutput(g)

        # Initial write
        contents = "first line\nwe have an a here\nand another one\nnoone knows me"
        cResExpected = "we have an a here\nand another one\n"
        eResExpected = "and another one\nwe have an a here\n"
        gResExpected = "dna rehtona eno\new evah na a ereh\n"

        with DROPWaiterCtx(self, g):
            a.write(contents)
            a.setCompleted()

        # Get intermediate and final results and compare
        actualRes   = []
        for i in [c, e, g]:
            actualRes.append(droputils.allDropContents(i))
        map(lambda x, y: self.assertEqual(x, y), [cResExpected, eResExpected, gResExpected], actualRes)
Ejemplo n.º 28
0
    def _createGraph(self):
        """
        Creates the following graph of DROPs:

        A |--> B ----> D --> G --> I --|
          |--> C -|--> E --------------|-> H --> J
                  |--> F

        B, C, G and H are AppDOs. The names have been given in breadth-first
        order (although H has a dependency on I)
        """
        a = InMemoryDROP('a', 'a')
        b = BarrierAppDROP('b', 'b')
        c = BarrierAppDROP('c', 'c')
        d = InMemoryDROP('d', 'd')
        e = InMemoryDROP('e', 'e')
        f = InMemoryDROP('f', 'f')
        g = BarrierAppDROP('g', 'g')
        h = BarrierAppDROP('h', 'h')
        i = InMemoryDROP('i', 'i')
        j = InMemoryDROP('j', 'j')

        a.addConsumer(b)
        a.addConsumer(c)
        b.addOutput(d)
        c.addOutput(e)
        c.addOutput(f)
        d.addConsumer(g)
        e.addConsumer(h)
        g.addOutput(i)
        i.addConsumer(h)
        h.addOutput(j)

        return a, b, c, d, e, f, g, h, i, j
Ejemplo n.º 29
0
def lofar_standard_pip_pg():
    """
    Lofar simple imaging pipeline
    based on
    Ger's email and
    https://github.com/lofar-astron/lofar-profiling
    """
    roots = []

    num_time_slice = 3
    total_num_subband = 8
    num_subb_per_image = 2

    Cal_model_oid = "A-team/Calibrator"
    dob_cal_model = InMemoryDROP(Cal_model_oid, Cal_model_oid, lifespan=lifespan)
    dob_cal_model.location = "catalogue.Groningen"
    roots.append(dob_cal_model)

    make_source_app_oid = "makesourcedb"
    make_source_app = SleepAndCopyApp(make_source_app_oid, make_source_app_oid)
    make_source_app.location = dob_cal_model.location
    dob_cal_model.addConsumer(make_source_app)

    source_db_oid = "CalSource.db"
    dob_source_db = InMemoryDROP(source_db_oid, source_db_oid, lifespan=lifespan)
    dob_source_db.location = dob_cal_model.location
    make_source_app.addOutput(dob_source_db)

    GSM_oid = "GlobalSkyModel"
    dob_gsm = InMemoryDROP(GSM_oid, GSM_oid, lifespan=lifespan)
    dob_gsm.location = "catalogue.Groningen"
    roots.append(dob_gsm)

    dob_img_dict = dict()
    sb_chunks = chunks(range(1, total_num_subband + 1), num_subb_per_image)
    for k, img_list in enumerate(sb_chunks):
        imger_oid = "AWImager_SB_{0}~{1}".format(img_list[0], img_list[-1])
        dob_img = SleepAndCopyApp(imger_oid, imger_oid, lifespan=lifespan)
        kk = k + 1
        #print k, img_list, kk
        img_prod_oid = "Image_SB_{0}~{1}".format(img_list[0], img_list[-1])
        dob_img_prod = InMemoryDROP(img_prod_oid, img_prod_oid, lifespan=lifespan)
        dob_img.addOutput(dob_img_prod)
        dob_img_dict[kk] = (dob_img, dob_img_prod)

    for j in range(1, num_time_slice + 1):
        sli = "T%02d" % j
        for i in range(1, total_num_subband + 1):
            # for LBA, assume each subband should have two beams
            # one for calibration pipeline, the other for target / imaging pipeline
            stri = "%s_SB%02d" % (sli, i)
            time_slice_oid = "Socket_{0}".format(stri)
            dob_time_slice_sl = SocketListenerApp(time_slice_oid + "_SL", time_slice_oid + "_SL", lifespan=lifespan)
            dob_time_slice = InMemoryDROP(time_slice_oid, time_slice_oid, lifespan=lifespan)
            dob_time_slice_sl.addOutput(dob_time_slice)
            # all the same sub-bands (i) will be on the same node regardless of its time slice j
            dob_time_slice.location = "Node%03d.Groningen" % i
            roots.append(dob_time_slice_sl)

            oid_data_writer = "DataWriter_{0}".format(stri)
            dob_ingest = SleepAndCopyApp(oid_data_writer, oid_data_writer, lifespan=lifespan)
            dob_ingest.location = dob_time_slice.location
            dob_time_slice.addConsumer(dob_ingest)

            # assume a single measuremenset has multiple beams
            oid = "CalNTgt.MS_{0}".format(stri)
            dob_input = InMemoryDROP(oid, oid, lifespan=lifespan)
            dob_ingest.addOutput(dob_input)
            dob_input.location = dob_ingest.location

            #split by beam - calibrator beam and target beam
            appOid = "Split_{0}".format(stri)
            splitApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
            splitApp.location = dob_ingest.location
            dob_input.addConsumer(splitApp)

            oid = "Cal_BEAM_{0}".format(stri)
            dob_cal_beam = InMemoryDROP(oid, oid, lifespan=lifespan)
            dob_cal_beam.location = dob_ingest.location
            splitApp.addOutput(dob_cal_beam)

            oid = "Tgt_BEAM_{0}".format(stri)
            dob_tgt_beam = InMemoryDROP(oid, oid, lifespan=lifespan)
            dob_tgt_beam.location = dob_ingest.location
            splitApp.addOutput(dob_tgt_beam)

            # flag the cal beam
            oid = "FlgCal.MS_{0}".format(stri)
            appOid = "NDPPP_Cal" + stri
            dob_cal_flagged = InMemoryDROP(oid, oid, lifespan=lifespan)
            flagCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
            dob_cal_beam.addConsumer(flagCalApp)
            dob_source_db.addConsumer(flagCalApp)
            flagCalApp.addOutput(dob_cal_flagged)
            flagCalApp.location = dob_ingest.location
            dob_cal_flagged.location = dob_ingest.location #"C%03d.Groningen" % i

            # flag the target beam
            oid = "FlgTgt.MS_{0}".format(stri)
            appOid = "NDPPP_Tgt" + stri
            dob_tgt_flagged = InMemoryDROP(oid, oid, lifespan=lifespan)
            flagTgtApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
            dob_tgt_beam.addConsumer(flagTgtApp)
            dob_source_db.addConsumer(flagTgtApp)
            flagTgtApp.addOutput(dob_tgt_flagged)
            flagTgtApp.location = dob_ingest.location
            dob_tgt_flagged.location = dob_ingest.location

            # For calibration, each time slice is calibrated independently, and
            # the result is an output MS per subband
            # solve the gain
            oid = "Gain.TBL_{0}".format(stri)
            appOid = "BBS_GainCal_{0}".format(stri)
            dob_param_tbl = InMemoryDROP(oid, oid, lifespan=lifespan)
            gainCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
            dob_cal_flagged.addConsumer(gainCalApp)
            gainCalApp.addOutput(dob_param_tbl)
            gainCalApp.location = dob_ingest.location
            dob_param_tbl.location = dob_ingest.location

            # or apply the gain to the dataset
            oid = "CAL.MS_{0}".format(stri)
            appOid = "BBS_ApplyCal_{0}".format(stri)
            dob_calibrated = InMemoryDROP(oid, oid, lifespan=lifespan)
            applyCalApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
            dob_tgt_flagged.addConsumer(applyCalApp)
            dob_param_tbl.addConsumer(applyCalApp)
            applyCalApp.addOutput(dob_calibrated)
            applyCalApp.location = dob_ingest.location
            dob_calibrated.location = dob_ingest.location

            # extract local sky model (LSM) from GSM
            appOid = "Extract_LSM_{0}".format(stri)
            extractLSMApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
            extractLSMApp.location = dob_ingest.location
            dob_gsm.addConsumer(extractLSMApp)
            LSM_oid = "LSM_{0}".format(stri)
            dob_lsm = InMemoryDROP(LSM_oid, LSM_oid, lifespan=lifespan)
            dob_lsm.location = dob_ingest.location
            extractLSMApp.addOutput(dob_lsm)

            # convert LSM to source.db
            makesource_app_oid = "makesourcedb_{0}".format(stri)
            makesource_app = SleepAndCopyApp(makesource_app_oid, makesource_app_oid)
            makesource_app.location = dob_ingest.location
            dob_lsm.addConsumer(makesource_app)

            sourcedb_oid = "LSM_db_{0}".format(stri)
            dob_lsmdb = InMemoryDROP(sourcedb_oid, sourcedb_oid, lifespan=lifespan)
            dob_lsmdb.location = dob_ingest.location
            makesource_app.addOutput(dob_lsmdb)

            # add awimager
            img_k = (i - 1) / num_subb_per_image + 1
            dob_img, dob_img_prod = dob_img_dict[img_k]
            dob_calibrated.addConsumer(dob_img)
            dob_lsmdb.addConsumer(dob_img)
            dob_img.location = dob_calibrated.location #overwrite to the last i
            dob_img_prod.location = dob_img.location

    return roots
Ejemplo n.º 30
0
    def branch_failure(self, tooManyFailures):
        """
        Using the container data object to implement a join/barrier dataflow.

        A1, A2 and A3 are FileDROPs
        B1, B2 and B3 are SumupContainerChecksum
        C1, C2 and C3 are InMemoryDROPs
        D is a SumupContainerChecksum
        E is a InMemoryDROP

        --> A1 --> B1 --> C1 --|
        --> A2 --> B2 --> C2 --|--> D --> E
        --> A3 --> B3 --> C3 --|

        Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
        after which "C" will transition to COMPLETE. Once all "C"s have moved to
        COMPLETED "D"'s execution will also be triggered, and finally E will
        hold the sum of B1, B2 and B3's checksums
        """

        #create file data objects
        a1 = InMemoryDROP('oid:A1', 'uid:A1')
        a2 = InMemoryDROP('oid:A2', 'uid:A2')
        a3 = InMemoryDROP('oid:A3', 'uid:A3')

        # CRC Result DROPs, storing the result in memory
        b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
        b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
        b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
        c1 = InMemoryDROP('oid:C1', 'uid:C1')
        c2 = InMemoryDROP('oid:C2', 'uid:C2')
        c3 = InMemoryDROP('oid:C3', 'uid:C3')

        # The final DROP that sums up the CRCs from the container DROP
        d = SumupContainerChecksum('oid:D', 'uid:D', input_error_threshold = 33)
        e = InMemoryDROP('oid:E', 'uid:E')

        # Wire together
        dropAList = [a1,a2,a3]
        dropBList = [b1,b2,b3]
        dropCList = [c1,c2,c3]
        for dropA,dropB in zip(dropAList, dropBList):
            dropA.addConsumer(dropB)
        for dropB,dropC in zip(dropBList, dropCList):
            dropB.addOutput(dropC)
        for dropC in dropCList:
            dropC.addConsumer(d)
        d.addOutput(e)

        # Write data into the initial "A" DROPs, which should trigger
        # the whole chain explained above
        with DROPWaiterCtx(self, e):
            #for dropA in dropAList: # this should be parallel for
            a1.write(' '); a1.setCompleted()
            if tooManyFailures:
                a2.setError()
            else:
                a2.write(' '); a2.setCompleted()
            a3.setError()

        if tooManyFailures:
            completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
            errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
        else:
            completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
            errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]

        for drop in completedDrops:
            self.assertEqual(drop.status, DROPStates.COMPLETED)
        for drop in errorDrops:
            self.assertEqual(drop.status, DROPStates.ERROR)

        # The results we want to compare
        # (only in case that at least two branches executed)
        if not tooManyFailures:
            sum_crc = c1.checksum + c2.checksum
            dropEData = int(droputils.allDropContents(e))

            self.assertNotEqual(sum_crc, 0)
            self.assertEqual(sum_crc, dropEData)
Ejemplo n.º 31
0
def pip_cont_img_pg():
    """
    PIP continuum imaging pipeline
    """
    num_beam = 1
    num_time = 2
    num_freq = 2
    num_facet = 2
    num_grid = 4
    stokes = ['I', 'Q', 'U', 'V']
    dob_root_sl = SocketListenerApp("FullDataset_SL", "FullDataset_SL", lifespan=lifespan)
    dob_root = InMemoryDROP("FullDataset", "FullDataset", lifespan=lifespan)
    dob_root_sl.addOutput(dob_root)
    dob_root.location = "Buf01"

    adob_sp_beam = SleepAndCopyApp("SPLT_BEAM", "SPLT_BEAM", lifespan=lifespan)
    adob_sp_beam.location = "Buf01"
    dob_root.addConsumer(adob_sp_beam)

    for i in range(1, num_beam + 1):
        uid = i
        dob_beam = InMemoryDROP("BEAM_{0}".format(uid), "BEAM_{0}".format(uid), lifespan=lifespan)
        adob_sp_beam.addOutput(dob_beam)
        adob_sp_time = SleepAndCopyApp("SPLT_TIME_{0}".format(uid), "SPLT_TIME_{0}".format(uid), lifespan=lifespan)
        dob_beam.addConsumer(adob_sp_time)
        for j in range(1, num_time + 1):
            uid = "%d-%d" % (i, j)
            dob_time = InMemoryDROP("TIME_{0}".format(uid), "TIME_{0}".format(uid), lifespan=lifespan)
            adob_sp_time.addOutput(dob_time)
            adob_sp_freq = SleepAndCopyApp("SPLT_FREQ_{0}".format(uid), "SPLT_FREQ_{0}".format(uid), lifespan=lifespan)
            dob_time.addConsumer(adob_sp_freq)
            for k in range(1, num_freq + 1):
                uid = "%d-%d-%d" % (i, j, k)
                dob_freq = InMemoryDROP("FREQ_{0}".format(uid), "FREQ_{0}".format(uid), lifespan=lifespan)
                adob_sp_freq.addOutput(dob_freq)
                adob_sp_facet = SleepAndCopyApp("SPLT_FACET_{0}".format(uid), "SPLT_FACET_{0}".format(uid), lifespan=lifespan)
                dob_freq.addConsumer(adob_sp_facet)
                for l in range(1, num_facet + 1):
                    uid = "%d-%d-%d-%d" % (i, j, k, l)
                    dob_facet = InMemoryDROP("FACET_{0}".format(uid), "FACET_{0}".format(uid), lifespan=lifespan)
                    adob_sp_facet.addOutput(dob_facet)

                    adob_ph_rot = SleepAndCopyApp("PH_ROTATN_{0}".format(uid), "PH_ROTATN_{0}".format(uid), lifespan=lifespan)
                    dob_facet.addConsumer(adob_ph_rot)
                    dob_ph_rot = InMemoryDROP("PH_ROTD_{0}".format(uid), "PH_ROTD_{0}".format(uid), lifespan=lifespan)
                    adob_ph_rot.addOutput(dob_ph_rot)
                    adob_sp_stokes = SleepAndCopyApp("SPLT_STOKES_{0}".format(uid), "SPLT_STOKES_{0}".format(uid), lifespan=lifespan)
                    dob_ph_rot.addConsumer(adob_sp_stokes)

                    adob_w_kernel = SleepAndCopyApp("CACL_W_Knl_{0}".format(uid), "CACL_W_Knl_{0}".format(uid), lifespan=lifespan)
                    dob_facet.addConsumer(adob_w_kernel)
                    dob_w_knl = InMemoryDROP("W_Knl_{0}".format(uid), "W_Knl_{0}".format(uid), lifespan=lifespan)
                    adob_w_kernel.addOutput(dob_w_knl)

                    adob_a_kernel = SleepAndCopyApp("CACL_A_Knl_{0}".format(uid), "CACL_A_Knl_{0}".format(uid), lifespan=lifespan)
                    dob_facet.addConsumer(adob_a_kernel)
                    dob_a_knl = InMemoryDROP("A_Knl_{0}".format(uid), "A_Knl_{0}".format(uid), lifespan=lifespan)
                    adob_a_kernel.addOutput(dob_a_knl)

                    adob_create_kernel = SleepAndCopyApp("CREATE_Knl_{0}".format(uid), "CREATE_Knl_{0}".format(uid), lifespan=lifespan)
                    dob_w_knl.addConsumer(adob_create_kernel)
                    dob_a_knl.addConsumer(adob_create_kernel)


                    for stoke in stokes:
                        uid = "%s-%d-%d-%d-%d" % (stoke, i, j, k, l)
                        #print "uid = {0}".format(uid)

                        dob_stoke = InMemoryDROP("STOKE_{0}".format(uid), "STOKE_{0}".format(uid), lifespan=lifespan)
                        adob_sp_stokes.addOutput(dob_stoke)

                        dob_stoke.addConsumer(adob_create_kernel)


                        dob_aw = InMemoryDROP("A_{0}".format(uid), "A_{0}".format(uid), lifespan=lifespan)
                        adob_create_kernel.addOutput(dob_aw)

                        # we do not do loop yet
                        griders = []
                        for m in range(1, num_grid + 1):
                            gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
                            adob_gridding = SleepAndCopyApp("Gridding_{0}".format(gid), "Gridding_{0}".format(gid), lifespan=lifespan)
                            dob_stoke.addConsumer(adob_gridding)
                            dob_gridded_cell = InMemoryDROP("Grided_Cell_{0}".format(gid), "Grided_Cell_{0}".format(gid), lifespan=lifespan)
                            adob_gridding.addOutput(dob_gridded_cell)
                            griders.append(dob_gridded_cell)
                        adob_gridded_bar = SleepAndCopyApp("GRIDDED_BARRIER_{0}".format(uid), "GRIDDED_BARRIER_{0}".format(uid), lifespan=lifespan)
                        for grider in griders:
                            grider.addConsumer(adob_gridded_bar)
                        dob_gridded_stoke = InMemoryDROP("GRIDDED_STOKE_{0}".format(uid), "GRIDDED_STOKE_{0}".format(uid), lifespan=lifespan)
                        adob_gridded_bar.addOutput(dob_gridded_stoke)

                        FFTers = []
                        for m in range(1, num_grid + 1):
                            gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
                            adob_fft = SleepAndCopyApp("FFT_{0}".format(gid), "FFT_{0}".format(gid), lifespan=lifespan)
                            dob_gridded_stoke.addConsumer(adob_fft)

                            dob_ffted_cell = InMemoryDROP("FFTed_Cell_{0}".format(gid), "FFTed_Cell_{0}".format(gid), lifespan=lifespan)
                            adob_fft.addOutput(dob_ffted_cell)
                            FFTers.append(dob_ffted_cell)
                        adob_ffted_bar = SleepAndCopyApp("FFTed_BARRIER_{0}".format(uid), "FFTed_BARRIER_{0}".format(uid), lifespan=lifespan)
                        for ffter in FFTers:
                            ffter.addConsumer(adob_ffted_bar)
                        dob_ffted_stoke = InMemoryDROP("FFTed_STOKE_{0}".format(uid), "FFTed_STOKE_{0}".format(uid), lifespan=lifespan)
                        adob_ffted_bar.addOutput(dob_ffted_stoke)

                        cleaners = []
                        for m in range(1, num_grid + 1):
                            gid = "%s-%d-%d-%d-%d-%d" % (stoke, i, j, k, l, m)
                            adob_cleaner = SleepAndCopyApp("DECONV_{0}".format(gid), "DECONV_{0}".format(gid), lifespan=lifespan)
                            dob_ffted_stoke.addConsumer(adob_cleaner)

                            dob_cleaned_cell = InMemoryDROP("CLEANed_Cell_{0}".format(gid), "CLEANed_Cell_{0}".format(gid), lifespan=lifespan)
                            adob_cleaner.addOutput(dob_cleaned_cell)
                            cleaners.append(dob_cleaned_cell)
                        adob_cleaned_bar = SleepAndCopyApp("CLEANed_BARRIER_{0}".format(uid), "CLEANed_BARRIER_{0}".format(uid), lifespan=lifespan)
                        for cleaner in cleaners:
                            cleaner.addConsumer(adob_cleaned_bar)
                        dob_decon_stoke = InMemoryDROP("CLEANed_STOKE_{0}".format(uid), "CLEANed_STOKE_{0}".format(uid), lifespan=lifespan)
                        adob_cleaned_bar.addOutput(dob_decon_stoke)

                        adob_create_prod = SleepAndCopyApp("CRT-PROD_{0}".format(uid), "CRT-PROD_{0}".format(uid), lifespan=lifespan)
                        dob_decon_stoke.addConsumer(adob_create_prod)

                        dob_prod = InMemoryDROP("PRODUCT_{0}".format(uid), "PRODUCT_{0}".format(uid), lifespan=lifespan)
                        adob_create_prod.addOutput(dob_prod)


    return dob_root_sl
Ejemplo n.º 32
0
    def test_objectAsNormalAndStreamingInput(self):
        """
        A test that checks that a DROP can act as normal and streaming input of
        different AppDROPs at the same time. We use the following graph:

        A --|--> B --> D
            |--> C --> E

        Here B uses A as a streaming input, while C uses it as a normal
        input
        """

        class LastCharWriterApp(AppDROP):
            def initialize(self, **kwargs):
                super(LastCharWriterApp, self).initialize(**kwargs)
                self._lastChar = None

            def dataWritten(self, uid, data):
                self.execStatus = AppDROPStates.RUNNING
                outputDrop = self.outputs[0]
                self._lastChar = data[-1]
                outputDrop.write(self._lastChar)

            def dropCompleted(self, uid, status):
                self.execStatus = AppDROPStates.FINISHED
                self._notifyAppIsFinished()

        a = InMemoryDROP("a", "a")
        b = LastCharWriterApp("b", "b")
        c = SumupContainerChecksum("c", "c")
        d = InMemoryDROP("d", "d")
        e = InMemoryDROP("e", "e")
        a.addStreamingConsumer(b)
        a.addConsumer(c)
        b.addOutput(d)
        c.addOutput(e)

        # Consumer cannot be normal and streaming at the same time
        self.assertRaises(Exception, lambda: a.addConsumer(b))
        self.assertRaises(Exception, lambda: a.addStreamingConsumer(c))

        # Write a little, then check the consumers
        def checkDropStates(aStatus, dStatus, eStatus, lastChar):
            self.assertEquals(aStatus, a.status)
            self.assertEquals(dStatus, d.status)
            self.assertEquals(eStatus, e.status)
            self.assertEquals(lastChar, b._lastChar)

        checkDropStates(DROPStates.INITIALIZED, DROPStates.INITIALIZED, DROPStates.INITIALIZED, None)
        a.write("abcde")
        checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "e")
        a.write("fghij")
        checkDropStates(DROPStates.WRITING, DROPStates.WRITING, DROPStates.INITIALIZED, "j")
        a.write("k")
        with DROPWaiterCtx(self, [d, e]):
            a.setCompleted()
        checkDropStates(DROPStates.COMPLETED, DROPStates.COMPLETED, DROPStates.COMPLETED, "k")

        self.assertEquals("ejk", droputils.allDropContents(d))
Ejemplo n.º 33
0
    def test_stateMachine(self):
        """
        A simple test to check that some transitions are invalid
        """

        # Nice and easy
        drop = InMemoryDROP("a", "a")
        self.assertEquals(drop.status, DROPStates.INITIALIZED)
        drop.write("a")
        self.assertEquals(drop.status, DROPStates.WRITING)
        drop.setCompleted()
        self.assertEquals(drop.status, DROPStates.COMPLETED)

        # Try to overwrite the DROP's checksum and size
        self.assertRaises(Exception, lambda: setattr(drop, "checksum", 0))
        self.assertRaises(Exception, lambda: setattr(drop, "size", 0))

        # Try to write on a DROP that is already COMPLETED
        self.assertRaises(Exception, drop.write, "")

        # Invalid reading on a DROP that isn't COMPLETED yet
        drop = InMemoryDROP("a", "a")
        self.assertRaises(Exception, drop.open)
        self.assertRaises(Exception, drop.read, 1)
        self.assertRaises(Exception, drop.close, 1)

        # Invalid file descriptors used to read/close
        drop.setCompleted()
        fd = drop.open()
        otherFd = random.SystemRandom().randint(0, 1000)
        self.assertNotEquals(fd, otherFd)
        self.assertRaises(Exception, drop.read, otherFd)
        self.assertRaises(Exception, drop.close, otherFd)
        # but using the correct one should be OK
        drop.read(fd)
        self.assertTrue(drop.isBeingRead())
        drop.close(fd)

        # Expire it, then try to set it as COMPLETED again
        drop.status = DROPStates.EXPIRED
        self.assertRaises(Exception, drop.setCompleted)
Ejemplo n.º 34
0
def chiles_pg():

    total_bandwidth = 480
    num_obs = 8 # the same as num of data island
    subband_width = 60 # MHz
    num_subb = total_bandwidth / subband_width
    subband_dict = collections.defaultdict(list) # for corner turning
    img_list = []
    start_freq = 940

    # this should be removed
    roots = []

    for i in range(1, num_obs + 1):
        stri = "%02d" % i
        oid    = "Obs_day_{0}".format(stri)
        appOid = "Receive_" + oid
        dob_obs_sl = SocketListenerApp(oid + "_SL", oid + "_SL", lifespan=lifespan, port=1110+i)
        dob_obs = InMemoryDROP(oid, oid, lifespan=lifespan)
        dob_obs_sl.addOutput(dob_obs)
        dob_obs.location = "{0}.aws-ec2.sydney".format(i)
        roots.append(dob_obs_sl)

        for j in range(1, num_subb + 1):
            app_oid = "mstransform_{0}_{1}".format(stri, "%02d" % j)
            adob_split = SleepAndCopyApp(app_oid, app_oid, lifespan=lifespan)
            adob_split.location = dob_obs.location
            dob_obs.addConsumer(adob_split)

            dob_sboid = "Split_{0}_{1}~{2}MHz".format(stri,
                                                      start_freq + subband_width * j,
                                                      start_freq + subband_width * (j + 1))
            dob_sb = InMemoryDROP(dob_sboid, dob_sboid, lifespan=lifespan)
            dob_sb.location = dob_obs.location
            adob_split.addOutput(dob_sb)

            subband_dict[j].append(dob_sb)

    for j, v in subband_dict.items():
        oid = "Subband_{0}~{1}MHz".format(start_freq + subband_width * j,
                                          start_freq + subband_width * (j + 1))
        appOid = "Combine_" + oid
        dob = InMemoryDROP(oid, oid)
        dob.location = "{0}.aws-ec2.sydney".format(j % num_obs)
        subbandCombineAppDrop = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
        subbandCombineAppDrop.addOutput(dob)
        for dob_sb in v:
            subbandCombineAppDrop.addInput(dob_sb)

        app_oid = oid.replace("Subband_", "Clean_")
        adob_clean = SleepAndCopyApp(app_oid, app_oid, lifespan=lifespan)
        adob_clean.location = dob.location
        dob.addConsumer(adob_clean)

        img_oid = oid.replace("Subband_", "Image_")
        dob_img = InMemoryDROP(img_oid, img_oid, lifespan=lifespan)
        dob_img.location = dob.location
        adob_clean.addOutput(dob_img)
        img_list.append(dob_img)

    #container + app that produces it
    combineAppDrop = SleepAndCopyApp('Combine', 'Combine')
    dob_comb_img_oid = "Combined_image"
    dob_comb_img = InMemoryDROP(dob_comb_img_oid, dob_comb_img_oid, lifespan=lifespan)
    dob_comb_img.location = "10.1.1.100:7777"
    combineAppDrop.addOutput(dob_comb_img)
    for dob_img in img_list:
        combineAppDrop.addInput(dob_img)

    #concatenate all images
    adob_concat_oid = "Concat_image"
    adob_concat = SleepAndCopyApp(adob_concat_oid, adob_concat_oid, lifespan=lifespan)
    adob_concat.location = dob_comb_img.location
    dob_comb_img.addConsumer(adob_concat)

    # produce cube
    dob_cube_oid = "Cube_Day{0}~{1}_{2}~{3}MHz".format(1,
                                                       num_obs,
                                                       start_freq,
                                                       start_freq + total_bandwidth)

    dob_cube = InMemoryDROP(dob_cube_oid, dob_cube_oid, lifespan=lifespan)
    dob_cube.location = dob_comb_img.location
    adob_concat.addOutput(dob_cube)

    return roots
Ejemplo n.º 35
0
    def branch_failure(self, tooManyFailures):
        """
        Using the container data object to implement a join/barrier dataflow.

        A1, A2 and A3 are FileDROPs
        B1, B2 and B3 are SumupContainerChecksum
        C1, C2 and C3 are InMemoryDROPs
        D is a SumupContainerChecksum
        E is a InMemoryDROP

        --> A1 --> B1 --> C1 --|
        --> A2 --> B2 --> C2 --|--> D --> E
        --> A3 --> B3 --> C3 --|

        Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
        after which "C" will transition to COMPLETE. Once all "C"s have moved to
        COMPLETED "D"'s execution will also be triggered, and finally E will
        hold the sum of B1, B2 and B3's checksums
        """

        # create file data objects
        a1 = InMemoryDROP("oid:A1", "uid:A1")
        a2 = InMemoryDROP("oid:A2", "uid:A2")
        a3 = InMemoryDROP("oid:A3", "uid:A3")

        # CRC Result DROPs, storing the result in memory
        b1 = SumupContainerChecksum("oid:B1", "uid:B1")
        b2 = SumupContainerChecksum("oid:B2", "uid:B2")
        b3 = SumupContainerChecksum("oid:B3", "uid:B3")
        c1 = InMemoryDROP("oid:C1", "uid:C1")
        c2 = InMemoryDROP("oid:C2", "uid:C2")
        c3 = InMemoryDROP("oid:C3", "uid:C3")

        # The final DROP that sums up the CRCs from the container DROP
        d = SumupContainerChecksum("oid:D", "uid:D", input_error_threshold=33)
        e = InMemoryDROP("oid:E", "uid:E")

        # Wire together
        dropAList = [a1, a2, a3]
        dropBList = [b1, b2, b3]
        dropCList = [c1, c2, c3]
        for dropA, dropB in map(lambda a, b: (a, b), dropAList, dropBList):
            dropA.addConsumer(dropB)
        for dropB, dropC in map(lambda b, c: (b, c), dropBList, dropCList):
            dropB.addOutput(dropC)
        for dropC in dropCList:
            dropC.addConsumer(d)
        d.addOutput(e)

        # Write data into the initial "A" DROPs, which should trigger
        # the whole chain explained above
        with DROPWaiterCtx(self, e):
            # for dropA in dropAList: # this should be parallel for
            a1.write(" ")
            a1.setCompleted()
            if tooManyFailures:
                a2.setError()
            else:
                a2.write(" ")
                a2.setCompleted()
            a3.setError()

        if tooManyFailures:
            completedDrops = dropAList[0:1] + dropBList[0:1] + dropCList[0:1]
            errorDrops = dropAList[1:] + dropBList[1:] + dropCList[1:] + [d, e]
        else:
            completedDrops = dropAList[0:2] + dropBList[0:2] + dropCList[0:2] + [d, e]
            errorDrops = dropAList[2:] + dropBList[2:] + dropCList[2:]

        for drop in completedDrops:
            self.assertEquals(drop.status, DROPStates.COMPLETED)
        for drop in errorDrops:
            self.assertEquals(drop.status, DROPStates.ERROR)

        # The results we want to compare
        # (only in case that at least two branches executed)
        if not tooManyFailures:
            sum_crc = c1.checksum + c2.checksum
            dropEData = int(droputils.allDropContents(e))

            self.assertNotEquals(sum_crc, 0)
            self.assertEquals(sum_crc, dropEData)
Ejemplo n.º 36
0
    def test_join(self):
        """
        Using the container data object to implement a join/barrier dataflow.

        A1, A2 and A3 are FileDROPs
        B1, B2 and B3 are SumupContainerChecksum
        C1, C2 and C3 are InMemoryDROPs
        D is a SumupContainerChecksum
        E is a InMemoryDROP

        --> A1 --> B1 --> C1 --|
        --> A2 --> B2 --> C2 --|--> D --> E
        --> A3 --> B3 --> C3 --|

        Upon writing all A* DROPs, the execution of B* DROPs should be triggered,
        after which "C" will transition to COMPLETE. Once all "C"s have moved to
        COMPLETED "D"'s execution will also be triggered, and finally E will
        hold the sum of B1, B2 and B3's checksums
        """

        filelen = self._test_drop_sz * ONE_MB
        #create file data objects
        a1 = FileDROP('oid:A1', 'uid:A1', expectedSize=filelen)
        a2 = FileDROP('oid:A2', 'uid:A2', expectedSize=filelen)
        a3 = FileDROP('oid:A3', 'uid:A3', expectedSize=filelen)

        # CRC Result DROPs, storing the result in memory
        b1 = SumupContainerChecksum('oid:B1', 'uid:B1')
        b2 = SumupContainerChecksum('oid:B2', 'uid:B2')
        b3 = SumupContainerChecksum('oid:B3', 'uid:B3')
        c1 = InMemoryDROP('oid:C1', 'uid:C1')
        c2 = InMemoryDROP('oid:C2', 'uid:C2')
        c3 = InMemoryDROP('oid:C3', 'uid:C3')

        # The final DROP that sums up the CRCs from the container DROP
        d = SumupContainerChecksum('oid:D', 'uid:D')
        e = InMemoryDROP('oid:E', 'uid:E')

        # Wire together
        dropAList = [a1,a2,a3]
        dropBList = [b1,b2,b3]
        dropCList = [c1,c2,c3]
        for dropA,dropB in map(lambda a,b: (a,b), dropAList, dropBList):
            dropA.addConsumer(dropB)
        for dropB,dropC in map(lambda b,c: (b,c), dropBList, dropCList):
            dropB.addOutput(dropC)
        for dropC in dropCList:
            dropC.addConsumer(d)
        d.addOutput(e)

        # Write data into the initial "A" DROPs, which should trigger
        # the whole chain explained above
        with DROPWaiterCtx(self, e):
            for dropA in dropAList: # this should be parallel for
                for _ in range(self._test_num_blocks):
                    dropA.write(self._test_block)

        # All DROPs are completed now that the chain executed correctly
        for drop in dropAList + dropBList + dropCList:
            self.assertEqual(drop.status, DROPStates.COMPLETED)

        # The results we want to compare
        sum_crc = c1.checksum + c2.checksum + c3.checksum
        dropEData = int(droputils.allDropContents(e))

        self.assertNotEqual(sum_crc, 0)
        self.assertEqual(sum_crc, dropEData)
Ejemplo n.º 37
0
def mwa_fornax_pg():
    num_coarse_ch = 24
    num_split = 3 # number of time splits per channel

    dob_root_sl = SocketListenerApp("MWA_LTA_SL", "MWA_LTA_SL", lifespan=lifespan)
    dob_root = InMemoryDROP("MWA_LTA", "MWA_LTA", lifespan=lifespan)
    dob_root.location = "Pawsey"
    dob_root_sl.addOutput(dob_root)

    #container
    dob_comb_img_oid = "Combined_image"
    combineImgAppOid = "Combine image"
    dob_comb_img = InMemoryDROP(dob_comb_img_oid, dob_comb_img_oid, lifespan=lifespan)
    combineImgApp = SleepAndCopyApp(combineImgAppOid, combineImgAppOid)
    combineImgApp.addOutput(dob_comb_img)
    dob_comb_img.location = "f032.fornax"

    for i in range(1, num_coarse_ch + 1):

        stri = "%02d" % i
        oid = "Subband_{0}".format(stri)
        appOid = "Combine_" + oid
        dob_obs = InMemoryDROP(oid, oid, lifespan=lifespan)
        combineSubbandApp = SleepAndCopyApp(appOid, appOid, lifespan=lifespan)
        combineSubbandApp.addOutput(dob_obs)
        dob_obs.location = "f%03d.fornax" % i

        oid_ingest = "NGAS_{0}".format(stri)
        dob_ingest = SleepAndCopyApp(oid_ingest, oid_ingest, lifespan=lifespan)
        dob_ingest.location = "f%03d.fornax:7777" % i

        dob_root.addConsumer(dob_ingest)

        for j in range(1, num_split + 1):
            strj = "%02d" % j
            split_oid = "Subband_{0}_Split_{1}".format(stri, strj)
            dob_split = InMemoryDROP(split_oid, split_oid, lifespan=lifespan)
            dob_split.location = dob_obs.location
            dob_ingest.addOutput(dob_split)
            combineSubbandApp.addInput(dob_split)

        oid_rts = "RTS_{0}".format(stri)
        dob_rts = SleepAndCopyApp(oid_rts, oid_rts, lifespan=lifespan)
        dob_rts.location = dob_obs.location
        dob_obs.addConsumer(dob_rts)

        oid_subimg = "Subcube_{0}".format(stri)
        dob_subimg = InMemoryDROP(oid_subimg, oid_subimg, lifespan=lifespan)
        dob_subimg.location = dob_obs.location
        dob_rts.addOutput(dob_subimg)
        combineImgApp.addInput(dob_subimg)

    #concatenate all images
    adob_concat_oid = "Concat_image"
    adob_concat = SleepAndCopyApp(adob_concat_oid, adob_concat_oid, lifespan=lifespan)
    adob_concat.location = dob_comb_img.location
    dob_comb_img.addConsumer(adob_concat)

    # produce cube
    dob_cube_oid = "Cube_30.72MHz"
    dob_cube = InMemoryDROP(dob_cube_oid, dob_cube_oid, lifespan=lifespan)
    dob_cube.location = dob_comb_img.location
    adob_concat.addOutput(dob_cube)

    return dob_root_sl
Ejemplo n.º 38
0
    def _createGraph(self):
        """
        Creates the following graph of DROPs:

        A |--> B ----> D --> G --> I --|
          |--> C -|--> E --------------|-> H --> J
                  |--> F

        B, C, G and H are AppDOs. The names have been given in breadth-first
        order (although H has a dependency on I)
        """
        a =          InMemoryDROP('a', 'a')
        b =        BarrierAppDROP('b', 'b')
        c =        BarrierAppDROP('c', 'c')
        d =          InMemoryDROP('d', 'd')
        e =          InMemoryDROP('e', 'e')
        f =          InMemoryDROP('f', 'f')
        g =        BarrierAppDROP('g', 'g')
        h =        BarrierAppDROP('h', 'h')
        i =          InMemoryDROP('i', 'i')
        j =          InMemoryDROP('j', 'j')

        a.addConsumer(b)
        a.addConsumer(c)
        b.addOutput(d)
        c.addOutput(e)
        c.addOutput(f)
        d.addConsumer(g)
        e.addConsumer(h)
        g.addOutput(i)
        i.addConsumer(h)
        h.addOutput(j)

        return a, b, c, d, e, f, g, h, i, j
Ejemplo n.º 39
0
 def createDrop(self, oid, uid, **kwargs):
     return InMemoryDROP(oid, uid, **kwargs)
Ejemplo n.º 40
0
        self._evt = threading.Event()
        drop.addConsumer(self)

    def dropCompleted(self, drop, state):
        self._evt.set()

    def wait(self, timeout=None):
        return self._evt.wait(timeout)


if __name__ == '__main__':
    try:

        vis_in_a = []

        flux_out = InMemoryDROP(uuid.uuid1(), uuid.uuid1())

        flux = SourceFlux(uuid.uuid1(), uuid.uuid1(), casapy_path=CASAPY)

        cl = Clean(uuid.uuid1(),
                   uuid.uuid1(),
                   field='deepfield',
                   mode='frequency',
                   restfreq='1420.405752MHz',
                   nchan=-1,
                   start='',
                   width='',
                   interpolation='nearest',
                   gain=0.1,
                   imsize=[256],
                   cell=['1.0arcsec'],
Ejemplo n.º 41
0
    def test_simple_chain(self):
        """
        Simple test that creates a pipeline-like chain of commands.
        In this case we simulate a pipeline that does this, holding
        each intermediate result in memory:

        cat someFile | grep 'a' | sort | rev
        """

        class GrepResult(BarrierAppDROP):
            def initialize(self, **kwargs):
                super(GrepResult, self).initialize(**kwargs)
                self._substring = kwargs["substring"]

            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                allLines = StringIO(droputils.allDropContents(drop)).readlines()
                for line in allLines:
                    if self._substring in line:
                        output.write(line)

        class SortResult(BarrierAppDROP):
            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                sortedLines = StringIO(droputils.allDropContents(drop)).readlines()
                sortedLines.sort()
                for line in sortedLines:
                    output.write(line)

        class RevResult(BarrierAppDROP):
            def run(self):
                drop = self.inputs[0]
                output = self.outputs[0]
                allLines = StringIO(droputils.allDropContents(drop)).readlines()
                for line in allLines:
                    buf = ""
                    for c in line:
                        if c == " " or c == "\n":
                            output.write(buf[::-1])
                            output.write(c)
                            buf = ""
                        else:
                            buf += c

        a = InMemoryDROP("oid:A", "uid:A")
        b = GrepResult("oid:B", "uid:B", substring="a")
        c = InMemoryDROP("oid:C", "uid:C")
        d = SortResult("oid:D", "uid:D")
        e = InMemoryDROP("oid:E", "uid:E")
        f = RevResult("oid:F", "oid:F")
        g = InMemoryDROP("oid:G", "uid:G")

        a.addConsumer(b)
        b.addOutput(c)
        c.addConsumer(d)
        d.addOutput(e)
        e.addConsumer(f)
        f.addOutput(g)

        # Initial write
        contents = "first line\nwe have an a here\nand another one\nnoone knows me"
        cResExpected = "we have an a here\nand another one\n"
        eResExpected = "and another one\nwe have an a here\n"
        gResExpected = "dna rehtona eno\new evah na a ereh\n"

        with DROPWaiterCtx(self, g):
            a.write(contents)
            a.setCompleted()

        # Get intermediate and final results and compare
        actualRes = []
        for i in [c, e, g]:
            actualRes.append(droputils.allDropContents(i))
        map(lambda x, y: self.assertEquals(x, y), [cResExpected, eResExpected, gResExpected], actualRes)