Ejemplo n.º 1
0
    def test_to_first_split(self):
        s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
        copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
        measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
        copy_from_s3.addInput(s3_drop)
        copy_from_s3.addOutput(measurement_set)

        outputs = []
        frequencies = make_groups_of_frequencies(FREQUENCY_GROUPS, 5)
        frequencies = frequencies[0]
        for group in frequencies:
            casa_py_drop = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='casa_py.sh /dfms_root/%i0 /dfms_root/%o0 {0} {1}'.format(group[0], group[1]), user='******')
            result = FileDROP(self.get_oid('file'), uuid.uuid4(), dirname=TestChiles02._temp)
            copy_to_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_to_s3.sh /dfms_root/%i0 %oDataURL0', user='******')
            s3_drop_out = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='{0}_{1}/key123'.format(group[0], group[1]), profile_name='aws-profile')

            casa_py_drop.addInput(measurement_set)
            casa_py_drop.addOutput(result)
            copy_to_s3.addInput(result)
            copy_to_s3.addOutput(s3_drop_out)
            outputs.append(s3_drop_out)

        barrier_drop = BarrierAppDROP(self.get_oid('barrier'), uuid.uuid4())
        barrier_drop.addInput(measurement_set)

        for output in outputs:
            barrier_drop.addInput(output)

        with droputils.DROPWaiterCtx(self, barrier_drop, 50000):
            s3_drop.setCompleted()
Ejemplo n.º 2
0
    def test_start_graph(self):
        s3_drop = MockS3DROP(self.get_oid('s3'), uuid.uuid4(), bucket='mock', key='key123', profile_name='aws-profile')
        copy_from_s3 = DockerApp(self.get_oid('app'), uuid.uuid4(), image='mock:latest', command='copy_from_s3.sh %iDataURL0 /dfms_root/%o0', user='******')
        measurement_set = DirectoryContainer(self.get_oid('dir'), uuid.uuid4(), dirname=TestChiles02._temp)
        copy_from_s3.addInput(s3_drop)
        copy_from_s3.addOutput(measurement_set)

        with droputils.DROPWaiterCtx(self, measurement_set, 50000):
            s3_drop.setCompleted()
Ejemplo n.º 3
0
    def test_directoryContainer(self):
        """
        A small, simple test for the DirectoryContainer DROP that checks it allows
        only valid children to be added
        """

        # Prepare our playground
        cwd = os.getcwd()
        os.chdir('/tmp')
        dirname  = ".hidden"
        dirname2 = ".hidden/inside"
        if not os.path.exists(dirname2):
            os.makedirs(dirname2)

        # DROPs involved
        a = FileDROP('a', 'a', dirname=dirname)
        b = FileDROP('b', 'b', dirname=dirname)
        c = FileDROP('c', 'c', dirname=dirname2)
        d = FileDROP('d', 'd', dirname=dirname2)
        cont1 = DirectoryContainer('e', 'e', dirname=dirname)
        cont2 = DirectoryContainer('f', 'f', dirname=dirname2)

        # Paths are absolutely reported
        self.assertEqual(os.path.realpath('/tmp/.hidden'), os.path.realpath(cont1.path))
        self.assertEqual(os.path.realpath('/tmp/.hidden/inside'), os.path.realpath(cont2.path))

        # Certain children-to-be are rejected
        self.assertRaises(TypeError, cont1.addChild, NullDROP('g', 'g'))
        self.assertRaises(TypeError, cont1.addChild, InMemoryDROP('h', 'h'))
        self.assertRaises(TypeError, cont1.addChild, ContainerDROP('i', 'i'))
        self.assertRaises(Exception, cont1.addChild, c)
        self.assertRaises(Exception, cont1.addChild, d)
        self.assertRaises(Exception, cont2.addChild, a)
        self.assertRaises(Exception, cont2.addChild, b)

        # These children are correct
        cont1.addChild(a)
        cont1.addChild(b)
        cont2.addChild(c)
        cont2.addChild(d)

        # Revert to previous state
        shutil.rmtree(dirname, True)
        os.chdir(cwd)
Ejemplo n.º 4
0
    def test_directoryContainer(self):
        """
        A small, simple test for the DirectoryContainer DROP that checks it allows
        only valid children to be added
        """

        # Prepare our playground
        cwd = os.getcwd()
        os.chdir("/tmp")
        dirname = ".hidden"
        dirname2 = ".hidden/inside"
        if not os.path.exists(dirname2):
            os.makedirs(dirname2)

        # DROPs involved
        a = FileDROP("a", "a", dirname=dirname)
        b = FileDROP("b", "b", dirname=dirname)
        c = FileDROP("c", "c", dirname=dirname2)
        d = FileDROP("d", "d", dirname=dirname2)
        cont1 = DirectoryContainer("e", "e", dirname=dirname)
        cont2 = DirectoryContainer("f", "f", dirname=dirname2)

        # Paths are absolutely reported
        self.assertEquals(os.path.realpath("/tmp/.hidden"), os.path.realpath(cont1.path))
        self.assertEquals(os.path.realpath("/tmp/.hidden/inside"), os.path.realpath(cont2.path))

        # Certain children-to-be are rejected
        self.assertRaises(TypeError, cont1.addChild, NullDROP("g", "g"))
        self.assertRaises(TypeError, cont1.addChild, InMemoryDROP("h", "h"))
        self.assertRaises(TypeError, cont1.addChild, ContainerDROP("i", "i"))
        self.assertRaises(Exception, cont1.addChild, c)
        self.assertRaises(Exception, cont1.addChild, d)
        self.assertRaises(Exception, cont2.addChild, a)
        self.assertRaises(Exception, cont2.addChild, b)

        # These children are correct
        cont1.addChild(a)
        cont1.addChild(b)
        cont2.addChild(c)
        cont2.addChild(d)

        # Revert to previous state
        shutil.rmtree(dirname, True)
        os.chdir(cwd)
Ejemplo n.º 5
0
                   field='deepfield',
                   mode='frequency',
                   restfreq='1420.405752MHz',
                   nchan=-1,
                   start='',
                   width='',
                   interpolation='nearest',
                   gain=0.1,
                   imsize=[256],
                   cell=['1.0arcsec'],
                   phasecenter='10h01m53.9,+02d24m52s',
                   weighting='natural',
                   casapy_path=CASAPY)

        image_out = DirectoryContainer(uuid.uuid1(),
                                       uuid.uuid1(),
                                       dirname=CUBE_OUT + CUBE_NAME,
                                       check_exists=False)
        cl.addOutput(image_out)
        flux.addInput(image_out)
        flux.addOutput(flux_out)

        for v in VIS:
            vis_in = DirectoryContainer(uuid.uuid1(),
                                        uuid.uuid1(),
                                        dirname=v[0])
            split_out = DirectoryContainer(uuid.uuid1(),
                                           uuid.uuid1(),
                                           dirname=v[1],
                                           check_exists=False)

            vis_in_a.append(vis_in)
Ejemplo n.º 6
0
    def test_expireAfterUse(self):
        """
        Simple test for the expireAfterUse flag. Two DROPs are created with
        different values, and after they are used we check whether their data
        is still there or not
        """
        with dlm.DataLifecycleManager(checkPeriod=0.5, cleanupPeriod=2) as manager:
            a = DirectoryContainer('a', 'a', precious=False, expireAfterUse=True, dirname=tempfile.mkdtemp())
            b_dirname = tempfile.mkdtemp()
            b = DirectoryContainer('b', 'b', precious=False, expireAfterUse=False, dirname=b_dirname)
            c = BarrierAppDROP('c', 'c')
            d = BarrierAppDROP('d', 'd')
            a.addConsumer(c)
            a.addConsumer(d)
            b.addConsumer(c)
            b.addConsumer(d)

            manager.addDrop(a)
            manager.addDrop(b)
            manager.addDrop(b)
            manager.addDrop(c)

            # Make sure all consumers are done
            with DROPWaiterCtx(self, [c,d], 1):
                a.setCompleted()
                b.setCompleted()

            # Both directories should be there, but after cleanup A's shouldn't
            # be there anymore
            self.assertTrue(a.exists())
            self.assertTrue(b.exists())
            time.sleep(2)
            self.assertFalse(a.exists())
            self.assertTrue(b.exists())
            b.delete()
Ejemplo n.º 7
0
    def test_expireAfterUse(self):
        """
        Simple test for the expireAfterUse flag. Two DROPs are created with
        different values, and after they are used we check whether their data
        is still there or not
        """
        with dlm.DataLifecycleManager(checkPeriod=0.5, cleanupPeriod=2) as manager:
            a = DirectoryContainer('a', 'a', precious=False, expireAfterUse=True, dirname=tempfile.mkdtemp())
            b_dirname = tempfile.mkdtemp()
            b = DirectoryContainer('b', 'b', precious=False, expireAfterUse=False, dirname=b_dirname)
            c = BarrierAppDROP('c', 'c')
            d = BarrierAppDROP('d', 'd')
            a.addConsumer(c)
            a.addConsumer(d)
            b.addConsumer(c)
            b.addConsumer(d)

            manager.addDrop(a)
            manager.addDrop(b)
            manager.addDrop(b)
            manager.addDrop(c)

            # Make sure all consumers are done
            with DROPWaiterCtx(self, [c,d], 1):
                a.setCompleted()
                b.setCompleted()

            # Both directories should be there, but after cleanup A's shouldn't
            # be there anymore
            self.assertTrue(a.exists())
            self.assertTrue(b.exists())
            time.sleep(2)
            self.assertFalse(a.exists())
            self.assertTrue(b.exists())
            b.delete()