Exemplo n.º 1
0
    def test_nested_task(self):
        pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a', ), ('b', ))
        subgraph = G(
            C(START_TICK, 'a', 1, 'a'),
            C(START_TICK, 'context', 1, 'context'),
            T(1, exec_task, {
                'name': 'test_exec',
                'path': 'test_exec'
            }),
            C(1, 'b', FINAL_TICK, 'b'),
        )

        subgraph_task_tick = subgraph.get_all_ticks()[0]
        graph = Graph()
        task = NestedGraphTask(subgraph, 'test_nested')
        tick = START_TICK + 1
        graph.add_task(tick, task, {
            'name': 'test_nested',
            'path': 'test_nested'
        })
        source = Endpoint(START_TICK, 'a')
        dest = Endpoint(tick, 'a')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'context')
        dest = Endpoint(tick, 'context')
        graph.connect(source, dest)
        source = Endpoint(tick, 'b')
        dest = Endpoint(FINAL_TICK, 'b')
        graph.connect(source, dest)

        task.refine(graph, Tick.parse_tick(1), {'context': {}, 'a': {}})

        expected_graph_task_tick = subgraph_task_tick << Tick.parse_tick(1)
        expected = G(
            C(START_TICK, 'a', expected_graph_task_tick, 'a'),
            C(START_TICK, 'context', expected_graph_task_tick, 'context'),
            T(expected_graph_task_tick, exec_task, {
                'name': 'test_exec',
                'path': 'test_nested.test_exec'
            }),
            C(expected_graph_task_tick, 'b', FINAL_TICK, 'b'),
        )
        utils.assert_graph_equal(expected, graph)
Exemplo n.º 2
0
    def test_parallel_split(self):
        pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a', 'b'), ('c', 'd'))
        subgraph = G(
            C(START_TICK, 'a', 1, 'a'),
            C(START_TICK, 'b', 1, 'b'),
            C(START_TICK, 'context', 1, 'context'),
            T(1, exec_task, {
                'name': 'test_exec',
                'path': 'test_exec'
            }),
            C(1, 'c', FINAL_TICK, 'c'),
            C(1, 'd', FINAL_TICK, 'd'),
        )

        subgraph_task_tick = subgraph.get_all_ticks()[0]
        graph = Graph()
        task = ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps')
        tick = START_TICK + 1
        graph.add_task(tick, task, {'name': 'ps', 'path': 'ps'})
        source = Endpoint(START_TICK, 'a')
        dest = Endpoint(tick, 'a')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'b')
        dest = Endpoint(tick, 'b')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'context')
        dest = Endpoint(tick, 'context')
        graph.connect(source, dest)
        source = Endpoint(tick, 'cd_list')
        dest = Endpoint(FINAL_TICK, 'cd_list')
        graph.connect(source, dest)

        testdir = tempfile.mkdtemp()
        workdir = os.path.join(testdir, 'workdir')
        os.mkdir(workdir)
        testfile = os.path.join(workdir, 'testfile.pickle')
        local_workdir = os.path.join(testdir, 'local_workdir')
        os.mkdir(local_workdir)
        with open(testfile, 'wb') as f:
            pickle.dump(['x', 'y', 'z'], f)
        context = {
            'transporter': LocalFileTransporter(),
            'workdir': workdir,
            'local_workdir': local_workdir
        }
        task.refine(graph, Tick.parse_tick(1), {
            'b': 'testfile.pickle',
            'context': context
        })

        mapTask = _mapPortsTask('b', ['x', 'y', 'z'])
        map_task_tick = START_TICK + 1 << tick

        iterations_tick = map_task_tick + 1
        it_1 = START_TICK + 1 << iterations_tick
        it_2 = START_TICK + 2 << iterations_tick
        it_3 = START_TICK + 3 << iterations_tick
        task_tick_1 = subgraph_task_tick << it_1
        task_tick_2 = subgraph_task_tick << it_2
        task_tick_3 = subgraph_task_tick << it_3

        reduceTask = _reducePortsTask(3, ['c', 'd'], 'cd_list')
        reduce_tick = iterations_tick + 1

        expected = G(
            C(START_TICK, 'context', map_task_tick, 'context'),
            C(START_TICK, 'a', task_tick_1, 'a'),
            C(START_TICK, 'a', task_tick_2, 'a'),
            C(START_TICK, 'a', task_tick_3, 'a'),
            C(START_TICK, 'b', map_task_tick, 'inputlist'),
            C(START_TICK, 'context', task_tick_1, 'context'),
            C(START_TICK, 'context', task_tick_2, 'context'),
            C(START_TICK, 'context', task_tick_3, 'context'),
            T(map_task_tick, mapTask, {
                'name': 'map',
                'path': 'ps.map'
            }),
            C(map_task_tick, 'b_1', task_tick_1, 'b'),
            C(map_task_tick, 'b_2', task_tick_2, 'b'),
            C(map_task_tick, 'b_3', task_tick_3, 'b'),
            T(task_tick_1, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.1.test_exec'
            }),
            T(task_tick_2, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.2.test_exec'
            }),
            T(task_tick_3, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.3.test_exec'
            }),
            C(task_tick_1, 'c', reduce_tick, 'c_1'),
            C(task_tick_2, 'c', reduce_tick, 'c_2'),
            C(task_tick_3, 'c', reduce_tick, 'c_3'),
            C(task_tick_1, 'd', reduce_tick, 'd_1'),
            C(task_tick_2, 'd', reduce_tick, 'd_2'),
            C(task_tick_3, 'd', reduce_tick, 'd_3'),
            C(START_TICK, 'context', reduce_tick, 'context'),
            T(reduce_tick, reduceTask, {
                'name': 'reduce',
                'path': 'ps.reduce'
            }),
            C(reduce_tick, 'cd_list', FINAL_TICK, 'cd_list'),
        )
        utils.assert_graph_equal(expected, graph)
        self.assertTrue(
            os.path.exists(os.path.join(local_workdir, 'testfile.pickle')))