def test_nested_task(self):
        pkg=PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a',), ('b',))
        subgraph =  G(
            C(START_TICK, 'a', 1,'a'),
            C(START_TICK, 'context', 1,'context'),
            T(1, exec_task, {'name': 'test_exec', 'path':'test_exec'}),
            C(1, 'b', FINAL_TICK,'b'),
        )
        
        subgraph_task_tick=subgraph.get_all_ticks()[0]
        graph=Graph()
        task=NestedGraphTask(subgraph, 'test_nested')
        tick=START_TICK+1
        graph.add_task(tick, task, {'name':'test_nested','path':'test_nested'})
        source=Endpoint(START_TICK, 'a')
        dest=Endpoint(tick, 'a')
        graph.connect(source,dest)
        source=Endpoint(START_TICK, 'context')
        dest=Endpoint(tick, 'context')
        graph.connect(source,dest)
        source=Endpoint(tick, 'b')
        dest=Endpoint(FINAL_TICK, 'b')
        graph.connect(source,dest)

        task.refine(graph, Tick.parse_tick(1), {'context':{},'a':{}})

        expected_graph_task_tick=subgraph_task_tick<<Tick.parse_tick(1)        
        expected =  G(
            C(START_TICK, 'a', expected_graph_task_tick,'a'),
            C(START_TICK, 'context', expected_graph_task_tick,'context'),
            T(expected_graph_task_tick, exec_task, {'name': 'test_exec', 'path':'test_nested.test_exec'}),
            C(expected_graph_task_tick, 'b', FINAL_TICK,'b'),
        )
        utils.assert_graph_equal(expected, graph)
Exemplo n.º 2
0
    def test_nested_task(self):
        pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a', ), ('b', ))
        subgraph = G(
            C(START_TICK, 'a', 1, 'a'),
            C(START_TICK, 'context', 1, 'context'),
            T(1, exec_task, {
                'name': 'test_exec',
                'path': 'test_exec'
            }),
            C(1, 'b', FINAL_TICK, 'b'),
        )

        subgraph_task_tick = subgraph.get_all_ticks()[0]
        graph = Graph()
        task = NestedGraphTask(subgraph, 'test_nested')
        tick = START_TICK + 1
        graph.add_task(tick, task, {
            'name': 'test_nested',
            'path': 'test_nested'
        })
        source = Endpoint(START_TICK, 'a')
        dest = Endpoint(tick, 'a')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'context')
        dest = Endpoint(tick, 'context')
        graph.connect(source, dest)
        source = Endpoint(tick, 'b')
        dest = Endpoint(FINAL_TICK, 'b')
        graph.connect(source, dest)

        task.refine(graph, Tick.parse_tick(1), {'context': {}, 'a': {}})

        expected_graph_task_tick = subgraph_task_tick << Tick.parse_tick(1)
        expected = G(
            C(START_TICK, 'a', expected_graph_task_tick, 'a'),
            C(START_TICK, 'context', expected_graph_task_tick, 'context'),
            T(expected_graph_task_tick, exec_task, {
                'name': 'test_exec',
                'path': 'test_nested.test_exec'
            }),
            C(expected_graph_task_tick, 'b', FINAL_TICK, 'b'),
        )
        utils.assert_graph_equal(expected, graph)
Exemplo n.º 3
0
    def test_parallel_split(self):
        pkg = PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a', 'b'), ('c', 'd'))
        subgraph = G(
            C(START_TICK, 'a', 1, 'a'),
            C(START_TICK, 'b', 1, 'b'),
            C(START_TICK, 'context', 1, 'context'),
            T(1, exec_task, {
                'name': 'test_exec',
                'path': 'test_exec'
            }),
            C(1, 'c', FINAL_TICK, 'c'),
            C(1, 'd', FINAL_TICK, 'd'),
        )

        subgraph_task_tick = subgraph.get_all_ticks()[0]
        graph = Graph()
        task = ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps')
        tick = START_TICK + 1
        graph.add_task(tick, task, {'name': 'ps', 'path': 'ps'})
        source = Endpoint(START_TICK, 'a')
        dest = Endpoint(tick, 'a')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'b')
        dest = Endpoint(tick, 'b')
        graph.connect(source, dest)
        source = Endpoint(START_TICK, 'context')
        dest = Endpoint(tick, 'context')
        graph.connect(source, dest)
        source = Endpoint(tick, 'cd_list')
        dest = Endpoint(FINAL_TICK, 'cd_list')
        graph.connect(source, dest)

        testdir = tempfile.mkdtemp()
        workdir = os.path.join(testdir, 'workdir')
        os.mkdir(workdir)
        testfile = os.path.join(workdir, 'testfile.pickle')
        local_workdir = os.path.join(testdir, 'local_workdir')
        os.mkdir(local_workdir)
        with open(testfile, 'wb') as f:
            pickle.dump(['x', 'y', 'z'], f)
        context = {
            'transporter': LocalFileTransporter(),
            'workdir': workdir,
            'local_workdir': local_workdir
        }
        task.refine(graph, Tick.parse_tick(1), {
            'b': 'testfile.pickle',
            'context': context
        })

        mapTask = _mapPortsTask('b', ['x', 'y', 'z'])
        map_task_tick = START_TICK + 1 << tick

        iterations_tick = map_task_tick + 1
        it_1 = START_TICK + 1 << iterations_tick
        it_2 = START_TICK + 2 << iterations_tick
        it_3 = START_TICK + 3 << iterations_tick
        task_tick_1 = subgraph_task_tick << it_1
        task_tick_2 = subgraph_task_tick << it_2
        task_tick_3 = subgraph_task_tick << it_3

        reduceTask = _reducePortsTask(3, ['c', 'd'], 'cd_list')
        reduce_tick = iterations_tick + 1

        expected = G(
            C(START_TICK, 'context', map_task_tick, 'context'),
            C(START_TICK, 'a', task_tick_1, 'a'),
            C(START_TICK, 'a', task_tick_2, 'a'),
            C(START_TICK, 'a', task_tick_3, 'a'),
            C(START_TICK, 'b', map_task_tick, 'inputlist'),
            C(START_TICK, 'context', task_tick_1, 'context'),
            C(START_TICK, 'context', task_tick_2, 'context'),
            C(START_TICK, 'context', task_tick_3, 'context'),
            T(map_task_tick, mapTask, {
                'name': 'map',
                'path': 'ps.map'
            }),
            C(map_task_tick, 'b_1', task_tick_1, 'b'),
            C(map_task_tick, 'b_2', task_tick_2, 'b'),
            C(map_task_tick, 'b_3', task_tick_3, 'b'),
            T(task_tick_1, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.1.test_exec'
            }),
            T(task_tick_2, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.2.test_exec'
            }),
            T(task_tick_3, exec_task, {
                'name': 'test_exec',
                'path': 'ps.iterations.3.test_exec'
            }),
            C(task_tick_1, 'c', reduce_tick, 'c_1'),
            C(task_tick_2, 'c', reduce_tick, 'c_2'),
            C(task_tick_3, 'c', reduce_tick, 'c_3'),
            C(task_tick_1, 'd', reduce_tick, 'd_1'),
            C(task_tick_2, 'd', reduce_tick, 'd_2'),
            C(task_tick_3, 'd', reduce_tick, 'd_3'),
            C(START_TICK, 'context', reduce_tick, 'context'),
            T(reduce_tick, reduceTask, {
                'name': 'reduce',
                'path': 'ps.reduce'
            }),
            C(reduce_tick, 'cd_list', FINAL_TICK, 'cd_list'),
        )
        utils.assert_graph_equal(expected, graph)
        self.assertTrue(
            os.path.exists(os.path.join(local_workdir, 'testfile.pickle')))
    def test_parallel_split(self):
        pkg=PackageSource('testpkg', 'pkgfile', 'NONE')
        exec_task = ExecTask("exec_name", pkg, ('a','b'), ('c','d'))
        subgraph =  G(
            C(START_TICK, 'a', 1,'a'),
            C(START_TICK, 'b', 1,'b'),
            C(START_TICK, 'context', 1,'context'),
            T(1, exec_task, {'name': 'test_exec', 'path':'test_exec'}),
            C(1, 'c', FINAL_TICK,'c'),
            C(1, 'd', FINAL_TICK,'d'),
        )
        
        subgraph_task_tick=subgraph.get_all_ticks()[0]
        graph=Graph()
        task=ParallelSplitTask(subgraph, 'b', 'cd_list', 'ps')
        tick=START_TICK+1
        graph.add_task(tick, task, {'name':'ps','path':'ps'})
        source=Endpoint(START_TICK, 'a')
        dest=Endpoint(tick, 'a')
        graph.connect(source,dest)
        source=Endpoint(START_TICK, 'b')
        dest=Endpoint(tick, 'b')
        graph.connect(source,dest)
        source=Endpoint(START_TICK, 'context')
        dest=Endpoint(tick, 'context')
        graph.connect(source,dest)
        source=Endpoint(tick, 'cd_list')
        dest=Endpoint(FINAL_TICK, 'cd_list')
        graph.connect(source,dest)

        testdir=tempfile.mkdtemp()
        workdir=os.path.join(testdir,'workdir')
        os.mkdir(workdir)
        testfile=os.path.join(workdir, 'testfile.pickle')
        local_workdir=os.path.join(testdir,'local_workdir')
        os.mkdir(local_workdir)
        with open(testfile,'wb') as f:
            pickle.dump(['x','y','z'],f)
        context={'transporter': LocalFileTransporter(), 'workdir':workdir, 'local_workdir':local_workdir}
        task.refine(graph, Tick.parse_tick(1), {'b':'testfile.pickle', 'context':context})

        mapTask=_mapPortsTask('b', ['x','y','z'])
        map_task_tick=START_TICK+1<<tick

        iterations_tick=map_task_tick+1
        it_1 = START_TICK + 1 << iterations_tick
        it_2 = START_TICK + 2 << iterations_tick
        it_3 = START_TICK + 3 << iterations_tick
        task_tick_1=subgraph_task_tick<<it_1
        task_tick_2=subgraph_task_tick<<it_2
        task_tick_3=subgraph_task_tick<<it_3
                
        reduceTask=_reducePortsTask(3, ['c','d'], 'cd_list')
        reduce_tick=iterations_tick+1
        
        expected =  G(
            C(START_TICK, 'context', map_task_tick,'context'),
            C(START_TICK, 'a', task_tick_1,'a'),
            C(START_TICK, 'a', task_tick_2,'a'),
            C(START_TICK, 'a', task_tick_3,'a'),
            C(START_TICK, 'b', map_task_tick,'inputlist'),
            C(START_TICK, 'context', task_tick_1,'context'),
            C(START_TICK, 'context', task_tick_2,'context'),
            C(START_TICK, 'context', task_tick_3,'context'),
            T(map_task_tick, mapTask, {'name': 'map', 'path':'ps.map'}),
            C(map_task_tick, 'b_1', task_tick_1,'b'),
            C(map_task_tick, 'b_2', task_tick_2,'b'),
            C(map_task_tick, 'b_3', task_tick_3,'b'),
            T(task_tick_1, exec_task, {'name': 'test_exec', 'path':'ps.iterations.1.test_exec'}),
            T(task_tick_2, exec_task, {'name': 'test_exec', 'path':'ps.iterations.2.test_exec'}),
            T(task_tick_3, exec_task, {'name': 'test_exec', 'path':'ps.iterations.3.test_exec'}),
            C(task_tick_1, 'c', reduce_tick,'c_1'),
            C(task_tick_2, 'c', reduce_tick,'c_2'),
            C(task_tick_3, 'c', reduce_tick,'c_3'),
            C(task_tick_1, 'd', reduce_tick,'d_1'),
            C(task_tick_2, 'd', reduce_tick,'d_2'),
            C(task_tick_3, 'd', reduce_tick,'d_3'),
            C(START_TICK, 'context', reduce_tick,'context'),
            T(reduce_tick, reduceTask, {'name': 'reduce', 'path':'ps.reduce'}),
            C(reduce_tick, 'cd_list', FINAL_TICK,'cd_list'),
        )
        utils.assert_graph_equal(expected, graph)        
        self.assertTrue(os.path.exists(os.path.join(local_workdir,'testfile.pickle')))