コード例 #1
0
    def test_simple_task(self):
        """
        A simple one-step pipeline 
        """
        def testpipe(x,y):
            u,v=test_exec(a=x, b=y)
            return u,v
        
        graph=build_graph(testpipe)
        self.assertTrue(isinstance(graph,Graph))
        
        # there is just a single task - hence just a single node / tick
        self.assertEqual(1, len(graph.get_all_ticks()))
        task=graph.get_task(graph.get_all_ticks()[0])
        self.assertTrue(isinstance(task,ExecTask))
        self.assertEqual('test_exec',task.command)
        self.assertEqual(set(['a','b']),set(task.inputnames))
        self.assertEqual(set(['c','d']),set(task.outputnames))

        expected =  G(
            C(START_TICK, 'a', 1,'a'),
            C(START_TICK, 'b', 1,'b'),
            C(START_TICK, 'context', 1,'context'),
            T(1, task, {'name': 'test_exec', 'path':'test_exec'}),
            C(1, 'c', FINAL_TICK,'test_exec.c'),
            C(1, 'd', FINAL_TICK,'test_exec.d')
        )
        expected.set_task_property(FINAL_TICK,'aliases', 
            {'test_exec.c':'test_exec.c', 'test_exec.d':'test_exec.d'})
        
        utils.assert_graph_equal(expected, graph)
コード例 #2
0
def _create_graph(pkgrepos):
    @pipeline(outputs=('c'))
    def testpipeline(a, b):
        return test_exec(a=a, b=b)

    def test_exec(**kwargs):
        inputnames = ("a", "b")
        outputnames = ("c")
        pkgsource = PackageSource("testpkg", pkgrepos)
        taskprops = TaskProperties("test_exec", pkgsource)
        return invoke_task(taskprops, inputnames, outputnames, **kwargs)

    return build_graph(testpipeline)
コード例 #3
0
def _create_graph(pkgrepos):

    @pipeline(outputs=('c'))        
    def testpipeline(a,b):
        return test_exec(a=a,b=b)
    
    def test_exec(**kwargs):
        inputnames=("a","b")
        outputnames=("c")
        pkgsource=PackageSource("testpkg", pkgrepos)
        taskprops=TaskProperties("test_exec", pkgsource)
        return invoke_task(taskprops, inputnames, outputnames, **kwargs)
    
    return build_graph(testpipeline)
コード例 #4
0
    def test_nested(self):
        
        graph = build_graph(testpipe_nested)

        # there are two tasks
        self.assertEqual(1, len(graph.get_all_ticks()))
        nested=graph.get_task(graph.get_all_ticks()[0])

        self.assertTrue(isinstance(nested,NestedGraphTask))
        self.assertEqual('test_task_nested',nested.name)

        task1=ExecTask('test_exec', Package('test'), ['a','b'], ['c','d'])
        task2=ExecTask('test_exec', Package('test'), ['a','b'], ['c','d'])

        subgraph =  G(
            C(START_TICK, 'a', 1,'a'),
            C(START_TICK, 'b', 1,'b'),
            C(START_TICK, 'context', 1,'context'),
            C(START_TICK, 'context', 2,'context'),
            T(1, task1, {'name': 'test_exec', 'path':'test_exec'}),
            C(1, 'c', 2, 'a'),
            C(1, 'd', 2, 'b'),
            T(2, task2, {'name': 'test_exec2', 'path':'test_exec2'}),
            C(1, 'c', FINAL_TICK,'test_exec.c'),
            C(2, 'c', FINAL_TICK,'test_exec2.c'),
            C(2, 'd', FINAL_TICK,'test_exec2.d')
        )
        subgraph.set_task_property(FINAL_TICK,'aliases', 
            {'test_exec.c':'test_exec.c', 'test_exec2.c':'test_exec2.c', 'test_exec2.d':'test_exec2.d'})
        utils.assert_graph_equal(subgraph, nested.body_graph)
        
        expected =  G(
            C(START_TICK, 'x', 1,'x'),
            C(START_TICK, 'y', 1,'y'),
            C(START_TICK, 'context', 1,'context'),
            T(1, nested, {'name': 'test_task_nested', 'path':'test_task_nested'}),
            C(1, 'test_exec.c', FINAL_TICK,'test_task_nested.test_exec.c'),
            C(1, 'test_exec2.c', FINAL_TICK,'test_task_nested.test_exec2.c'),
            C(1, 'test_exec2.d', FINAL_TICK,'test_task_nested.test_exec2.d')
        )
        expected.set_task_property(FINAL_TICK,'aliases',
            {'test_task_nested.test_exec.c':'test_task_nested.test_exec.c', 
             'test_task_nested.test_exec2.c':'test_task_nested.test_exec2.c',
             'test_task_nested.test_exec2.d':'test_task_nested.test_exec2.d'})

        utils.assert_graph_equal(expected, graph)
コード例 #5
0
    def test_inline(self):

        def testpipe(x,y):
            return test_task(x=x,y=y)

        def test_task(x,y):
            u,v=test_exec(a=x, b=y)
            w,z=test_exec(name="test_exec2", a=u, b=v)
            return u,w,z
        
        graph = build_graph(testpipe)

        # there are two tasks
        self.assertEqual(2, len(graph.get_all_ticks()))
        ticks=sorted(graph.get_all_ticks())        
        task1=graph.get_task(ticks[0])
        task2=graph.get_task(ticks[1])

        self.assertTrue(isinstance(task1,ExecTask))
        self.assertEqual('test_exec',task1.command)
        self.assertEqual(set(['a','b']),set(task1.inputnames))
        self.assertEqual(set(['c','d']),set(task1.outputnames))

        self.assertTrue(isinstance(task2,ExecTask))
        self.assertEqual('test_exec',task2.command)
        self.assertEqual(set(['a','b']),set(task2.inputnames))
        self.assertEqual(set(['c','d']),set(task2.outputnames))

        expected =  G(
            C(START_TICK, 'a', 1,'a'),
            C(START_TICK, 'b', 1,'b'),
            C(START_TICK, 'context', 1,'context'),
            C(START_TICK, 'context', 2,'context'),
            T(1, task1, {'name': 'test_exec', 'path':'test_exec'}),
            C(1, 'c', 2, 'a'),
            C(1, 'd', 2, 'b'),
            T(2, task2, {'name': 'test_exec2', 'path':'test_exec2'}),
            C(1, 'c', FINAL_TICK,'test_exec.c'),
            C(2, 'c', FINAL_TICK,'test_exec2.c'),
            C(2, 'd', FINAL_TICK,'test_exec2.d')
        )
        expected.set_task_property(FINAL_TICK,'aliases', 
            {'test_exec.c':'test_exec.c', 'test_exec2.c':'test_exec2.c', 'test_exec2.d':'test_exec2.d'})
        
        utils.assert_graph_equal(expected, graph)
コード例 #6
0
ファイル: runner.py プロジェクト: istoney/eucliddatasimulator
 def initialize(self):
     '''
     First, configures PYTHONPATH so that the pipeline specification can be parsed.
     Then, loads pipeline and creates the design time graph.
     Finally, it prepares all for executing the pipeline by creating a runtime context
     and instantiating a graph traverser. 
     '''
     add_to_path([self.pkgRepository, self.pipelineDir])
     self.pipeline=load_pipeline_from_file(self.path_to_script)
     
     # build the design time dataflow graph
     self.dataflow = build_graph(self.pipeline)   
     
     # initialize the context
     self.data[CONTEXT]=context.create_context(self)
     
     # instantiate the traverser
     self.callbacks=NodeCallbacks(self.config, self.credentials, self.pkgRepository)
     self.traverser=Traverser(self.callbacks.schedule_refinement, self.callbacks.submit_task)
コード例 #7
0
    def build_graph_from_file(self, file_path):
        pipeline_globals = {}
        pipeline_locals = {}
        execfile(file_path)

        pipeline2 = None
        pipeline_found = False

        keys = locals().keys()
        for k in keys:
            var = locals().get(k)
            globals().update({k: var})
            if hasattr(var, '__call__') and hasattr(var, 'ispipeline') and not pipeline_found:
                pipeline_found = True
                pipeline2 = var

        assert getattr(pipeline2, 'ispipeline')
        assert pipeline2 is not None

        pydron_graph = build_graph(pipeline2)
        return pydron_graph
コード例 #8
0
    def initialize(self):
        '''
        First, configures PYTHONPATH so that the pipeline specification can be parsed.
        Then, loads pipeline and creates the design time graph.
        Finally, it prepares all for executing the pipeline by creating a runtime context
        and instantiating a graph traverser. 
        '''
        add_to_path([self.pkgRepository, self.pipelineDir])
        self.pipeline = load_pipeline_from_file(self.path_to_script)

        # build the design time dataflow graph
        self.dataflow = build_graph(self.pipeline)

        # initialize the context
        self.data[CONTEXT] = context.create_context(self)

        # instantiate the traverser
        self.callbacks = NodeCallbacks(self.config, self.credentials,
                                       self.pkgRepository)
        self.traverser = Traverser(self.callbacks.schedule_refinement,
                                   self.callbacks.submit_task)
コード例 #9
0
    def build_graph_from_file(self, file_path):
        pipeline_globals = {}
        pipeline_locals = {}
        execfile(file_path)

        pipeline2 = None
        pipeline_found = False

        keys = locals().keys()
        for k in keys:
            var = locals().get(k)
            globals().update({k: var})
            if hasattr(var, '__call__') and hasattr(
                    var, 'ispipeline') and not pipeline_found:
                pipeline_found = True
                pipeline2 = var

        assert getattr(pipeline2, 'ispipeline')
        assert pipeline2 is not None

        pydron_graph = build_graph(pipeline2)
        return pydron_graph
コード例 #10
0
from vis import vis_correct_dark, vis_correct_flat, vis_split_quadrants, vis_combine_quadrants


@parallel(iterable="quadrant")
def correct(quadrant, master_dark, master_flat, control_params):

    quadrant_dark_corr, hot_pixels_map = vis_correct_dark(
        quadrant=quadrant, master_dark=master_dark, control_params=control_params
    )
    quadrant_flat_corr, dead_pixels_map = vis_correct_flat(
        quadrant=quadrant_dark_corr, master_flat=master_flat, control_params=control_params
    )

    return quadrant_flat_corr, hot_pixels_map, dead_pixels_map


@pipeline(outputs=("corrected_exposures", "hot_pixels_map", "dead_pixels_map"))
def vis_two_steps_nested(exposures, master_dark, master_flat, control_params):
    quadrants = vis_split_quadrants(exposures=exposures)
    quadrants_correction_output = correct(
        quadrant=quadrants, master_dark=master_dark, master_flat=master_flat, control_params=control_params
    )
    corrected_exposures, hot_pixels_map, dead_pixels_map = vis_combine_quadrants(inputlist=quadrants_correction_output)
    return corrected_exposures, hot_pixels_map, dead_pixels_map


if __name__ == "__main__":
    from euclidwf.framework.graph_builder import build_graph

    print build_graph(vis_two_steps_nested)
コード例 #11
0
    sim_image, sim_image2 = simulate(catalog_in=catalog_in,
                                     spectra_templates=spectra_templates,
                                     dispersion_relation=dispersion_relation,
                                     sensitivity_fct=sensitivity_fct,
                                     exposure_time=exposure_time,
                                     bkgr_value=bkgr_value)

    spectra, spectra_props = extract_spectra(
        image=sim_image,
        catalog_in=catalog_in,
        dispersion_relation=dispersion_relation,
        sensitivity_fct=sensitivity_fct,
        exposure_time=exposure_time)

    redshifts, reliabilities = spectrum_redshift(
        spectra=spectra,
        galaxy_templates=galaxy_templates,
        redshift_corr=redshift_corr)

    catalog = redshift_catalog(redshifts=redshifts,
                               redshift_reliabilities=reliabilities)

    return sim_image, sim_image2, spectra, spectra_props, catalog


if __name__ == '__main__':
    from euclidwf.framework.graph_builder import build_graph
    pydron_graph = build_graph(simulate_and_reduce_image)
    print str(pydron_graph)
    visualizer.visualize_graph(pydron_graph)
コード例 #12
0
'''
Created on Mar 26, 2015
@author: martin.melchior
'''

from euclidwf.framework.workflow_dsl import pipeline
from vis import vis_correct_dark


@pipeline(outputs=('quadrant_dark_corr', 'hot_pixels_map'))
def vis_single_step(quadrant, master_dark, control_params):
    quadrant_dark_corr, hot_pixels_map = vis_correct_dark(
        quadrant=quadrant,
        master_dark=master_dark,
        control_params=control_params)
    return quadrant_dark_corr, hot_pixels_map


if __name__ == '__main__':
    from euclidwf.framework.graph_builder import build_graph
    print build_graph(vis_single_step)
コード例 #13
0
                                spectra_templates=spectra_templates,
                                dispersion_relation=dispersion_relation, 
                                sensitivity_fct=sensitivity_fct,
                                exposure_time=exposure_time, 
                                bkgr_value=bkgr_value)

    spectra, spectra_props = extract_spectra(image=sim_image, 
                                             catalog_in=catalog_in,
                                             dispersion_relation=dispersion_relation, 
                                             sensitivity_fct=sensitivity_fct,
                                             exposure_time=exposure_time)

    redshifts, reliabilities = spectrum_redshift(spectra=spectra, 
                                                 galaxy_templates=galaxy_templates, 
                                                 redshift_corr=redshift_corr)

    catalog = redshift_catalog(redshifts=redshifts,
                                redshift_reliabilities=reliabilities)

    return sim_image, sim_image2, spectra, spectra_props, catalog 


if __name__ == '__main__':
    from euclidwf.framework.graph_builder import build_graph
    pydron_graph=build_graph(simulate_and_reduce_image)
    print str(pydron_graph)    
    visualizer.visualize_graph(pydron_graph)
    
            
    
コード例 #14
0
@parallel(iterable='quadrant')
def correct(quadrant, master_dark, master_flat, control_params):

    quadrant_dark_corr, hot_pixels_map = vis_correct_dark(
        quadrant=quadrant,
        master_dark=master_dark,
        control_params=control_params)
    quadrant_flat_corr, dead_pixels_map = vis_correct_flat(
        quadrant=quadrant_dark_corr,
        master_flat=master_flat,
        control_params=control_params)

    return quadrant_flat_corr, hot_pixels_map, dead_pixels_map


@pipeline(outputs=('corrected_exposures', 'hot_pixels_map', 'dead_pixels_map'))
def vis_two_steps_nested(exposures, master_dark, master_flat, control_params):
    quadrants = vis_split_quadrants(exposures=exposures)
    quadrants_correction_output = correct(quadrant=quadrants,
                                          master_dark=master_dark,
                                          master_flat=master_flat,
                                          control_params=control_params)
    corrected_exposures, hot_pixels_map, dead_pixels_map = vis_combine_quadrants(
        inputlist=quadrants_correction_output)
    return corrected_exposures, hot_pixels_map, dead_pixels_map


if __name__ == '__main__':
    from euclidwf.framework.graph_builder import build_graph
    print build_graph(vis_two_steps_nested)
コード例 #15
0
        master_illum=master_illum,
        illum_model=illum_model,
        control_params=control_params)

    corrected_exposures, hot_pixels_map, dead_pixels_map = vis_combine_quadrants(
        inputlist=quadrants_correction_output)

    detector_frames = vis_split_detectors(exposures=corrected_exposures)

    vis_correct_detectors_output = vis_correct_detectors(
        frame=detector_frames,
        scattered_light_model=scattered_light_model,
        control_params=control_params)

    frame, bkgr_map, noise_map, cosmic_rays_map, weights_map = vis_combine_detectors(
        inputlist=vis_correct_detectors_output)
    return frame, bkgr_map, noise_map, cosmic_rays_map, weights_map, hot_pixels_map, dead_pixels_map


if __name__ == '__main__':
    #for pkg,execs in workflow_dsl.get_invoked_execs(vis_pipeline).iteritems():
    #    print str(pkg)
    #    for ex in execs:
    #        print "    "+ex

    from euclidwf.framework.graph_builder import build_graph
    from euclidwf.utilities import visualizer
    pydron_graph = build_graph(vis_pipeline)
    print str(pydron_graph)
    visualizer.visualize_graph(pydron_graph)
コード例 #16
0
'''
Created on Mar 26, 2015
@author: martin.melchior
'''

from euclidwf.framework.workflow_dsl import pipeline
from euclidwf.utilities import visualizer
from vis import vis_correct_dark, vis_correct_flat


@pipeline(outputs=('quadrant_flat_corr', 'hot_pixels_map', 'dead_pixels_map'))
def vis_two_steps(quadrant, master_dark, master_flat, control_params):

    quadrant_dark_corr, hot_pixels_map = vis_correct_dark(
        quadrant=quadrant,
        master_dark=master_dark,
        control_params=control_params)
    quadrant_flat_corr, dead_pixels_map = vis_correct_flat(
        quadrant=quadrant_dark_corr,
        master_flat=master_flat,
        control_params=control_params)

    return quadrant_flat_corr, hot_pixels_map, dead_pixels_map


if __name__ == '__main__':
    from euclidwf.framework.graph_builder import build_graph
    visualizer.visualize_graph(build_graph(vis_two_steps))
コード例 #17
0
                                                        master_dark=master_dark, master_flat=master_flat, 
                                                        master_illum=master_illum, illum_model=illum_model, 
                                                        control_params=control_params)
    
    corrected_exposures, hot_pixels_map, dead_pixels_map = vis_combine_quadrants(inputlist=quadrants_correction_output)
    
    detector_frames = vis_split_detectors(exposures=corrected_exposures)
    
    vis_correct_detectors_output = vis_correct_detectors(frame=detector_frames, 
                                                          scattered_light_model=scattered_light_model, 
                                                          control_params=control_params)
    
    frame, bkgr_map, noise_map, cosmic_rays_map, weights_map = vis_combine_detectors(inputlist=vis_correct_detectors_output)
    return frame, bkgr_map, noise_map, cosmic_rays_map, weights_map, hot_pixels_map, dead_pixels_map
    



if __name__ == '__main__':
    #for pkg,execs in workflow_dsl.get_invoked_execs(vis_pipeline).iteritems():
    #    print str(pkg)
    #    for ex in execs:
    #        print "    "+ex
    
    from euclidwf.framework.graph_builder import build_graph
    from euclidwf.utilities import visualizer
    pydron_graph=build_graph(vis_pipeline)
    print str(pydron_graph)    
    visualizer.visualize_graph(pydron_graph)
    
コード例 #18
0
'''
Created on Mar 26, 2015
@author: martin.melchior
'''


from euclidwf.framework.workflow_dsl import pipeline
from euclidwf.utilities import visualizer
from vis import vis_correct_dark, vis_correct_flat

@pipeline(outputs=('quadrant_flat_corr', 'hot_pixels_map', 'dead_pixels_map'))
def vis_two_steps(quadrant, master_dark, master_flat, control_params):

    quadrant_dark_corr, hot_pixels_map = vis_correct_dark(quadrant=quadrant, 
                                                          master_dark=master_dark, 
                                                          control_params=control_params)
    quadrant_flat_corr, dead_pixels_map = vis_correct_flat(quadrant=quadrant_dark_corr, 
                                                           master_flat=master_flat, 
                                                           control_params=control_params)
    
    return quadrant_flat_corr, hot_pixels_map, dead_pixels_map



if __name__ == '__main__':
    from euclidwf.framework.graph_builder import build_graph
    visualizer.visualize_graph(build_graph(vis_two_steps))

コード例 #19
0
'''
Created on Mar 26, 2015
@author: martin.melchior
'''


from euclidwf.framework.workflow_dsl import pipeline
from vis import vis_correct_dark

@pipeline(outputs=('quadrant_dark_corr', 'hot_pixels_map'))
def vis_single_step(quadrant, master_dark, control_params):
    quadrant_dark_corr, hot_pixels_map = vis_correct_dark(quadrant=quadrant, 
                                                          master_dark=master_dark, 
                                                          control_params=control_params)    
    return quadrant_dark_corr, hot_pixels_map



if __name__ == '__main__':
    from euclidwf.framework.graph_builder import build_graph
    print build_graph(vis_single_step)