def test_simple_task(self):
"""
A simple one-step pipeline
"""
def testpipe(x,y):
u,v=test_exec(a=x, b=y)
return u,v
graph=build_graph(testpipe)
self.assertTrue(isinstance(graph,Graph))
# there is just a single task - hence just a single node / tick
self.assertEqual(1, len(graph.get_all_ticks()))
task=graph.get_task(graph.get_all_ticks()[0])
self.assertTrue(isinstance(task,ExecTask))
self.assertEqual('test_exec',task.command)
self.assertEqual(set(['a','b']),set(task.inputnames))
self.assertEqual(set(['c','d']),set(task.outputnames))
expected = G(
C(START_TICK, 'a', 1,'a'),
C(START_TICK, 'b', 1,'b'),
C(START_TICK, 'context', 1,'context'),
T(1, task, {'name': 'test_exec', 'path':'test_exec'}),
C(1, 'c', FINAL_TICK,'test_exec.c'),
C(1, 'd', FINAL_TICK,'test_exec.d')
)
expected.set_task_property(FINAL_TICK,'aliases',
{'test_exec.c':'test_exec.c', 'test_exec.d':'test_exec.d'})
utils.assert_graph_equal(expected, graph)
def _create_graph(pkgrepos):
@pipeline(outputs=('c'))
def testpipeline(a, b):
return test_exec(a=a, b=b)
def test_exec(**kwargs):
inputnames = ("a", "b")
outputnames = ("c")
pkgsource = PackageSource("testpkg", pkgrepos)
taskprops = TaskProperties("test_exec", pkgsource)
return invoke_task(taskprops, inputnames, outputnames, **kwargs)
return build_graph(testpipeline)
def _create_graph(pkgrepos):
@pipeline(outputs=('c'))
def testpipeline(a,b):
return test_exec(a=a,b=b)
def test_exec(**kwargs):
inputnames=("a","b")
outputnames=("c")
pkgsource=PackageSource("testpkg", pkgrepos)
taskprops=TaskProperties("test_exec", pkgsource)
return invoke_task(taskprops, inputnames, outputnames, **kwargs)
return build_graph(testpipeline)
def test_nested(self):
graph = build_graph(testpipe_nested)
# there are two tasks
self.assertEqual(1, len(graph.get_all_ticks()))
nested=graph.get_task(graph.get_all_ticks()[0])
self.assertTrue(isinstance(nested,NestedGraphTask))
self.assertEqual('test_task_nested',nested.name)
task1=ExecTask('test_exec', Package('test'), ['a','b'], ['c','d'])
task2=ExecTask('test_exec', Package('test'), ['a','b'], ['c','d'])
subgraph = G(
C(START_TICK, 'a', 1,'a'),
C(START_TICK, 'b', 1,'b'),
C(START_TICK, 'context', 1,'context'),
C(START_TICK, 'context', 2,'context'),
T(1, task1, {'name': 'test_exec', 'path':'test_exec'}),
C(1, 'c', 2, 'a'),
C(1, 'd', 2, 'b'),
T(2, task2, {'name': 'test_exec2', 'path':'test_exec2'}),
C(1, 'c', FINAL_TICK,'test_exec.c'),
C(2, 'c', FINAL_TICK,'test_exec2.c'),
C(2, 'd', FINAL_TICK,'test_exec2.d')
)
subgraph.set_task_property(FINAL_TICK,'aliases',
{'test_exec.c':'test_exec.c', 'test_exec2.c':'test_exec2.c', 'test_exec2.d':'test_exec2.d'})
utils.assert_graph_equal(subgraph, nested.body_graph)
expected = G(
C(START_TICK, 'x', 1,'x'),
C(START_TICK, 'y', 1,'y'),
C(START_TICK, 'context', 1,'context'),
T(1, nested, {'name': 'test_task_nested', 'path':'test_task_nested'}),
C(1, 'test_exec.c', FINAL_TICK,'test_task_nested.test_exec.c'),
C(1, 'test_exec2.c', FINAL_TICK,'test_task_nested.test_exec2.c'),
C(1, 'test_exec2.d', FINAL_TICK,'test_task_nested.test_exec2.d')
)
expected.set_task_property(FINAL_TICK,'aliases',
{'test_task_nested.test_exec.c':'test_task_nested.test_exec.c',
'test_task_nested.test_exec2.c':'test_task_nested.test_exec2.c',
'test_task_nested.test_exec2.d':'test_task_nested.test_exec2.d'})
utils.assert_graph_equal(expected, graph)
def test_inline(self):
def testpipe(x,y):
return test_task(x=x,y=y)
def test_task(x,y):
u,v=test_exec(a=x, b=y)
w,z=test_exec(name="test_exec2", a=u, b=v)
return u,w,z
graph = build_graph(testpipe)
# there are two tasks
self.assertEqual(2, len(graph.get_all_ticks()))
ticks=sorted(graph.get_all_ticks())
task1=graph.get_task(ticks[0])
task2=graph.get_task(ticks[1])
self.assertTrue(isinstance(task1,ExecTask))
self.assertEqual('test_exec',task1.command)
self.assertEqual(set(['a','b']),set(task1.inputnames))
self.assertEqual(set(['c','d']),set(task1.outputnames))
self.assertTrue(isinstance(task2,ExecTask))
self.assertEqual('test_exec',task2.command)
self.assertEqual(set(['a','b']),set(task2.inputnames))
self.assertEqual(set(['c','d']),set(task2.outputnames))
expected = G(
C(START_TICK, 'a', 1,'a'),
C(START_TICK, 'b', 1,'b'),
C(START_TICK, 'context', 1,'context'),
C(START_TICK, 'context', 2,'context'),
T(1, task1, {'name': 'test_exec', 'path':'test_exec'}),
C(1, 'c', 2, 'a'),
C(1, 'd', 2, 'b'),
T(2, task2, {'name': 'test_exec2', 'path':'test_exec2'}),
C(1, 'c', FINAL_TICK,'test_exec.c'),
C(2, 'c', FINAL_TICK,'test_exec2.c'),
C(2, 'd', FINAL_TICK,'test_exec2.d')
)
expected.set_task_property(FINAL_TICK,'aliases',
{'test_exec.c':'test_exec.c', 'test_exec2.c':'test_exec2.c', 'test_exec2.d':'test_exec2.d'})
utils.assert_graph_equal(expected, graph)
def initialize(self):
'''
First, configures PYTHONPATH so that the pipeline specification can be parsed.
Then, loads pipeline and creates the design time graph.
Finally, it prepares all for executing the pipeline by creating a runtime context
and instantiating a graph traverser.
'''
add_to_path([self.pkgRepository, self.pipelineDir])
self.pipeline=load_pipeline_from_file(self.path_to_script)
# build the design time dataflow graph
self.dataflow = build_graph(self.pipeline)
# initialize the context
self.data[CONTEXT]=context.create_context(self)
# instantiate the traverser
self.callbacks=NodeCallbacks(self.config, self.credentials, self.pkgRepository)
self.traverser=Traverser(self.callbacks.schedule_refinement, self.callbacks.submit_task)
def build_graph_from_file(self, file_path):
pipeline_globals = {}
pipeline_locals = {}
execfile(file_path)
pipeline2 = None
pipeline_found = False
keys = locals().keys()
for k in keys:
var = locals().get(k)
globals().update({k: var})
if hasattr(var, '__call__') and hasattr(var, 'ispipeline') and not pipeline_found:
pipeline_found = True
pipeline2 = var
assert getattr(pipeline2, 'ispipeline')
assert pipeline2 is not None
pydron_graph = build_graph(pipeline2)
return pydron_graph
def initialize(self):
'''
First, configures PYTHONPATH so that the pipeline specification can be parsed.
Then, loads pipeline and creates the design time graph.
Finally, it prepares all for executing the pipeline by creating a runtime context
and instantiating a graph traverser.
'''
add_to_path([self.pkgRepository, self.pipelineDir])
self.pipeline = load_pipeline_from_file(self.path_to_script)
# build the design time dataflow graph
self.dataflow = build_graph(self.pipeline)
# initialize the context
self.data[CONTEXT] = context.create_context(self)
# instantiate the traverser
self.callbacks = NodeCallbacks(self.config, self.credentials,
self.pkgRepository)
self.traverser = Traverser(self.callbacks.schedule_refinement,
self.callbacks.submit_task)
def build_graph_from_file(self, file_path):
pipeline_globals = {}
pipeline_locals = {}
execfile(file_path)
pipeline2 = None
pipeline_found = False
keys = locals().keys()
for k in keys:
var = locals().get(k)
globals().update({k: var})
if hasattr(var, '__call__') and hasattr(
var, 'ispipeline') and not pipeline_found:
pipeline_found = True
pipeline2 = var
assert getattr(pipeline2, 'ispipeline')
assert pipeline2 is not None
pydron_graph = build_graph(pipeline2)
return pydron_graph
from vis import vis_correct_dark, vis_correct_flat, vis_split_quadrants, vis_combine_quadrants
@parallel(iterable="quadrant")
def correct(quadrant, master_dark, master_flat, control_params):
quadrant_dark_corr, hot_pixels_map = vis_correct_dark(
quadrant=quadrant, master_dark=master_dark, control_params=control_params
)
quadrant_flat_corr, dead_pixels_map = vis_correct_flat(
quadrant=quadrant_dark_corr, master_flat=master_flat, control_params=control_params
)
return quadrant_flat_corr, hot_pixels_map, dead_pixels_map
@pipeline(outputs=("corrected_exposures", "hot_pixels_map", "dead_pixels_map"))
def vis_two_steps_nested(exposures, master_dark, master_flat, control_params):
quadrants = vis_split_quadrants(exposures=exposures)
quadrants_correction_output = correct(
quadrant=quadrants, master_dark=master_dark, master_flat=master_flat, control_params=control_params
)
corrected_exposures, hot_pixels_map, dead_pixels_map = vis_combine_quadrants(inputlist=quadrants_correction_output)
return corrected_exposures, hot_pixels_map, dead_pixels_map
if __name__ == "__main__":
from euclidwf.framework.graph_builder import build_graph
print build_graph(vis_two_steps_nested)
sim_image, sim_image2 = simulate(catalog_in=catalog_in,
spectra_templates=spectra_templates,
dispersion_relation=dispersion_relation,
sensitivity_fct=sensitivity_fct,
exposure_time=exposure_time,
bkgr_value=bkgr_value)
spectra, spectra_props = extract_spectra(
image=sim_image,
catalog_in=catalog_in,
dispersion_relation=dispersion_relation,
sensitivity_fct=sensitivity_fct,
exposure_time=exposure_time)
redshifts, reliabilities = spectrum_redshift(
spectra=spectra,
galaxy_templates=galaxy_templates,
redshift_corr=redshift_corr)
catalog = redshift_catalog(redshifts=redshifts,
redshift_reliabilities=reliabilities)
return sim_image, sim_image2, spectra, spectra_props, catalog
if __name__ == '__main__':
from euclidwf.framework.graph_builder import build_graph
pydron_graph = build_graph(simulate_and_reduce_image)
print str(pydron_graph)
visualizer.visualize_graph(pydron_graph)
'''
Created on Mar 26, 2015
@author: martin.melchior
'''
from euclidwf.framework.workflow_dsl import pipeline
from vis import vis_correct_dark
@pipeline(outputs=('quadrant_dark_corr', 'hot_pixels_map'))
def vis_single_step(quadrant, master_dark, control_params):
quadrant_dark_corr, hot_pixels_map = vis_correct_dark(
quadrant=quadrant,
master_dark=master_dark,
control_params=control_params)
return quadrant_dark_corr, hot_pixels_map
if __name__ == '__main__':
from euclidwf.framework.graph_builder import build_graph
print build_graph(vis_single_step)
spectra_templates=spectra_templates,
dispersion_relation=dispersion_relation,
sensitivity_fct=sensitivity_fct,
exposure_time=exposure_time,
bkgr_value=bkgr_value)
spectra, spectra_props = extract_spectra(image=sim_image,
catalog_in=catalog_in,
dispersion_relation=dispersion_relation,
sensitivity_fct=sensitivity_fct,
exposure_time=exposure_time)
redshifts, reliabilities = spectrum_redshift(spectra=spectra,
galaxy_templates=galaxy_templates,
redshift_corr=redshift_corr)
catalog = redshift_catalog(redshifts=redshifts,
redshift_reliabilities=reliabilities)
return sim_image, sim_image2, spectra, spectra_props, catalog
if __name__ == '__main__':
from euclidwf.framework.graph_builder import build_graph
pydron_graph=build_graph(simulate_and_reduce_image)
print str(pydron_graph)
visualizer.visualize_graph(pydron_graph)
@parallel(iterable='quadrant')
def correct(quadrant, master_dark, master_flat, control_params):
quadrant_dark_corr, hot_pixels_map = vis_correct_dark(
quadrant=quadrant,
master_dark=master_dark,
control_params=control_params)
quadrant_flat_corr, dead_pixels_map = vis_correct_flat(
quadrant=quadrant_dark_corr,
master_flat=master_flat,
control_params=control_params)
return quadrant_flat_corr, hot_pixels_map, dead_pixels_map
@pipeline(outputs=('corrected_exposures', 'hot_pixels_map', 'dead_pixels_map'))
def vis_two_steps_nested(exposures, master_dark, master_flat, control_params):
quadrants = vis_split_quadrants(exposures=exposures)
quadrants_correction_output = correct(quadrant=quadrants,
master_dark=master_dark,
master_flat=master_flat,
control_params=control_params)
corrected_exposures, hot_pixels_map, dead_pixels_map = vis_combine_quadrants(
inputlist=quadrants_correction_output)
return corrected_exposures, hot_pixels_map, dead_pixels_map
if __name__ == '__main__':
from euclidwf.framework.graph_builder import build_graph
print build_graph(vis_two_steps_nested)
master_illum=master_illum,
illum_model=illum_model,
control_params=control_params)
corrected_exposures, hot_pixels_map, dead_pixels_map = vis_combine_quadrants(
inputlist=quadrants_correction_output)
detector_frames = vis_split_detectors(exposures=corrected_exposures)
vis_correct_detectors_output = vis_correct_detectors(
frame=detector_frames,
scattered_light_model=scattered_light_model,
control_params=control_params)
frame, bkgr_map, noise_map, cosmic_rays_map, weights_map = vis_combine_detectors(
inputlist=vis_correct_detectors_output)
return frame, bkgr_map, noise_map, cosmic_rays_map, weights_map, hot_pixels_map, dead_pixels_map
if __name__ == '__main__':
#for pkg,execs in workflow_dsl.get_invoked_execs(vis_pipeline).iteritems():
# print str(pkg)
# for ex in execs:
# print " "+ex
from euclidwf.framework.graph_builder import build_graph
from euclidwf.utilities import visualizer
pydron_graph = build_graph(vis_pipeline)
print str(pydron_graph)
visualizer.visualize_graph(pydron_graph)
'''
Created on Mar 26, 2015
@author: martin.melchior
'''
from euclidwf.framework.workflow_dsl import pipeline
from euclidwf.utilities import visualizer
from vis import vis_correct_dark, vis_correct_flat
@pipeline(outputs=('quadrant_flat_corr', 'hot_pixels_map', 'dead_pixels_map'))
def vis_two_steps(quadrant, master_dark, master_flat, control_params):
quadrant_dark_corr, hot_pixels_map = vis_correct_dark(
quadrant=quadrant,
master_dark=master_dark,
control_params=control_params)
quadrant_flat_corr, dead_pixels_map = vis_correct_flat(
quadrant=quadrant_dark_corr,
master_flat=master_flat,
control_params=control_params)
return quadrant_flat_corr, hot_pixels_map, dead_pixels_map
if __name__ == '__main__':
from euclidwf.framework.graph_builder import build_graph
visualizer.visualize_graph(build_graph(vis_two_steps))
master_dark=master_dark, master_flat=master_flat,
master_illum=master_illum, illum_model=illum_model,
control_params=control_params)
corrected_exposures, hot_pixels_map, dead_pixels_map = vis_combine_quadrants(inputlist=quadrants_correction_output)
detector_frames = vis_split_detectors(exposures=corrected_exposures)
vis_correct_detectors_output = vis_correct_detectors(frame=detector_frames,
scattered_light_model=scattered_light_model,
control_params=control_params)
frame, bkgr_map, noise_map, cosmic_rays_map, weights_map = vis_combine_detectors(inputlist=vis_correct_detectors_output)
return frame, bkgr_map, noise_map, cosmic_rays_map, weights_map, hot_pixels_map, dead_pixels_map
if __name__ == '__main__':
#for pkg,execs in workflow_dsl.get_invoked_execs(vis_pipeline).iteritems():
# print str(pkg)
# for ex in execs:
# print " "+ex
from euclidwf.framework.graph_builder import build_graph
from euclidwf.utilities import visualizer
pydron_graph=build_graph(vis_pipeline)
print str(pydron_graph)
visualizer.visualize_graph(pydron_graph)
'''
Created on Mar 26, 2015
@author: martin.melchior
'''
from euclidwf.framework.workflow_dsl import pipeline
from euclidwf.utilities import visualizer
from vis import vis_correct_dark, vis_correct_flat
@pipeline(outputs=('quadrant_flat_corr', 'hot_pixels_map', 'dead_pixels_map'))
def vis_two_steps(quadrant, master_dark, master_flat, control_params):
quadrant_dark_corr, hot_pixels_map = vis_correct_dark(quadrant=quadrant,
master_dark=master_dark,
control_params=control_params)
quadrant_flat_corr, dead_pixels_map = vis_correct_flat(quadrant=quadrant_dark_corr,
master_flat=master_flat,
control_params=control_params)
return quadrant_flat_corr, hot_pixels_map, dead_pixels_map
if __name__ == '__main__':
from euclidwf.framework.graph_builder import build_graph
visualizer.visualize_graph(build_graph(vis_two_steps))
'''
Created on Mar 26, 2015
@author: martin.melchior
'''
from euclidwf.framework.workflow_dsl import pipeline
from vis import vis_correct_dark
@pipeline(outputs=('quadrant_dark_corr', 'hot_pixels_map'))
def vis_single_step(quadrant, master_dark, control_params):
quadrant_dark_corr, hot_pixels_map = vis_correct_dark(quadrant=quadrant,
master_dark=master_dark,
control_params=control_params)
return quadrant_dark_corr, hot_pixels_map
if __name__ == '__main__':
from euclidwf.framework.graph_builder import build_graph
print build_graph(vis_single_step)