def test_composite_execute():
result = execute_on_dask(
ExecutionTargetHandle.for_pipeline_python_file(__file__, 'dask_composite_pipeline'),
env_config={'storage': {'filesystem': {}}},
dask_config=DaskConfig(timeout=30),
)
assert result.success
def test_execute_on_dask():
result = execute_on_dask(
ExecutionTargetHandle.for_pipeline_python_file(__file__, 'dask_engine_pipeline'),
env_config={'storage': {'filesystem': {}}},
dask_config=DaskConfig(timeout=30),
)
assert result.result_for_solid('simple').output_value() == 1
def test_execute_on_dask():
result = execute_on_dask(
ExecutionTargetHandle.for_pipeline_fn(define_dask_test_pipeline),
env_config={'storage': {
'filesystem': {}
}},
dask_config=DaskConfig(timeout=30),
)
assert result.result_for_solid('simple').result_value() == 1
def test_execute_on_dask():
'''This test is flaky on py27, I believe because of
https://github.com/dask/distributed/issues/2446. For now, we just retry a couple times...
'''
result = execute_on_dask(
ExecutionTargetHandle.for_pipeline_fn(define_dask_test_pipeline),
env_config={'storage': {
'filesystem': {}
}},
run_config=RunConfig(storage_mode=RunStorageMode.FILESYSTEM),
dask_config=DaskConfig(timeout=30),
)
assert result.result_for_solid('simple').transformed_value() == 1
def test_pandas_dask():
environment_dict = {
'solids': {
'pandas_solid': {
'inputs': {'df': {'csv': {'path': file_relative_path(__file__, 'ex.csv')}}}
}
}
}
result = execute_on_dask(
ExecutionTargetHandle.for_pipeline_python_file(__file__, pandas_pipeline.name),
env_config={'storage': {'filesystem': {}}, **environment_dict},
dask_config=DaskConfig(timeout=30),
)
assert result.success
def test_dask_cluster():
result = execute_on_dask(
ExecutionTargetHandle.for_pipeline_module(
'dagster_examples.toys.hammer', 'define_hammer_pipeline'),
env_config={
'storage': {
's3': {
's3_bucket': 'dagster-airflow-scratch'
}
}
},
run_config=RunConfig(storage_mode=RunStorageMode.S3),
dask_config=DaskConfig(address='%s:8786' % os.getenv('DASK_ADDRESS')),
)
assert result.success
assert result.result_for_solid('total').transformed_value() == 4
SolidInstance('hammer', alias='hammer_2'): {
'chase_duration': DependencyDefinition('giver', 'out_2')
},
SolidInstance('hammer', alias='hammer_3'): {
'chase_duration': DependencyDefinition('giver', 'out_3')
},
SolidInstance('hammer', alias='hammer_4'): {
'chase_duration': DependencyDefinition('giver', 'out_4')
},
SolidInstance('total'): {
'in_1': DependencyDefinition('hammer_1', 'total'),
'in_2': DependencyDefinition('hammer_2', 'total'),
'in_3': DependencyDefinition('hammer_3', 'total'),
'in_4': DependencyDefinition('hammer_4', 'total'),
},
},
mode_definitions=[ModeDefinition()],
)
if __name__ == '__main__':
result = execute_on_dask(
ExecutionTargetHandle.for_pipeline_fn(define_hammer_pipeline),
env_config={'storage': {
'filesystem': {}
}},
run_config=RunConfig(storage_mode=RunStorageMode.FILESYSTEM),
)
print('Total Hammer Time: ',
result.result_for_solid('total').transformed_value())
},
SolidInvocation('hammer', alias='hammer_3'): {
'chase_duration': DependencyDefinition('giver', 'out_3')
},
SolidInvocation('hammer', alias='hammer_4'): {
'chase_duration': DependencyDefinition('giver', 'out_4')
},
SolidInvocation('total'): {
'in_1': DependencyDefinition('hammer_1', 'total'),
'in_2': DependencyDefinition('hammer_2', 'total'),
'in_3': DependencyDefinition('hammer_3', 'total'),
'in_4': DependencyDefinition('hammer_4', 'total'),
},
},
mode_definitions=[
ModeDefinition(resources={'s3': s3_resource},
system_storage_defs=s3_plus_default_storage_defs)
],
)
if __name__ == '__main__':
result = execute_on_dask(
ExecutionTargetHandle.for_pipeline_fn(define_hammer_pipeline),
env_config={'storage': {
'filesystem': {}
}},
)
print('Total Hammer Time: ',
result.result_for_solid('total').result_value())