def test_engine_error():
with pytest.raises(DagsterSubprocessError):
with seven.TemporaryDirectory() as tempdir:
storage = os.path.join(tempdir, 'flakey_storage')
execute_pipeline(
ReconstructablePipeline(
FileCodePointer(__file__, 'engine_error')),
environment_dict={
'storage': {
'filesystem': {
'config': {
'base_dir': storage
}
}
},
'execution': {
'celery': {
'config': {
'config_source': {
'task_always_eager': True
}
}
}
},
'solids': {
'destroy': {
'config': storage
}
},
},
instance=DagsterInstance.local_temp(tempdir=tempdir),
)
def test_pandas_dask():
environment_dict = {
'solids': {
'pandas_solid': {
'inputs': {
'df': {
'csv': {
'path': file_relative_path(__file__, 'ex.csv')
}
}
}
}
}
}
result = execute_pipeline(
ReconstructablePipeline(FileCodePointer(__file__,
pandas_pipeline.name)),
environment_dict={
'storage': {
'filesystem': {}
},
'execution': {
'dask': {
'config': {
'timeout': 30
}
}
},
**environment_dict,
},
instance=DagsterInstance.local_temp(),
)
assert result.success
def execute_eagerly_on_celery(pipeline_name, instance=None, subset=None):
with seven.TemporaryDirectory() as tempdir:
instance = instance or DagsterInstance.local_temp(tempdir=tempdir)
result = execute_pipeline(
ReconstructablePipeline(FileCodePointer(
__file__, pipeline_name)).subset_for_execution(subset),
environment_dict={
'storage': {
'filesystem': {
'config': {
'base_dir': tempdir
}
}
},
'execution': {
'celery': {
'config': {
'config_source': {
'task_always_eager': True
}
}
}
},
},
instance=instance,
)
yield result
def test_pipeline_python_file():
python_file = file_relative_path(__file__, 'foo_pipeline.py')
recon_pipeline = recon_pipeline_for_cli_args({
'module_name': None,
'fn_name': 'define_pipeline',
'pipeline_name': None,
'python_file': python_file,
'repository_yaml': None,
})
assert isinstance(recon_pipeline, ReconstructablePipeline)
assert recon_pipeline.pointer == FileCodePointer(python_file,
'define_pipeline')
def execute_pipeline_on_celery(pipeline_name):
with seven.TemporaryDirectory() as tempdir:
pipeline_def = ReconstructablePipeline(FileCodePointer(__file__, pipeline_name))
instance = DagsterInstance.local_temp(tempdir=tempdir)
result = execute_pipeline(
pipeline_def,
environment_dict={
'storage': {'filesystem': {'config': {'base_dir': tempdir}}},
'execution': {'celery': {}},
},
instance=instance,
)
yield result
def execute_pipeline_on_celery(tempdir, pipeline_name, tags=None):
pipe = ReconstructablePipeline(FileCodePointer(__file__, pipeline_name))
instance = DagsterInstance.local_temp(tempdir=tempdir)
return execute_pipeline(
pipe,
environment_dict={
'storage': {
'filesystem': {
'config': {
'base_dir': tempdir
}
}
},
'execution': {
'celery': {}
},
},
instance=instance,
tags=tags,
)
def test_repository_python_file():
python_file = file_relative_path(__file__, 'bar_repo.py')
recon_pipeline = recon_pipeline_for_cli_args({
'pipeline_name': 'foo',
'python_file': python_file,
'fn_name': 'define_bar_repo'
})
assert isinstance(recon_pipeline, ReconstructablePipelineFromRepo)
assert recon_pipeline.repository.pointer == FileCodePointer(
python_file, 'define_bar_repo')
assert recon_pipeline.pipeline_name == 'foo'
with pytest.raises(UsageError):
recon_pipeline_for_cli_args({
'module_name':
'kdjfkd',
'pipeline_name':
'foo',
'python_file':
file_relative_path(__file__, 'bar_repo.py'),
'fn_name':
'define_bar_repo',
'repository_yaml':
None,
})
with pytest.raises(UsageError):
recon_pipeline_for_cli_args({
'module_name':
None,
'pipeline_name':
'foo',
'python_file':
file_relative_path(__file__, 'bar_repo.py'),
'fn_name':
'define_bar_repo',
'repository_yaml':
'kjdfkdjf',
})
def execute_eagerly_on_celery(tempdir, pipeline_name, tags=None):
return execute_pipeline(
ReconstructablePipeline(FileCodePointer(__file__, pipeline_name)),
environment_dict={
'storage': {
'filesystem': {
'config': {
'base_dir': tempdir
}
}
},
'execution': {
'celery': {
'config': {
'config_source': {
'task_always_eager': True
}
}
}
},
},
instance=DagsterInstance.local_temp(tempdir=tempdir),
tags=tags,
)