def test_repository_python_file():
python_file = file_relative_path(__file__, 'bar_repo.py')
recon_pipeline = recon_pipeline_for_cli_args(
{'pipeline_name': 'foo', 'python_file': python_file, 'fn_name': 'define_bar_repo'}
)
assert isinstance(recon_pipeline, ReconstructablePipelineFromRepo)
assert recon_pipeline.repository.pointer == FileCodePointer(python_file, 'define_bar_repo')
assert recon_pipeline.pipeline_name == 'foo'
with pytest.raises(UsageError):
recon_pipeline_for_cli_args(
{
'module_name': 'kdjfkd',
'pipeline_name': 'foo',
'python_file': file_relative_path(__file__, 'bar_repo.py'),
'fn_name': 'define_bar_repo',
'repository_yaml': None,
}
)
with pytest.raises(UsageError):
recon_pipeline_for_cli_args(
{
'module_name': None,
'pipeline_name': 'foo',
'python_file': file_relative_path(__file__, 'bar_repo.py'),
'fn_name': 'define_bar_repo',
'repository_yaml': 'kjdfkdjf',
}
)
def test_yaml_file():
recon_pipeline = recon_pipeline_for_cli_args({
'module_name':
None,
'pipeline_name':
'foobar',
'python_file':
None,
'fn_name':
None,
'repository_yaml':
file_relative_path(__file__, 'repository_module.yaml'),
})
assert isinstance(recon_pipeline, ReconstructablePipeline)
assert recon_pipeline.repository.pointer == ModuleCodePointer(
'dagster_examples.intro_tutorial.repos', 'define_repo')
with pytest.raises(UsageError):
assert recon_pipeline_for_cli_args({
'module_name': 'kdjfdk',
'pipeline_name': 'foobar'
})
with pytest.raises(UsageError):
assert recon_pipeline_for_cli_args({
'fn_name': 'kjdfkd',
'pipeline_name': 'foobar'
})
with pytest.raises(UsageError):
assert recon_pipeline_for_cli_args({
'pipeline_name': 'foobar',
'python_file': 'kjdfkdj'
})
def pipeline_launch_command(env, preset_name, mode, **kwargs):
env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str))
pipeline = recon_pipeline_for_cli_args(kwargs)
instance = DagsterInstance.get()
log_repo_stats(instance=instance, pipeline=pipeline)
if preset_name:
if env:
raise click.UsageError('Can not use --preset with --env.')
if mode:
raise click.UsageError('Can not use --preset with --mode.')
preset = pipeline.get_preset(preset_name)
else:
preset = None
run_tags = get_tags_from_args(kwargs)
# FIXME need to check the env against environment_dict
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
solid_subset=preset.solid_subset if preset else None,
environment_dict=preset.environment_dict
if preset else load_yaml_from_glob_list(env),
mode=(preset.mode if preset else mode) or 'default',
tags=run_tags,
)
return instance.launch_run(pipeline_run.run_id)
def execute_pipeline_command(output_file, solid_subset, environment_dict, mode,
**kwargs):
'''
This command might want to take a runId instead of current arguments
1. Should take optional flags to determine where to store the log output
2. Investigate python logging library to see what we can do there
'''
with ipc_write_stream(output_file) as stream:
recon_pipeline = recon_pipeline_for_cli_args(kwargs)
definition = recon_pipeline.get_definition()
if solid_subset:
definition = definition.subset_for_execution(
solid_subset.split(","))
# This can raise a ValueError, but this is caught by the broad-except
# and the exception is serialized as a SerializableErrorInfo
environment_dict = json.loads(environment_dict)
instance = DagsterInstance.get()
for event in execute_pipeline_iterator(
definition,
environment_dict=environment_dict,
mode=mode,
instance=instance,
):
stream.send(event)
def test_load_from_pipeline_file():
recon_pipeline = recon_pipeline_for_cli_args(
{'fn_name': 'define_foo_pipeline', 'python_file': __file__}
)
pipeline_def = recon_pipeline.get_definition()
assert isinstance(pipeline_def, PipelineDefinition)
assert pipeline_def.name == 'foo'
def test_load_from_repository_file():
recon_pipeline = recon_pipeline_for_cli_args(
{'pipeline_name': 'foo', 'python_file': __file__, 'fn_name': 'define_bar_repo'}
)
pipeline_def = recon_pipeline.get_definition()
assert isinstance(pipeline_def, PipelineDefinition)
assert pipeline_def.name == 'foo'
def pipeline_snapshot_command(output_file, solid_subset, **kwargs):
recon_pipeline = recon_pipeline_for_cli_args(kwargs)
definition = recon_pipeline.get_definition()
if solid_subset:
definition = definition.subset_for_execution(solid_subset.split(","))
ipc_write_unary_response(output_file, external_pipeline_data_from_def(definition))
def pipeline_subset_snapshot_command(output_file, solid_selection, **kwargs):
recon_pipeline = recon_pipeline_for_cli_args(kwargs)
if solid_selection:
solid_selection = json.loads(solid_selection)
ipc_write_unary_response(
output_file,
get_external_pipeline_subset_result(recon_pipeline, solid_selection))
def test_load_from_pipeline_module():
recon_pipeline = recon_pipeline_for_cli_args(
{'module_name': 'dagster_examples.intro_tutorial.repos', 'fn_name': 'hello_cereal_pipeline'}
)
pipeline_def = recon_pipeline.get_definition()
assert isinstance(pipeline_def, PipelineDefinition)
assert pipeline_def.name == 'hello_cereal_pipeline'
def pipeline_launch_command(env, preset_name, mode, **kwargs):
env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str))
pipeline = recon_pipeline_for_cli_args(kwargs)
instance = DagsterInstance.get()
log_repo_stats(instance=instance, pipeline=pipeline, source='pipeline_launch_command')
if preset_name:
if env:
raise click.UsageError('Can not use --preset with --env.')
if mode:
raise click.UsageError('Can not use --preset with --mode.')
preset = pipeline.get_preset(preset_name)
else:
preset = None
run_tags = get_tags_from_args(kwargs)
solid_selection = get_solid_selection_from_args(kwargs)
if preset and preset.solid_selection is not None:
check.invariant(
solid_selection is None or solid_selection == preset.solid_selection,
'The solid_selection set in preset \'{preset}\', {preset_subset}, does not agree with '
'the `solid_selection` argument: {solid_selection}'.format(
preset=preset,
preset_subset=preset.solid_selection,
solid_selection=solid_selection,
),
)
solid_selection = preset.solid_selection
# generate pipeline subset from the given solid_selection
if solid_selection:
pipeline = pipeline.subset_for_execution(solid_selection)
# FIXME need to check the env against environment_dict
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
solid_selection=solid_selection,
solids_to_execute=pipeline.solids_to_execute,
environment_dict=preset.environment_dict if preset else load_yaml_from_glob_list(env),
mode=(preset.mode if preset else mode) or 'default',
tags=run_tags,
)
recon_repo = pipeline.get_reconstructable_repository()
repo_location = InProcessRepositoryLocation(recon_repo)
external_pipeline = (
repo_location.get_repository(recon_repo.get_definition().name).get_full_external_pipeline(
pipeline.get_definition().name
),
)
return instance.launch_run(pipeline_run.run_id, external_pipeline)
def pipeline_snapshot_command(solid_subset, **kwargs):
recon_pipeline = recon_pipeline_for_cli_args(kwargs)
definition = recon_pipeline.get_definition()
if solid_subset:
definition = definition.subset_for_execution(solid_subset.split(","))
active_data = external_pipeline_data_from_def(definition)
click.echo(serialize_dagster_namedtuple(active_data))
def execute_execute_command_with_preset(preset_name, cli_args, _mode):
pipeline = recon_pipeline_for_cli_args(cli_args)
tags = get_tags_from_args(cli_args)
return execute_pipeline(
pipeline,
preset=preset_name,
instance=DagsterInstance.get(),
raise_on_error=False,
tags=tags,
)
def test_pipeline_module():
recon_pipeline = recon_pipeline_for_cli_args({
'module_name': 'dagster',
'fn_name': 'define_pipeline',
'pipeline_name': None,
'python_file': None,
'repository_yaml': None,
})
assert isinstance(recon_pipeline, ReconstructablePipeline)
assert recon_pipeline.pointer == ModuleCodePointer('dagster',
'define_pipeline')
def test_loader_from_default_repository_file_yaml():
recon_pipeline = recon_pipeline_for_cli_args({
'pipeline_name':
'foo',
'repository_yaml':
file_relative_path(__file__, 'repository_file.yaml'),
})
pipeline_def = recon_pipeline.get_definition()
assert isinstance(pipeline_def, PipelineDefinition)
assert pipeline_def.name == 'foo'
def test_pipeline_python_file():
python_file = file_relative_path(__file__, 'foo_pipeline.py')
recon_pipeline = recon_pipeline_for_cli_args({
'module_name': None,
'fn_name': 'define_pipeline',
'pipeline_name': None,
'python_file': python_file,
'repository_yaml': None,
})
assert isinstance(recon_pipeline, ReconstructablePipeline)
assert recon_pipeline.repository.pointer == FileCodePointer(
python_file, 'define_pipeline')
def test_pipeline_module():
from dagster.core.errors import DagsterInvariantViolationError
with pytest.raises(DagsterInvariantViolationError):
# can't find it so it pukes
recon_pipeline = recon_pipeline_for_cli_args({
'module_name': 'dagster',
'fn_name': 'define_pipeline',
'pipeline_name': None,
'python_file': None,
'repository_yaml': None,
})
assert isinstance(recon_pipeline, ReconstructablePipeline)
assert recon_pipeline.repository.pointer == ModuleCodePointer(
'dagster', 'define_pipeline')
def get_pipeline_snapshot_from_cli_args(cli_args):
_cli_load_invariant(cli_args.get('pipeline_name') is not None)
if cli_args.get('image'):
_cli_load_invariant(
is_module_available('docker'),
msg=
'--image is not supported without dagster[docker] or the Python package docker installed.',
)
external_repo = get_external_repository_from_image(
cli_args.get('image'))
return external_repo.get_external_pipeline(
cli_args.get('pipeline_name')[0]).pipeline_snapshot
else:
pipeline_def = recon_pipeline_for_cli_args(cli_args).get_definition()
return PipelineSnapshot.from_pipeline_def(pipeline_def)
def execute_pipeline_command(output_file, solid_subset, environment_dict,
instance_ref, mode, **kwargs):
'''
This command might want to take a runId instead of current arguments
1. Should take optional flags to determine where to store the log output
2. Investigate python logging library to see what we can do there
'''
with ipc_write_stream(output_file) as stream:
recon_pipeline = recon_pipeline_for_cli_args(kwargs)
definition = recon_pipeline.get_definition()
if solid_subset:
definition = definition.subset_for_execution(
solid_subset.split(","))
# This can raise a ValueError, but this is caught by the broad-except
# and the exception is serialized as a SerializableErrorInfo
environment_dict = json.loads(environment_dict)
try:
instance = DagsterInstance.from_ref(
deserialize_json_to_dagster_namedtuple(instance_ref))
except: # pylint: disable=bare-except
stream.send_error(
sys.exc_info(),
message='Could not deserialize {json_string}'.format(
json_string=instance_ref),
)
return
for event in execute_pipeline_iterator(
definition,
environment_dict=environment_dict,
mode=mode,
instance=instance,
):
stream.send(event)
def execution_plan_snapshot_command(output_file, solid_selection,
environment_dict, mode,
step_keys_to_execute, snapshot_id,
**kwargs):
recon_pipeline = recon_pipeline_for_cli_args(kwargs)
environment_dict = json.loads(environment_dict)
if step_keys_to_execute:
step_keys_to_execute = json.loads(step_keys_to_execute)
if solid_selection:
solid_selection = json.loads(solid_selection)
recon_pipeline = recon_pipeline.subset_for_execution(solid_selection)
execution_plan_snapshot = snapshot_from_execution_plan(
create_execution_plan(
pipeline=recon_pipeline,
environment_dict=environment_dict,
mode=mode,
step_keys_to_execute=step_keys_to_execute,
),
snapshot_id,
)
ipc_write_unary_response(output_file, execution_plan_snapshot)
def get_pipeline_snapshot_from_cli_args(cli_args):
_cli_load_invariant(cli_args.get('pipeline_name') is not None)
pipeline_def = recon_pipeline_for_cli_args(cli_args).get_definition()
return PipelineSnapshot.from_pipeline_def(pipeline_def)
def execute_execute_command(env, cli_args, mode=None, tags=None):
pipeline = recon_pipeline_for_cli_args(cli_args)
solid_selection = get_solid_selection_from_args(cli_args)
return do_execute_command(pipeline, env, mode, tags, solid_selection)
def execute_scaffold_command(cli_args, print_fn):
pipeline = recon_pipeline_for_cli_args(cli_args)
skip_non_required = cli_args['print_only_required']
do_scaffold_command(pipeline.get_definition(), print_fn, skip_non_required)
def execute_execute_command(env, cli_args, mode=None, tags=None):
pipeline = recon_pipeline_for_cli_args(cli_args)
return do_execute_command(pipeline, env, mode, tags)
def pipeline_graphviz_command(only_solids, **kwargs):
pipeline = recon_pipeline_for_cli_args(kwargs)
build_graphviz_graph(pipeline, only_solids).view(cleanup=True)
