def execute_execute_command(instance, kwargs):
check.inst_param(instance, "instance", DagsterInstance)
config = list(
check.opt_tuple_param(kwargs.get("config"),
"config",
default=(),
of_type=str))
preset = kwargs.get("preset")
mode = kwargs.get("mode")
if preset and config:
raise click.UsageError("Can not use --preset with --config.")
tags = get_tags_from_args(kwargs)
pipeline_origin = get_pipeline_python_origin_from_kwargs(kwargs)
pipeline = recon_pipeline_from_origin(pipeline_origin)
solid_selection = get_solid_selection_from_args(kwargs)
result = do_execute_command(pipeline, instance, config, mode, tags,
solid_selection, preset)
if not result.success:
raise click.ClickException(
"Pipeline run {} resulted in failure.".format(result.run_id))
return result
def execute_list_versions_command(instance, kwargs):
check.inst_param(instance, "instance", DagsterInstance)
config = list(
check.opt_tuple_param(kwargs.get("config"),
"config",
default=(),
of_type=str))
preset = kwargs.get("preset")
mode = kwargs.get("mode")
if preset and config:
raise click.UsageError("Can not use --preset with --config.")
pipeline_origin = get_pipeline_python_origin_from_kwargs(kwargs)
pipeline = recon_pipeline_from_origin(pipeline_origin)
run_config = get_run_config_from_file_list(config)
pipeline_def = pipeline.get_definition()
pipeline_name = pipeline_def.name
execution_plan = create_execution_plan(pipeline.get_definition(),
run_config=run_config,
mode=mode)
step_output_versions = resolve_step_output_versions(
execution_plan,
environment_config=EnvironmentConfig.build(pipeline_def,
run_config=run_config,
mode=mode),
mode_def=pipeline_def.get_mode_definition(mode),
)
step_output_addresses = instance.get_addresses_for_step_output_versions({
(pipeline_name, step_output_handle): version
for step_output_handle, version in step_output_versions.items()
if version
})
table = []
for step_output_handle, version in step_output_versions.items():
address = step_output_addresses.get(
(pipeline_name, step_output_handle), "None")
table.append([
"{key}.{output}".format(key=step_output_handle.step_key,
output=step_output_handle.output_name),
version,
address,
])
table_str = tabulate(table,
headers=["Step Output", "Version", "Address"],
tablefmt="github")
click.echo(table_str)
def execute_list_versions_command(instance, kwargs):
check.inst_param(instance, "instance", DagsterInstance)
config = list(
check.opt_tuple_param(kwargs.get("config"),
"config",
default=(),
of_type=str))
preset = kwargs.get("preset")
mode = kwargs.get("mode")
if preset and config:
raise click.UsageError("Can not use --preset with --config.")
pipeline_origin = get_pipeline_python_origin_from_kwargs(kwargs)
pipeline = recon_pipeline_from_origin(pipeline_origin)
run_config = get_run_config_from_file_list(config)
environment_config = EnvironmentConfig.build(pipeline.get_definition(),
run_config,
mode=mode)
execution_plan = ExecutionPlan.build(pipeline, environment_config)
step_output_versions = resolve_step_output_versions(
pipeline.get_definition(), execution_plan, environment_config)
memoized_plan = resolve_memoized_execution_plan(execution_plan,
pipeline.get_definition(),
run_config, instance,
environment_config)
# the step keys that we need to execute are those which do not have their inputs populated.
step_keys_not_stored = set(memoized_plan.step_keys_to_execute)
table = []
for step_output_handle, version in step_output_versions.items():
table.append([
"{key}.{output}".format(key=step_output_handle.step_key,
output=step_output_handle.output_name),
version,
"stored" if step_output_handle.step_key not in step_keys_not_stored
else "to-be-recomputed",
])
table_str = tabulate(
table,
headers=["Step Output", "Version", "Status of Output"],
tablefmt="github")
click.echo(table_str)
def execute_scaffold_command(cli_args, print_fn):
pipeline_origih = get_pipeline_python_origin_from_kwargs(cli_args)
pipeline = recon_pipeline_from_origin(pipeline_origih)
skip_non_required = cli_args["print_only_required"]
do_scaffold_command(pipeline.get_definition(), print_fn, skip_non_required)
