def execute_execute_command(instance, kwargs): check.inst_param(instance, "instance", DagsterInstance) config = list( check.opt_tuple_param(kwargs.get("config"), "config", default=(), of_type=str)) preset = kwargs.get("preset") mode = kwargs.get("mode") if preset and config: raise click.UsageError("Can not use --preset with --config.") tags = get_tags_from_args(kwargs) pipeline_origin = get_pipeline_python_origin_from_kwargs(kwargs) pipeline = recon_pipeline_from_origin(pipeline_origin) solid_selection = get_solid_selection_from_args(kwargs) result = do_execute_command(pipeline, instance, config, mode, tags, solid_selection, preset) if not result.success: raise click.ClickException( "Pipeline run {} resulted in failure.".format(result.run_id)) return result
def execute_list_versions_command(instance, kwargs): check.inst_param(instance, "instance", DagsterInstance) config = list( check.opt_tuple_param(kwargs.get("config"), "config", default=(), of_type=str)) preset = kwargs.get("preset") mode = kwargs.get("mode") if preset and config: raise click.UsageError("Can not use --preset with --config.") pipeline_origin = get_pipeline_python_origin_from_kwargs(kwargs) pipeline = recon_pipeline_from_origin(pipeline_origin) run_config = get_run_config_from_file_list(config) pipeline_def = pipeline.get_definition() pipeline_name = pipeline_def.name execution_plan = create_execution_plan(pipeline.get_definition(), run_config=run_config, mode=mode) step_output_versions = resolve_step_output_versions( execution_plan, environment_config=EnvironmentConfig.build(pipeline_def, run_config=run_config, mode=mode), mode_def=pipeline_def.get_mode_definition(mode), ) step_output_addresses = instance.get_addresses_for_step_output_versions({ (pipeline_name, step_output_handle): version for step_output_handle, version in step_output_versions.items() if version }) table = [] for step_output_handle, version in step_output_versions.items(): address = step_output_addresses.get( (pipeline_name, step_output_handle), "None") table.append([ "{key}.{output}".format(key=step_output_handle.step_key, output=step_output_handle.output_name), version, address, ]) table_str = tabulate(table, headers=["Step Output", "Version", "Address"], tablefmt="github") click.echo(table_str)
def execute_list_versions_command(instance, kwargs): check.inst_param(instance, "instance", DagsterInstance) config = list( check.opt_tuple_param(kwargs.get("config"), "config", default=(), of_type=str)) preset = kwargs.get("preset") mode = kwargs.get("mode") if preset and config: raise click.UsageError("Can not use --preset with --config.") pipeline_origin = get_pipeline_python_origin_from_kwargs(kwargs) pipeline = recon_pipeline_from_origin(pipeline_origin) run_config = get_run_config_from_file_list(config) environment_config = EnvironmentConfig.build(pipeline.get_definition(), run_config, mode=mode) execution_plan = ExecutionPlan.build(pipeline, environment_config) step_output_versions = resolve_step_output_versions( pipeline.get_definition(), execution_plan, environment_config) memoized_plan = resolve_memoized_execution_plan(execution_plan, pipeline.get_definition(), run_config, instance, environment_config) # the step keys that we need to execute are those which do not have their inputs populated. step_keys_not_stored = set(memoized_plan.step_keys_to_execute) table = [] for step_output_handle, version in step_output_versions.items(): table.append([ "{key}.{output}".format(key=step_output_handle.step_key, output=step_output_handle.output_name), version, "stored" if step_output_handle.step_key not in step_keys_not_stored else "to-be-recomputed", ]) table_str = tabulate( table, headers=["Step Output", "Version", "Status of Output"], tablefmt="github") click.echo(table_str)
def execute_scaffold_command(cli_args, print_fn): pipeline_origih = get_pipeline_python_origin_from_kwargs(cli_args) pipeline = recon_pipeline_from_origin(pipeline_origih) skip_non_required = cli_args["print_only_required"] do_scaffold_command(pipeline.get_definition(), print_fn, skip_non_required)