def execute_scaffold_command(cli_args, print_fn):
external_pipeline = get_external_pipeline_from_kwargs(cli_args, DagsterInstance.get())
# We should move this to use external pipeline
# https://github.com/dagster-io/dagster/issues/2556
pipeline = recon_pipeline_from_origin(external_pipeline.get_origin())
skip_non_required = cli_args['print_only_required']
do_scaffold_command(pipeline.get_definition(), print_fn, skip_non_required)
def execute_execute_command(env, cli_args, mode=None, tags=None):
external_pipeline = get_external_pipeline_from_kwargs(cli_args)
# We should move this to use external pipeline
# https://github.com/dagster-io/dagster/issues/2556
pipeline = recon_pipeline_from_origin(external_pipeline.handle.get_origin())
solid_selection = get_solid_selection_from_args(cli_args)
return do_execute_command(pipeline, env, mode, tags, solid_selection)
def execute_print_command(verbose, cli_args, print_fn):
with get_external_pipeline_from_kwargs(cli_args) as external_pipeline:
pipeline_snapshot = external_pipeline.pipeline_snapshot
if verbose:
print_pipeline(pipeline_snapshot, print_fn=print_fn)
else:
print_solids(pipeline_snapshot, print_fn=print_fn)
def execute_print_command(verbose, cli_args, print_fn):
external_pipeline = get_external_pipeline_from_kwargs(cli_args, DagsterInstance.get())
pipeline_snapshot = external_pipeline.pipeline_snapshot
if verbose:
print_pipeline(pipeline_snapshot, print_fn=print_fn)
else:
print_solids(pipeline_snapshot, print_fn=print_fn)
def execute_print_command(instance, verbose, cli_args, print_fn):
with get_external_pipeline_from_kwargs(
instance, version=dagster_version, kwargs=cli_args
) as external_pipeline:
pipeline_snapshot = external_pipeline.pipeline_snapshot
if verbose:
print_pipeline(pipeline_snapshot, print_fn=print_fn)
else:
print_solids(pipeline_snapshot, print_fn=print_fn)
def execute_execute_command_with_preset(preset_name, cli_args, instance, _mode):
external_pipeline = get_external_pipeline_from_kwargs(cli_args, instance)
# We should move this to use external pipeline
# https://github.com/dagster-io/dagster/issues/2556
pipeline = recon_pipeline_from_origin(external_pipeline.handle.get_origin())
tags = get_tags_from_args(cli_args)
solid_selection = get_solid_selection_from_args(cli_args)
return execute_pipeline(
pipeline,
preset=preset_name,
instance=DagsterInstance.get(),
raise_on_error=False,
tags=tags,
solid_selection=solid_selection,
)
def execute_execute_command(env_file_list,
cli_args,
instance=None,
mode=None,
tags=None):
check.opt_list_param(env_file_list, 'env_file_list', of_type=str)
check.opt_inst_param(instance, 'instance', DagsterInstance)
instance = instance if instance else DagsterInstance.get()
external_pipeline = get_external_pipeline_from_kwargs(cli_args, instance)
# We should move this to use external pipeline
# https://github.com/dagster-io/dagster/issues/2556
pipeline = recon_pipeline_from_origin(
external_pipeline.handle.get_origin())
solid_selection = get_solid_selection_from_args(cli_args)
return do_execute_command(pipeline, instance, env_file_list, mode, tags,
solid_selection)
def command(**kwargs):
capture_result[
'external_pipeline'] = get_external_pipeline_from_kwargs(
kwargs, DagsterInstance.ephemeral())
def execute_backfill_command(cli_args, print_fn, instance=None):
instance = instance or DagsterInstance.get()
external_pipeline = get_external_pipeline_from_kwargs(cli_args, instance)
external_repository = get_external_repository_from_kwargs(cli_args, instance)
# We should move this to use external repository
# https://github.com/dagster-io/dagster/issues/2556
recon_repo = recon_repo_from_external_repo(external_repository)
repo_def = recon_repo.get_definition()
noprompt = cli_args.get('noprompt')
pipeline_def = repo_def.get_pipeline(external_pipeline.name)
# Resolve partition set
all_partition_sets = repo_def.partition_set_defs + [
schedule_def.get_partition_set()
for schedule_def in repo_def.schedule_defs
if isinstance(schedule_def, PartitionScheduleDefinition)
]
pipeline_partition_sets = [
x for x in all_partition_sets if x.pipeline_name == pipeline_def.name
]
if not pipeline_partition_sets:
raise click.UsageError(
'No partition sets found for pipeline `{}`'.format(pipeline_def.name)
)
partition_set_name = cli_args.get('partition_set')
if not partition_set_name:
if len(pipeline_partition_sets) == 1:
partition_set_name = pipeline_partition_sets[0].name
elif noprompt:
raise click.UsageError('No partition set specified (see option `--partition-set`)')
else:
partition_set_name = click.prompt(
'Select a partition set to use for backfill: {}'.format(
', '.join(x.name for x in pipeline_partition_sets)
)
)
partition_set = next((x for x in pipeline_partition_sets if x.name == partition_set_name), None)
if not partition_set:
raise click.UsageError('No partition set found named `{}`'.format(partition_set_name))
# Resolve partitions to backfill
partitions = gen_partitions_from_args(partition_set, cli_args)
# Print backfill info
print_fn('\n Pipeline: {}'.format(pipeline_def.name))
print_fn('Partition set: {}'.format(partition_set.name))
print_fn(' Partitions: {}\n'.format(print_partition_format(partitions, indent_level=15)))
# Confirm and launch
if noprompt or click.confirm(
'Do you want to proceed with the backfill ({} partitions)?'.format(len(partitions))
):
print_fn('Launching runs... ')
backfill_id = make_new_backfill_id()
run_tags = merge_dicts(
PipelineRun.tags_for_backfill_id(backfill_id), get_tags_from_args(cli_args),
)
for partition in partitions:
run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
mode=partition_set.mode,
solids_to_execute=frozenset(partition_set.solid_selection)
if partition_set and partition_set.solid_selection
else None,
run_config=partition_set.run_config_for_partition(partition),
tags=merge_dicts(partition_set.tags_for_partition(partition), run_tags),
)
instance.launch_run(run.run_id, external_pipeline)
# Remove once we can handle synchronous execution... currently limited by sqlite
time.sleep(0.1)
print_fn('Launched backfill job `{}`'.format(backfill_id))
else:
print_fn(' Aborted!')
def _logged_pipeline_launch_command(config, preset_name, mode, instance, kwargs):
check.inst_param(instance, 'instance', DagsterInstance)
env = (
canonicalize_backcompat_args(
(config if config else None),
'--config',
(kwargs.get('env') if kwargs.get('env') else None),
'--env',
'0.9.0',
stacklevel=2, # this stacklevel can point the warning to this line
)
or tuple() # back to default empty tuple
)
env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str))
instance = DagsterInstance.get()
external_pipeline = get_external_pipeline_from_kwargs(kwargs, instance)
# We should move this to use external pipeline
# https://github.com/dagster-io/dagster/issues/2556
pipeline = recon_pipeline_from_origin(external_pipeline.get_origin())
log_repo_stats(instance=instance, pipeline=pipeline, source='pipeline_launch_command')
if preset_name:
if env:
raise click.UsageError('Can not use --preset with --config.')
if mode:
raise click.UsageError('Can not use --preset with --mode.')
preset = pipeline.get_definition().get_preset(preset_name)
else:
preset = None
run_tags = get_tags_from_args(kwargs)
solid_selection = get_solid_selection_from_args(kwargs)
if preset and preset.solid_selection is not None:
check.invariant(
solid_selection is None or solid_selection == preset.solid_selection,
'The solid_selection set in preset \'{preset}\', {preset_subset}, does not agree with '
'the `solid_selection` argument: {solid_selection}'.format(
preset=preset,
preset_subset=preset.solid_selection,
solid_selection=solid_selection,
),
)
solid_selection = preset.solid_selection
# generate pipeline subset from the given solid_selection
if solid_selection:
pipeline = pipeline.subset_for_execution(solid_selection)
# FIXME need to check the env against run_config
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
solid_selection=solid_selection,
solids_to_execute=pipeline.solids_to_execute,
run_config=preset.run_config if preset else load_yaml_from_glob_list(env),
mode=(preset.mode if preset else mode) or 'default',
tags=run_tags,
)
recon_repo = pipeline.get_reconstructable_repository()
repo_location = InProcessRepositoryLocation(recon_repo)
external_pipeline = (
repo_location.get_repository(recon_repo.get_definition().name).get_full_external_pipeline(
pipeline.get_definition().name
),
)
return instance.launch_run(pipeline_run.run_id, external_pipeline)
def command(**kwargs):
with get_external_pipeline_from_kwargs(DagsterInstance.get(), "",
kwargs) as external_pipeline:
capture_result["external_pipeline"] = external_pipeline
def command(**kwargs):
with get_external_pipeline_from_kwargs(
kwargs, DagsterInstance.ephemeral()) as external_pipeline:
capture_result["external_pipeline"] = external_pipeline
def command(**kwargs):
with get_external_pipeline_from_kwargs(kwargs) as external_pipeline:
capture_result["external_pipeline"] = external_pipeline
def command(**kwargs):
capture_result['external_pipeline'] = get_external_pipeline_from_kwargs(kwargs)
