def execute_scaffold_command(cli_args, print_fn): external_pipeline = get_external_pipeline_from_kwargs(cli_args, DagsterInstance.get()) # We should move this to use external pipeline # https://github.com/dagster-io/dagster/issues/2556 pipeline = recon_pipeline_from_origin(external_pipeline.get_origin()) skip_non_required = cli_args['print_only_required'] do_scaffold_command(pipeline.get_definition(), print_fn, skip_non_required)
def execute_execute_command(env, cli_args, mode=None, tags=None): external_pipeline = get_external_pipeline_from_kwargs(cli_args) # We should move this to use external pipeline # https://github.com/dagster-io/dagster/issues/2556 pipeline = recon_pipeline_from_origin(external_pipeline.handle.get_origin()) solid_selection = get_solid_selection_from_args(cli_args) return do_execute_command(pipeline, env, mode, tags, solid_selection)
def execute_print_command(verbose, cli_args, print_fn): with get_external_pipeline_from_kwargs(cli_args) as external_pipeline: pipeline_snapshot = external_pipeline.pipeline_snapshot if verbose: print_pipeline(pipeline_snapshot, print_fn=print_fn) else: print_solids(pipeline_snapshot, print_fn=print_fn)
def execute_print_command(verbose, cli_args, print_fn): external_pipeline = get_external_pipeline_from_kwargs(cli_args, DagsterInstance.get()) pipeline_snapshot = external_pipeline.pipeline_snapshot if verbose: print_pipeline(pipeline_snapshot, print_fn=print_fn) else: print_solids(pipeline_snapshot, print_fn=print_fn)
def execute_print_command(instance, verbose, cli_args, print_fn): with get_external_pipeline_from_kwargs( instance, version=dagster_version, kwargs=cli_args ) as external_pipeline: pipeline_snapshot = external_pipeline.pipeline_snapshot if verbose: print_pipeline(pipeline_snapshot, print_fn=print_fn) else: print_solids(pipeline_snapshot, print_fn=print_fn)
def execute_execute_command_with_preset(preset_name, cli_args, instance, _mode): external_pipeline = get_external_pipeline_from_kwargs(cli_args, instance) # We should move this to use external pipeline # https://github.com/dagster-io/dagster/issues/2556 pipeline = recon_pipeline_from_origin(external_pipeline.handle.get_origin()) tags = get_tags_from_args(cli_args) solid_selection = get_solid_selection_from_args(cli_args) return execute_pipeline( pipeline, preset=preset_name, instance=DagsterInstance.get(), raise_on_error=False, tags=tags, solid_selection=solid_selection, )
def execute_execute_command(env_file_list, cli_args, instance=None, mode=None, tags=None): check.opt_list_param(env_file_list, 'env_file_list', of_type=str) check.opt_inst_param(instance, 'instance', DagsterInstance) instance = instance if instance else DagsterInstance.get() external_pipeline = get_external_pipeline_from_kwargs(cli_args, instance) # We should move this to use external pipeline # https://github.com/dagster-io/dagster/issues/2556 pipeline = recon_pipeline_from_origin( external_pipeline.handle.get_origin()) solid_selection = get_solid_selection_from_args(cli_args) return do_execute_command(pipeline, instance, env_file_list, mode, tags, solid_selection)
def command(**kwargs): capture_result[ 'external_pipeline'] = get_external_pipeline_from_kwargs( kwargs, DagsterInstance.ephemeral())
def execute_backfill_command(cli_args, print_fn, instance=None): instance = instance or DagsterInstance.get() external_pipeline = get_external_pipeline_from_kwargs(cli_args, instance) external_repository = get_external_repository_from_kwargs(cli_args, instance) # We should move this to use external repository # https://github.com/dagster-io/dagster/issues/2556 recon_repo = recon_repo_from_external_repo(external_repository) repo_def = recon_repo.get_definition() noprompt = cli_args.get('noprompt') pipeline_def = repo_def.get_pipeline(external_pipeline.name) # Resolve partition set all_partition_sets = repo_def.partition_set_defs + [ schedule_def.get_partition_set() for schedule_def in repo_def.schedule_defs if isinstance(schedule_def, PartitionScheduleDefinition) ] pipeline_partition_sets = [ x for x in all_partition_sets if x.pipeline_name == pipeline_def.name ] if not pipeline_partition_sets: raise click.UsageError( 'No partition sets found for pipeline `{}`'.format(pipeline_def.name) ) partition_set_name = cli_args.get('partition_set') if not partition_set_name: if len(pipeline_partition_sets) == 1: partition_set_name = pipeline_partition_sets[0].name elif noprompt: raise click.UsageError('No partition set specified (see option `--partition-set`)') else: partition_set_name = click.prompt( 'Select a partition set to use for backfill: {}'.format( ', '.join(x.name for x in pipeline_partition_sets) ) ) partition_set = next((x for x in pipeline_partition_sets if x.name == partition_set_name), None) if not partition_set: raise click.UsageError('No partition set found named `{}`'.format(partition_set_name)) # Resolve partitions to backfill partitions = gen_partitions_from_args(partition_set, cli_args) # Print backfill info print_fn('\n Pipeline: {}'.format(pipeline_def.name)) print_fn('Partition set: {}'.format(partition_set.name)) print_fn(' Partitions: {}\n'.format(print_partition_format(partitions, indent_level=15))) # Confirm and launch if noprompt or click.confirm( 'Do you want to proceed with the backfill ({} partitions)?'.format(len(partitions)) ): print_fn('Launching runs... ') backfill_id = make_new_backfill_id() run_tags = merge_dicts( PipelineRun.tags_for_backfill_id(backfill_id), get_tags_from_args(cli_args), ) for partition in partitions: run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, mode=partition_set.mode, solids_to_execute=frozenset(partition_set.solid_selection) if partition_set and partition_set.solid_selection else None, run_config=partition_set.run_config_for_partition(partition), tags=merge_dicts(partition_set.tags_for_partition(partition), run_tags), ) instance.launch_run(run.run_id, external_pipeline) # Remove once we can handle synchronous execution... currently limited by sqlite time.sleep(0.1) print_fn('Launched backfill job `{}`'.format(backfill_id)) else: print_fn(' Aborted!')
def _logged_pipeline_launch_command(config, preset_name, mode, instance, kwargs): check.inst_param(instance, 'instance', DagsterInstance) env = ( canonicalize_backcompat_args( (config if config else None), '--config', (kwargs.get('env') if kwargs.get('env') else None), '--env', '0.9.0', stacklevel=2, # this stacklevel can point the warning to this line ) or tuple() # back to default empty tuple ) env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str)) instance = DagsterInstance.get() external_pipeline = get_external_pipeline_from_kwargs(kwargs, instance) # We should move this to use external pipeline # https://github.com/dagster-io/dagster/issues/2556 pipeline = recon_pipeline_from_origin(external_pipeline.get_origin()) log_repo_stats(instance=instance, pipeline=pipeline, source='pipeline_launch_command') if preset_name: if env: raise click.UsageError('Can not use --preset with --config.') if mode: raise click.UsageError('Can not use --preset with --mode.') preset = pipeline.get_definition().get_preset(preset_name) else: preset = None run_tags = get_tags_from_args(kwargs) solid_selection = get_solid_selection_from_args(kwargs) if preset and preset.solid_selection is not None: check.invariant( solid_selection is None or solid_selection == preset.solid_selection, 'The solid_selection set in preset \'{preset}\', {preset_subset}, does not agree with ' 'the `solid_selection` argument: {solid_selection}'.format( preset=preset, preset_subset=preset.solid_selection, solid_selection=solid_selection, ), ) solid_selection = preset.solid_selection # generate pipeline subset from the given solid_selection if solid_selection: pipeline = pipeline.subset_for_execution(solid_selection) # FIXME need to check the env against run_config pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), solid_selection=solid_selection, solids_to_execute=pipeline.solids_to_execute, run_config=preset.run_config if preset else load_yaml_from_glob_list(env), mode=(preset.mode if preset else mode) or 'default', tags=run_tags, ) recon_repo = pipeline.get_reconstructable_repository() repo_location = InProcessRepositoryLocation(recon_repo) external_pipeline = ( repo_location.get_repository(recon_repo.get_definition().name).get_full_external_pipeline( pipeline.get_definition().name ), ) return instance.launch_run(pipeline_run.run_id, external_pipeline)
def command(**kwargs): with get_external_pipeline_from_kwargs(DagsterInstance.get(), "", kwargs) as external_pipeline: capture_result["external_pipeline"] = external_pipeline
def command(**kwargs): with get_external_pipeline_from_kwargs( kwargs, DagsterInstance.ephemeral()) as external_pipeline: capture_result["external_pipeline"] = external_pipeline
def command(**kwargs): with get_external_pipeline_from_kwargs(kwargs) as external_pipeline: capture_result["external_pipeline"] = external_pipeline
def command(**kwargs): capture_result['external_pipeline'] = get_external_pipeline_from_kwargs(kwargs)