def host_dagit_ui_with_workspace(workspace,
host,
port,
storage_fallback,
port_lookup=True):
check.inst_param(workspace, 'workspace', Workspace)
instance = DagsterInstance.get(storage_fallback)
if len(workspace.repository_location_handles) == 1:
repository_location_handle = workspace.repository_location_handles[0]
if len(repository_location_handle.repository_code_pointer_dict) == 1:
pointer = next(
iter(repository_location_handle.repository_code_pointer_dict.
values()))
from dagster.core.definitions.reconstructable import ReconstructableRepository
recon_repo = ReconstructableRepository(pointer)
log_repo_stats(instance=instance, repo=recon_repo, source='dagit')
app = create_app_from_workspace(workspace, instance)
start_server(host, port, app, port_lookup)
def host_dagit_ui_with_workspace(instance,
workspace,
host,
port,
path_prefix,
port_lookup=True):
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(workspace, 'workspace', Workspace)
if len(workspace.repository_location_handles) == 1:
repository_location_handle = workspace.repository_location_handles[0]
# Telemetry logic needs to be updated to support multi-repo / gRPC repo locations
# See https://github.com/dagster-io/dagster/issues/2752
if (hasattr(repository_location_handle, 'repository_code_pointer_dict')
and len(
repository_location_handle.repository_code_pointer_dict)
== 1):
pointer = next(
iter(repository_location_handle.repository_code_pointer_dict.
values()))
from dagster.core.definitions.reconstructable import ReconstructableRepository
recon_repo = ReconstructableRepository(pointer)
log_repo_stats(instance=instance, repo=recon_repo, source='dagit')
app = create_app_from_workspace(workspace, instance, path_prefix)
start_server(instance, host, port, path_prefix, app, port_lookup)
def pipeline_launch_command(env, preset_name, mode, **kwargs):
env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str))
pipeline = recon_pipeline_for_cli_args(kwargs)
instance = DagsterInstance.get()
log_repo_stats(instance=instance, pipeline=pipeline)
if preset_name:
if env:
raise click.UsageError('Can not use --preset with --env.')
if mode:
raise click.UsageError('Can not use --preset with --mode.')
preset = pipeline.get_preset(preset_name)
else:
preset = None
run_tags = get_tags_from_args(kwargs)
# FIXME need to check the env against environment_dict
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
solid_subset=preset.solid_subset if preset else None,
environment_dict=preset.environment_dict
if preset else load_yaml_from_glob_list(env),
mode=(preset.mode if preset else mode) or 'default',
tags=run_tags,
)
return instance.launch_run(pipeline_run.run_id)
def pipeline_launch_command(env, preset_name, mode, **kwargs):
env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str))
pipeline = recon_pipeline_for_cli_args(kwargs)
instance = DagsterInstance.get()
log_repo_stats(instance=instance, pipeline=pipeline, source='pipeline_launch_command')
if preset_name:
if env:
raise click.UsageError('Can not use --preset with --env.')
if mode:
raise click.UsageError('Can not use --preset with --mode.')
preset = pipeline.get_preset(preset_name)
else:
preset = None
run_tags = get_tags_from_args(kwargs)
solid_selection = get_solid_selection_from_args(kwargs)
if preset and preset.solid_selection is not None:
check.invariant(
solid_selection is None or solid_selection == preset.solid_selection,
'The solid_selection set in preset \'{preset}\', {preset_subset}, does not agree with '
'the `solid_selection` argument: {solid_selection}'.format(
preset=preset,
preset_subset=preset.solid_selection,
solid_selection=solid_selection,
),
)
solid_selection = preset.solid_selection
# generate pipeline subset from the given solid_selection
if solid_selection:
pipeline = pipeline.subset_for_execution(solid_selection)
# FIXME need to check the env against environment_dict
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
solid_selection=solid_selection,
solids_to_execute=pipeline.solids_to_execute,
environment_dict=preset.environment_dict if preset else load_yaml_from_glob_list(env),
mode=(preset.mode if preset else mode) or 'default',
tags=run_tags,
)
recon_repo = pipeline.get_reconstructable_repository()
repo_location = InProcessRepositoryLocation(recon_repo)
external_pipeline = (
repo_location.get_repository(recon_repo.get_definition().name).get_full_external_pipeline(
pipeline.get_definition().name
),
)
return instance.launch_run(pipeline_run.run_id, external_pipeline)
def _logged_execute_pipeline(
pipeline: Union[IPipeline, PipelineDefinition],
instance: DagsterInstance,
run_config: Optional[dict] = None,
mode: Optional[str] = None,
preset: Optional[str] = None,
tags: Optional[Dict[str, Any]] = None,
solid_selection: Optional[List[str]] = None,
raise_on_error: bool = True,
) -> PipelineExecutionResult:
check.inst_param(instance, "instance", DagsterInstance)
(
pipeline,
run_config,
mode,
tags,
solids_to_execute,
solid_selection,
) = _check_execute_pipeline_args(
pipeline=pipeline,
run_config=run_config,
mode=mode,
preset=preset,
tags=tags,
solid_selection=solid_selection,
)
log_repo_stats(instance=instance,
pipeline=pipeline,
source="execute_pipeline")
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
run_config=run_config,
mode=mode,
solid_selection=solid_selection,
solids_to_execute=solids_to_execute,
tags=tags,
pipeline_code_origin=(pipeline.get_python_origin() if isinstance(
pipeline, ReconstructablePipeline) else None),
)
return execute_run(
pipeline,
pipeline_run,
instance,
raise_on_error=raise_on_error,
)
def _logged_execute_pipeline(
pipeline,
run_config=None,
mode=None,
preset=None,
tags=None,
solid_selection=None,
instance=None,
raise_on_error=True,
):
(
pipeline,
run_config,
instance,
mode,
tags,
solids_to_execute,
solid_selection,
) = _check_execute_pipeline_args(
pipeline=pipeline,
run_config=run_config,
mode=mode,
preset=preset,
tags=tags,
solid_selection=solid_selection,
instance=instance,
)
log_repo_stats(instance=instance,
pipeline=pipeline,
source='execute_pipeline')
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
run_config=run_config,
mode=mode,
solid_selection=solid_selection,
solids_to_execute=solids_to_execute,
tags=tags,
)
return execute_run(pipeline,
pipeline_run,
instance,
raise_on_error=raise_on_error)
def _logged_execute_pipeline(
pipeline,
run_config=None,
mode=None,
preset=None,
tags=None,
solid_selection=None,
instance=None,
raise_on_error=True,
environment_dict=None,
):
# stack level is to punch through helper functions and telemetry wrapper
run_config = canonicalize_run_config(run_config, environment_dict, stacklevel=7)
(
pipeline,
run_config,
instance,
mode,
tags,
solids_to_execute,
solid_selection,
) = _check_execute_pipeline_args(
pipeline=pipeline,
run_config=run_config,
mode=mode,
preset=preset,
tags=tags,
solid_selection=solid_selection,
instance=instance,
)
log_repo_stats(instance=instance, pipeline=pipeline, source='execute_pipeline')
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
run_config=run_config,
mode=mode,
solid_selection=solid_selection,
solids_to_execute=solids_to_execute,
tags=tags,
)
return execute_run(pipeline, pipeline_run, instance, raise_on_error=raise_on_error)
def _logged_pipeline_launch_command(config, preset_name, mode, instance, kwargs):
check.inst_param(instance, 'instance', DagsterInstance)
env = (
canonicalize_backcompat_args(
(config if config else None),
'--config',
(kwargs.get('env') if kwargs.get('env') else None),
'--env',
'0.9.0',
stacklevel=2, # this stacklevel can point the warning to this line
)
or tuple() # back to default empty tuple
)
env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str))
instance = DagsterInstance.get()
external_pipeline = get_external_pipeline_from_kwargs(kwargs, instance)
# We should move this to use external pipeline
# https://github.com/dagster-io/dagster/issues/2556
pipeline = recon_pipeline_from_origin(external_pipeline.get_origin())
log_repo_stats(instance=instance, pipeline=pipeline, source='pipeline_launch_command')
if preset_name:
if env:
raise click.UsageError('Can not use --preset with --config.')
if mode:
raise click.UsageError('Can not use --preset with --mode.')
preset = pipeline.get_definition().get_preset(preset_name)
else:
preset = None
run_tags = get_tags_from_args(kwargs)
solid_selection = get_solid_selection_from_args(kwargs)
if preset and preset.solid_selection is not None:
check.invariant(
solid_selection is None or solid_selection == preset.solid_selection,
'The solid_selection set in preset \'{preset}\', {preset_subset}, does not agree with '
'the `solid_selection` argument: {solid_selection}'.format(
preset=preset,
preset_subset=preset.solid_selection,
solid_selection=solid_selection,
),
)
solid_selection = preset.solid_selection
# generate pipeline subset from the given solid_selection
if solid_selection:
pipeline = pipeline.subset_for_execution(solid_selection)
# FIXME need to check the env against run_config
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
solid_selection=solid_selection,
solids_to_execute=pipeline.solids_to_execute,
run_config=preset.run_config if preset else load_yaml_from_glob_list(env),
mode=(preset.mode if preset else mode) or 'default',
tags=run_tags,
)
recon_repo = pipeline.get_reconstructable_repository()
repo_location = InProcessRepositoryLocation(recon_repo)
external_pipeline = (
repo_location.get_repository(recon_repo.get_definition().name).get_full_external_pipeline(
pipeline.get_definition().name
),
)
return instance.launch_run(pipeline_run.run_id, external_pipeline)
def execute_pipeline(
pipeline,
run_config=None,
mode=None,
preset=None,
tags=None,
solid_selection=None,
instance=None,
raise_on_error=True,
environment_dict=None,
):
'''Execute a pipeline synchronously.
Users will typically call this API when testing pipeline execution, or running standalone
scripts.
Parameters:
pipeline (Union[ExecutablePipeline, PipelineDefinition]): The pipeline to execute.
environment_dict (Optional[dict]): The environment configuration that parametrizes this run,
as a dict.
mode (Optional[str]): The name of the pipeline mode to use. You may not set both ``mode``
and ``preset``.
preset (Optional[str]): The name of the pipeline preset to use. You may not set both
``mode`` and ``preset``.
tags (Optional[Dict[str, Any]]): Arbitrary key-value pairs that will be added to pipeline
logs.
instance (Optional[DagsterInstance]): The instance to execute against. If this is ``None``,
an ephemeral instance will be used, and no artifacts will be persisted from the run.
raise_on_error (Optional[bool]): Whether or not to raise exceptions when they occur.
Defaults to ``True``, since this is the most useful behavior in test.
solid_selection (Optional[List[str]]): A list of solid selection queries (including single
solid names) to execute. For example:
- ['some_solid']: select "some_solid" itself.
- ['*some_solid']: select "some_solid" and all its ancestors (upstream dependencies).
- ['*some_solid+++']: select "some_solid", all its ancestors, and its descendants
(downstream dependencies) within 3 levels down.
- ['*some_solid', 'other_solid_a', 'other_solid_b+']: select "some_solid" and all its
ancestors, "other_solid_a" itself, and "other_solid_b" and its direct child solids.
Returns:
:py:class:`PipelineExecutionResult`: The result of pipeline execution.
For the asynchronous version, see :py:func:`execute_pipeline_iterator`.
This is the entrypoint for dagster CLI execution. For the dagster-graphql entrypoint, see
``dagster.core.execution.api.execute_plan()``.
'''
# stack level is to punch through helper function and telemetry wrapper
environment_dict = canonicalize_run_config(run_config,
environment_dict,
stacklevel=5)
(
pipeline,
environment_dict,
instance,
mode,
tags,
solids_to_execute,
solid_selection,
) = _check_execute_pipeline_args(
pipeline=pipeline,
environment_dict=environment_dict,
mode=mode,
preset=preset,
tags=tags,
solid_selection=solid_selection,
instance=instance,
)
log_repo_stats(instance=instance,
pipeline=pipeline,
source='execute_pipeline')
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition(),
environment_dict=environment_dict,
mode=mode,
solid_selection=solid_selection,
solids_to_execute=solids_to_execute,
tags=tags,
)
return execute_run(pipeline,
pipeline_run,
instance,
raise_on_error=raise_on_error)
