def host_dagit_ui_with_workspace(workspace, host, port, storage_fallback, port_lookup=True): check.inst_param(workspace, 'workspace', Workspace) instance = DagsterInstance.get(storage_fallback) if len(workspace.repository_location_handles) == 1: repository_location_handle = workspace.repository_location_handles[0] if len(repository_location_handle.repository_code_pointer_dict) == 1: pointer = next( iter(repository_location_handle.repository_code_pointer_dict. values())) from dagster.core.definitions.reconstructable import ReconstructableRepository recon_repo = ReconstructableRepository(pointer) log_repo_stats(instance=instance, repo=recon_repo, source='dagit') app = create_app_from_workspace(workspace, instance) start_server(host, port, app, port_lookup)
def host_dagit_ui_with_workspace(instance, workspace, host, port, path_prefix, port_lookup=True): check.inst_param(instance, 'instance', DagsterInstance) check.inst_param(workspace, 'workspace', Workspace) if len(workspace.repository_location_handles) == 1: repository_location_handle = workspace.repository_location_handles[0] # Telemetry logic needs to be updated to support multi-repo / gRPC repo locations # See https://github.com/dagster-io/dagster/issues/2752 if (hasattr(repository_location_handle, 'repository_code_pointer_dict') and len( repository_location_handle.repository_code_pointer_dict) == 1): pointer = next( iter(repository_location_handle.repository_code_pointer_dict. values())) from dagster.core.definitions.reconstructable import ReconstructableRepository recon_repo = ReconstructableRepository(pointer) log_repo_stats(instance=instance, repo=recon_repo, source='dagit') app = create_app_from_workspace(workspace, instance, path_prefix) start_server(instance, host, port, path_prefix, app, port_lookup)
def pipeline_launch_command(env, preset_name, mode, **kwargs): env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str)) pipeline = recon_pipeline_for_cli_args(kwargs) instance = DagsterInstance.get() log_repo_stats(instance=instance, pipeline=pipeline) if preset_name: if env: raise click.UsageError('Can not use --preset with --env.') if mode: raise click.UsageError('Can not use --preset with --mode.') preset = pipeline.get_preset(preset_name) else: preset = None run_tags = get_tags_from_args(kwargs) # FIXME need to check the env against environment_dict pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), solid_subset=preset.solid_subset if preset else None, environment_dict=preset.environment_dict if preset else load_yaml_from_glob_list(env), mode=(preset.mode if preset else mode) or 'default', tags=run_tags, ) return instance.launch_run(pipeline_run.run_id)
def pipeline_launch_command(env, preset_name, mode, **kwargs): env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str)) pipeline = recon_pipeline_for_cli_args(kwargs) instance = DagsterInstance.get() log_repo_stats(instance=instance, pipeline=pipeline, source='pipeline_launch_command') if preset_name: if env: raise click.UsageError('Can not use --preset with --env.') if mode: raise click.UsageError('Can not use --preset with --mode.') preset = pipeline.get_preset(preset_name) else: preset = None run_tags = get_tags_from_args(kwargs) solid_selection = get_solid_selection_from_args(kwargs) if preset and preset.solid_selection is not None: check.invariant( solid_selection is None or solid_selection == preset.solid_selection, 'The solid_selection set in preset \'{preset}\', {preset_subset}, does not agree with ' 'the `solid_selection` argument: {solid_selection}'.format( preset=preset, preset_subset=preset.solid_selection, solid_selection=solid_selection, ), ) solid_selection = preset.solid_selection # generate pipeline subset from the given solid_selection if solid_selection: pipeline = pipeline.subset_for_execution(solid_selection) # FIXME need to check the env against environment_dict pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), solid_selection=solid_selection, solids_to_execute=pipeline.solids_to_execute, environment_dict=preset.environment_dict if preset else load_yaml_from_glob_list(env), mode=(preset.mode if preset else mode) or 'default', tags=run_tags, ) recon_repo = pipeline.get_reconstructable_repository() repo_location = InProcessRepositoryLocation(recon_repo) external_pipeline = ( repo_location.get_repository(recon_repo.get_definition().name).get_full_external_pipeline( pipeline.get_definition().name ), ) return instance.launch_run(pipeline_run.run_id, external_pipeline)
def _logged_execute_pipeline( pipeline: Union[IPipeline, PipelineDefinition], instance: DagsterInstance, run_config: Optional[dict] = None, mode: Optional[str] = None, preset: Optional[str] = None, tags: Optional[Dict[str, Any]] = None, solid_selection: Optional[List[str]] = None, raise_on_error: bool = True, ) -> PipelineExecutionResult: check.inst_param(instance, "instance", DagsterInstance) ( pipeline, run_config, mode, tags, solids_to_execute, solid_selection, ) = _check_execute_pipeline_args( pipeline=pipeline, run_config=run_config, mode=mode, preset=preset, tags=tags, solid_selection=solid_selection, ) log_repo_stats(instance=instance, pipeline=pipeline, source="execute_pipeline") pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), run_config=run_config, mode=mode, solid_selection=solid_selection, solids_to_execute=solids_to_execute, tags=tags, pipeline_code_origin=(pipeline.get_python_origin() if isinstance( pipeline, ReconstructablePipeline) else None), ) return execute_run( pipeline, pipeline_run, instance, raise_on_error=raise_on_error, )
def _logged_execute_pipeline( pipeline, run_config=None, mode=None, preset=None, tags=None, solid_selection=None, instance=None, raise_on_error=True, ): ( pipeline, run_config, instance, mode, tags, solids_to_execute, solid_selection, ) = _check_execute_pipeline_args( pipeline=pipeline, run_config=run_config, mode=mode, preset=preset, tags=tags, solid_selection=solid_selection, instance=instance, ) log_repo_stats(instance=instance, pipeline=pipeline, source='execute_pipeline') pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), run_config=run_config, mode=mode, solid_selection=solid_selection, solids_to_execute=solids_to_execute, tags=tags, ) return execute_run(pipeline, pipeline_run, instance, raise_on_error=raise_on_error)
def _logged_execute_pipeline( pipeline, run_config=None, mode=None, preset=None, tags=None, solid_selection=None, instance=None, raise_on_error=True, environment_dict=None, ): # stack level is to punch through helper functions and telemetry wrapper run_config = canonicalize_run_config(run_config, environment_dict, stacklevel=7) ( pipeline, run_config, instance, mode, tags, solids_to_execute, solid_selection, ) = _check_execute_pipeline_args( pipeline=pipeline, run_config=run_config, mode=mode, preset=preset, tags=tags, solid_selection=solid_selection, instance=instance, ) log_repo_stats(instance=instance, pipeline=pipeline, source='execute_pipeline') pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), run_config=run_config, mode=mode, solid_selection=solid_selection, solids_to_execute=solids_to_execute, tags=tags, ) return execute_run(pipeline, pipeline_run, instance, raise_on_error=raise_on_error)
def _logged_pipeline_launch_command(config, preset_name, mode, instance, kwargs): check.inst_param(instance, 'instance', DagsterInstance) env = ( canonicalize_backcompat_args( (config if config else None), '--config', (kwargs.get('env') if kwargs.get('env') else None), '--env', '0.9.0', stacklevel=2, # this stacklevel can point the warning to this line ) or tuple() # back to default empty tuple ) env = list(check.opt_tuple_param(env, 'env', default=(), of_type=str)) instance = DagsterInstance.get() external_pipeline = get_external_pipeline_from_kwargs(kwargs, instance) # We should move this to use external pipeline # https://github.com/dagster-io/dagster/issues/2556 pipeline = recon_pipeline_from_origin(external_pipeline.get_origin()) log_repo_stats(instance=instance, pipeline=pipeline, source='pipeline_launch_command') if preset_name: if env: raise click.UsageError('Can not use --preset with --config.') if mode: raise click.UsageError('Can not use --preset with --mode.') preset = pipeline.get_definition().get_preset(preset_name) else: preset = None run_tags = get_tags_from_args(kwargs) solid_selection = get_solid_selection_from_args(kwargs) if preset and preset.solid_selection is not None: check.invariant( solid_selection is None or solid_selection == preset.solid_selection, 'The solid_selection set in preset \'{preset}\', {preset_subset}, does not agree with ' 'the `solid_selection` argument: {solid_selection}'.format( preset=preset, preset_subset=preset.solid_selection, solid_selection=solid_selection, ), ) solid_selection = preset.solid_selection # generate pipeline subset from the given solid_selection if solid_selection: pipeline = pipeline.subset_for_execution(solid_selection) # FIXME need to check the env against run_config pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), solid_selection=solid_selection, solids_to_execute=pipeline.solids_to_execute, run_config=preset.run_config if preset else load_yaml_from_glob_list(env), mode=(preset.mode if preset else mode) or 'default', tags=run_tags, ) recon_repo = pipeline.get_reconstructable_repository() repo_location = InProcessRepositoryLocation(recon_repo) external_pipeline = ( repo_location.get_repository(recon_repo.get_definition().name).get_full_external_pipeline( pipeline.get_definition().name ), ) return instance.launch_run(pipeline_run.run_id, external_pipeline)
def execute_pipeline( pipeline, run_config=None, mode=None, preset=None, tags=None, solid_selection=None, instance=None, raise_on_error=True, environment_dict=None, ): '''Execute a pipeline synchronously. Users will typically call this API when testing pipeline execution, or running standalone scripts. Parameters: pipeline (Union[ExecutablePipeline, PipelineDefinition]): The pipeline to execute. environment_dict (Optional[dict]): The environment configuration that parametrizes this run, as a dict. mode (Optional[str]): The name of the pipeline mode to use. You may not set both ``mode`` and ``preset``. preset (Optional[str]): The name of the pipeline preset to use. You may not set both ``mode`` and ``preset``. tags (Optional[Dict[str, Any]]): Arbitrary key-value pairs that will be added to pipeline logs. instance (Optional[DagsterInstance]): The instance to execute against. If this is ``None``, an ephemeral instance will be used, and no artifacts will be persisted from the run. raise_on_error (Optional[bool]): Whether or not to raise exceptions when they occur. Defaults to ``True``, since this is the most useful behavior in test. solid_selection (Optional[List[str]]): A list of solid selection queries (including single solid names) to execute. For example: - ['some_solid']: select "some_solid" itself. - ['*some_solid']: select "some_solid" and all its ancestors (upstream dependencies). - ['*some_solid+++']: select "some_solid", all its ancestors, and its descendants (downstream dependencies) within 3 levels down. - ['*some_solid', 'other_solid_a', 'other_solid_b+']: select "some_solid" and all its ancestors, "other_solid_a" itself, and "other_solid_b" and its direct child solids. Returns: :py:class:`PipelineExecutionResult`: The result of pipeline execution. For the asynchronous version, see :py:func:`execute_pipeline_iterator`. This is the entrypoint for dagster CLI execution. For the dagster-graphql entrypoint, see ``dagster.core.execution.api.execute_plan()``. ''' # stack level is to punch through helper function and telemetry wrapper environment_dict = canonicalize_run_config(run_config, environment_dict, stacklevel=5) ( pipeline, environment_dict, instance, mode, tags, solids_to_execute, solid_selection, ) = _check_execute_pipeline_args( pipeline=pipeline, environment_dict=environment_dict, mode=mode, preset=preset, tags=tags, solid_selection=solid_selection, instance=instance, ) log_repo_stats(instance=instance, pipeline=pipeline, source='execute_pipeline') pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), environment_dict=environment_dict, mode=mode, solid_selection=solid_selection, solids_to_execute=solids_to_execute, tags=tags, ) return execute_run(pipeline, pipeline_run, instance, raise_on_error=raise_on_error)