def validate_run_config(
pipeline_def: PipelineDefinition,
run_config: Optional[Dict[str, Any]] = None,
mode: Optional[str] = None,
) -> Dict[str, Any]:
"""Function to validate a provided run config blob against a given pipeline and mode.
If validation is successful, this function will return a dictionary representation of the
validated config actually used during execution.
Args:
pipeline_def (PipelineDefinition): The pipeline definition to validate run config against
run_config (Optional[Dict[str, Any]]): The run config to validate
mode (str): The mode of the pipeline to validate against (different modes may require
different config)
Returns:
Dict[str, Any]: A dictionary representation of the validated config.
"""
experimental_fn_warning("validate_run_config")
pipeline_def = check.inst_param(pipeline_def, "pipeline_def",
PipelineDefinition)
run_config = check.opt_dict_param(run_config, "run_config", key_type=str)
mode = check.opt_str_param(mode,
"mode",
default=pipeline_def.get_default_mode_name())
return EnvironmentConfig.build(pipeline_def, run_config,
mode=mode).to_dict()
def build_sensor_context(
instance: DagsterInstance,
cursor: Optional[str] = None) -> SensorExecutionContext:
"""Builds sensor execution context using the provided parameters.
The instance provided to ``build_sensor_context`` must be persistent;
DagsterInstance.ephemeral() will result in an error.
Args:
instance (DagsterInstance): The dagster instance configured to run the sensor.
Examples:
.. code-block:: python
context = build_sensor_context(instance)
my_sensor.get_execution_data(context)
"""
experimental_fn_warning("build_sensor_context")
check.inst_param(instance, "instance", DagsterInstance)
check.opt_str_param(cursor, "cursor")
return SensorExecutionContext(
instance_ref=instance.get_ref(),
last_completion_time=None,
last_run_key=None,
cursor=cursor,
)
def build_schedule_context(
instance: DagsterInstance,
scheduled_execution_time: Optional[datetime] = None
) -> ScheduleExecutionContext:
"""Builds schedule execution context using the provided parameters.
The instance provided to ``build_schedule_context`` must be persistent;
DagsterInstance.ephemeral() will result in an error.
Args:
instance (DagsterInstance): The dagster instance configured to run the schedule.
scheduled_execution_time (datetime): The time in which the execution was scheduled to
happen. May differ slightly from both the actual execution time and the time at which
the run config is computed.
Examples:
.. code-block:: python
context = build_schedule_context(instance)
daily_schedule.get_execution_data(context)
"""
experimental_fn_warning("build_schedule_context")
check.inst_param(instance, "instance", DagsterInstance)
return ScheduleExecutionContext(
instance_ref=instance.get_ref(),
scheduled_execution_time=check.opt_inst_param(
scheduled_execution_time, "scheduled_execution_time", datetime),
)
def weekly_partitioned_config(
start_date: Union[datetime, str],
timezone: Optional[str] = None,
fmt: Optional[str] = None,
) -> Callable[[Callable[[datetime, datetime], Dict[str, Any]]], PartitionedConfig]:
"""Defines run config over a set of weekly partitions.
The decorated function should accept a start datetime and end datetime, which represent the date
partition the config should delineate.
The decorated function should return a run config dictionary.
The resulting object created by this decorator can be provided to the config argument of a Job.
Args:
start_date (Union[datetime.datetime, str]): The date from which to run the schedule. Can
provide in either a datetime or string format.
timezone (Optional[str]): The timezone in which each date should exist.
fmt (Optional[str]): The date format to use. Defaults to `%Y-%m-%d`.
"""
experimental_fn_warning("weekly_partitioned_config")
_fmt = fmt or DEFAULT_DATE_FORMAT
_timezone = timezone or "UTC"
if isinstance(start_date, str):
_start_date = datetime.strptime(start_date, _fmt)
else:
_start_date = start_date
def inner(fn: Callable[[datetime, datetime], Dict[str, Any]]) -> PartitionedConfig:
check.callable_param(fn, "fn")
return PartitionedConfig(
run_config_for_partition_fn=lambda partition: fn(
partition.value[0], partition.value[1]
),
partitions_def=TimeWindowPartitionsDefinition(
schedule_type=ScheduleType.WEEKLY,
start=_start_date,
timezone=_timezone,
fmt=_fmt,
),
)
return inner
def build_solid_context(
resources: Optional[Dict[str, Any]] = None,
config: Optional[Any] = None,
instance: Optional[DagsterInstance] = None,
) -> DirectSolidExecutionContext:
"""Builds solid execution context from provided parameters.
``build_solid_context`` can be used as either a function or context manager. If there is a
provided resource that is a context manager, then ``build_solid_context`` must be used as a
context manager. This function can be used to provide the context argument when directly
invoking a solid.
Args:
resources (Optional[Dict[str, Any]]): The resources to provide to the context. These can be
either values or resource definitions.
config (Optional[Any]): The solid config to provide to the context.
instance (Optional[DagsterInstance]): The dagster instance configured for the context.
Defaults to DagsterInstance.ephemeral().
Examples:
.. code-block:: python
context = build_solid_context()
solid_to_invoke(context)
with build_solid_context(resources={"foo": context_manager_resource}) as context:
solid_to_invoke(context)
"""
experimental_fn_warning("build_solid_context")
return DirectSolidExecutionContext(
resources_dict=check.opt_dict_param(resources,
"resources",
key_type=str),
solid_config=config,
instance=check.opt_inst_param(instance, "instance", DagsterInstance),
)
def build_input_context(
name: Optional[str] = None,
config: Optional[Any] = None,
metadata: Optional[Dict[str, Any]] = None,
upstream_output: Optional["OutputContext"] = None,
dagster_type: Optional["DagsterType"] = None,
resource_config: Optional[Dict[str, Any]] = None,
resources: Optional[Dict[str, Any]] = None,
) -> "InputContext":
"""Builds input context from provided parameters.
``build_input_context`` can be used as either a function, or a context manager. If resources
that are also context managers are provided, then ``build_input_context`` must be used as a
context manager.
Args:
name (Optional[str]): The name of the input that we're loading.
config (Optional[Any]): The config attached to the input that we're loading.
metadata (Optional[Dict[str, Any]]): A dict of metadata that is assigned to the
InputDefinition that we're loading for.
upstream_output (Optional[OutputContext]): Info about the output that produced the object
we're loading.
dagster_type (Optional[DagsterType]): The type of this input.
resource_config (Optional[Dict[str, Any]]): The resource config to make available from the
input context. This usually corresponds to the config provided to the resource that
loads the input manager.
resources (Optional[Dict[str, Any]]): The resources to make available from the context.
For a given key, you can provide either an actual instance of an object, or a resource
definition.
Examples:
.. code-block:: python
build_input_context(step_key, name)
with build_input_context(
step_key,
name,
resources={"foo": context_manager_resource}
) as context:
do_something
"""
from dagster.core.execution.context.output import OutputContext
from dagster.core.types.dagster_type import DagsterType
from dagster.core.execution.context_creation_pipeline import initialize_console_manager
experimental_fn_warning("build_input_context")
name = check.opt_str_param(name, "name")
metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
upstream_output = check.opt_inst_param(upstream_output, "upstream_output", OutputContext)
dagster_type = check.opt_inst_param(dagster_type, "dagster_type", DagsterType)
resource_config = check.opt_dict_param(resource_config, "resource_config", key_type=str)
resources = check.opt_dict_param(resources, "resources", key_type=str)
return InputContext(
name=name,
pipeline_name=None,
solid_def=None,
config=config,
metadata=metadata,
upstream_output=upstream_output,
dagster_type=dagster_type,
log_manager=initialize_console_manager(None),
resource_config=resource_config,
resources=resources,
step_context=None,
)
def my_experimental_function():
experimental_fn_warning("my_experimental_function")
def build_output_context(
step_key: str,
name: str,
metadata: Optional[Dict[str, Any]] = None,
mapping_key: Optional[str] = None,
config: Optional[Any] = None,
dagster_type: Optional["DagsterType"] = None,
version: Optional[str] = None,
resource_config: Optional[Dict[str, Any]] = None,
resources: Optional[Dict[str, Any]] = None,
) -> "OutputContext":
"""Builds output context from provided parameters.
``build_output_context`` can be used as either a function, or a context manager. If resources
that are also context managers are provided, then ``build_output_context`` must be used as a
context manager.
Args:
step_key (str): The step_key for the compute step that produced the output.
name (str): The name of the output that produced the output.
metadata (Optional[Dict[str, Any]]): A dict of the metadata that is assigned to the
OutputDefinition that produced the output.
mapping_key (Optional[str]): The key that identifies a unique mapped output. None for regular outputs.
config (Optional[Any]): The configuration for the output.
dagster_type (Optional[DagsterType]): The type of this output.
version (Optional[str]): (Experimental) The version of the output.
resource_config (Optional[Dict[str, Any]]): The resource config to make available from the
input context. This usually corresponds to the config provided to the resource that
loads the output manager.
resources (Optional[Resources]): The resources to make available from the context.
For a given key, you can provide either an actual instance of an object, or a resource
definition.
Examples:
.. code-block:: python
build_output_context(step_key, name)
with build_output_context(
step_key,
name,
resources={"foo": context_manager_resource}
) as context:
do_something
"""
from dagster.core.types.dagster_type import DagsterType
from dagster.core.execution.context_creation_pipeline import initialize_console_manager
experimental_fn_warning("build_output_context")
step_key = check.str_param(step_key, "step_key")
name = check.str_param(name, "name")
metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
mapping_key = check.opt_str_param(mapping_key, "mapping_key")
dagster_type = check.opt_inst_param(dagster_type, "dagster_type", DagsterType)
version = check.opt_str_param(version, "version")
resource_config = check.opt_dict_param(resource_config, "resource_config", key_type=str)
resources = check.opt_dict_param(resources, "resources", key_type=str)
return OutputContext(
step_key=step_key,
name=name,
pipeline_name=None,
run_id=None,
metadata=metadata,
mapping_key=mapping_key,
config=config,
solid_def=None,
dagster_type=dagster_type,
log_manager=initialize_console_manager(None),
version=version,
resource_config=resource_config,
resources=resources,
step_context=None,
)
