def validate_run_config( pipeline_def: PipelineDefinition, run_config: Optional[Dict[str, Any]] = None, mode: Optional[str] = None, ) -> Dict[str, Any]: """Function to validate a provided run config blob against a given pipeline and mode. If validation is successful, this function will return a dictionary representation of the validated config actually used during execution. Args: pipeline_def (PipelineDefinition): The pipeline definition to validate run config against run_config (Optional[Dict[str, Any]]): The run config to validate mode (str): The mode of the pipeline to validate against (different modes may require different config) Returns: Dict[str, Any]: A dictionary representation of the validated config. """ experimental_fn_warning("validate_run_config") pipeline_def = check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) run_config = check.opt_dict_param(run_config, "run_config", key_type=str) mode = check.opt_str_param(mode, "mode", default=pipeline_def.get_default_mode_name()) return EnvironmentConfig.build(pipeline_def, run_config, mode=mode).to_dict()
def build_sensor_context( instance: DagsterInstance, cursor: Optional[str] = None) -> SensorExecutionContext: """Builds sensor execution context using the provided parameters. The instance provided to ``build_sensor_context`` must be persistent; DagsterInstance.ephemeral() will result in an error. Args: instance (DagsterInstance): The dagster instance configured to run the sensor. Examples: .. code-block:: python context = build_sensor_context(instance) my_sensor.get_execution_data(context) """ experimental_fn_warning("build_sensor_context") check.inst_param(instance, "instance", DagsterInstance) check.opt_str_param(cursor, "cursor") return SensorExecutionContext( instance_ref=instance.get_ref(), last_completion_time=None, last_run_key=None, cursor=cursor, )
def build_schedule_context( instance: DagsterInstance, scheduled_execution_time: Optional[datetime] = None ) -> ScheduleExecutionContext: """Builds schedule execution context using the provided parameters. The instance provided to ``build_schedule_context`` must be persistent; DagsterInstance.ephemeral() will result in an error. Args: instance (DagsterInstance): The dagster instance configured to run the schedule. scheduled_execution_time (datetime): The time in which the execution was scheduled to happen. May differ slightly from both the actual execution time and the time at which the run config is computed. Examples: .. code-block:: python context = build_schedule_context(instance) daily_schedule.get_execution_data(context) """ experimental_fn_warning("build_schedule_context") check.inst_param(instance, "instance", DagsterInstance) return ScheduleExecutionContext( instance_ref=instance.get_ref(), scheduled_execution_time=check.opt_inst_param( scheduled_execution_time, "scheduled_execution_time", datetime), )
def weekly_partitioned_config( start_date: Union[datetime, str], timezone: Optional[str] = None, fmt: Optional[str] = None, ) -> Callable[[Callable[[datetime, datetime], Dict[str, Any]]], PartitionedConfig]: """Defines run config over a set of weekly partitions. The decorated function should accept a start datetime and end datetime, which represent the date partition the config should delineate. The decorated function should return a run config dictionary. The resulting object created by this decorator can be provided to the config argument of a Job. Args: start_date (Union[datetime.datetime, str]): The date from which to run the schedule. Can provide in either a datetime or string format. timezone (Optional[str]): The timezone in which each date should exist. fmt (Optional[str]): The date format to use. Defaults to `%Y-%m-%d`. """ experimental_fn_warning("weekly_partitioned_config") _fmt = fmt or DEFAULT_DATE_FORMAT _timezone = timezone or "UTC" if isinstance(start_date, str): _start_date = datetime.strptime(start_date, _fmt) else: _start_date = start_date def inner(fn: Callable[[datetime, datetime], Dict[str, Any]]) -> PartitionedConfig: check.callable_param(fn, "fn") return PartitionedConfig( run_config_for_partition_fn=lambda partition: fn( partition.value[0], partition.value[1] ), partitions_def=TimeWindowPartitionsDefinition( schedule_type=ScheduleType.WEEKLY, start=_start_date, timezone=_timezone, fmt=_fmt, ), ) return inner
def build_solid_context( resources: Optional[Dict[str, Any]] = None, config: Optional[Any] = None, instance: Optional[DagsterInstance] = None, ) -> DirectSolidExecutionContext: """Builds solid execution context from provided parameters. ``build_solid_context`` can be used as either a function or context manager. If there is a provided resource that is a context manager, then ``build_solid_context`` must be used as a context manager. This function can be used to provide the context argument when directly invoking a solid. Args: resources (Optional[Dict[str, Any]]): The resources to provide to the context. These can be either values or resource definitions. config (Optional[Any]): The solid config to provide to the context. instance (Optional[DagsterInstance]): The dagster instance configured for the context. Defaults to DagsterInstance.ephemeral(). Examples: .. code-block:: python context = build_solid_context() solid_to_invoke(context) with build_solid_context(resources={"foo": context_manager_resource}) as context: solid_to_invoke(context) """ experimental_fn_warning("build_solid_context") return DirectSolidExecutionContext( resources_dict=check.opt_dict_param(resources, "resources", key_type=str), solid_config=config, instance=check.opt_inst_param(instance, "instance", DagsterInstance), )
def build_input_context( name: Optional[str] = None, config: Optional[Any] = None, metadata: Optional[Dict[str, Any]] = None, upstream_output: Optional["OutputContext"] = None, dagster_type: Optional["DagsterType"] = None, resource_config: Optional[Dict[str, Any]] = None, resources: Optional[Dict[str, Any]] = None, ) -> "InputContext": """Builds input context from provided parameters. ``build_input_context`` can be used as either a function, or a context manager. If resources that are also context managers are provided, then ``build_input_context`` must be used as a context manager. Args: name (Optional[str]): The name of the input that we're loading. config (Optional[Any]): The config attached to the input that we're loading. metadata (Optional[Dict[str, Any]]): A dict of metadata that is assigned to the InputDefinition that we're loading for. upstream_output (Optional[OutputContext]): Info about the output that produced the object we're loading. dagster_type (Optional[DagsterType]): The type of this input. resource_config (Optional[Dict[str, Any]]): The resource config to make available from the input context. This usually corresponds to the config provided to the resource that loads the input manager. resources (Optional[Dict[str, Any]]): The resources to make available from the context. For a given key, you can provide either an actual instance of an object, or a resource definition. Examples: .. code-block:: python build_input_context(step_key, name) with build_input_context( step_key, name, resources={"foo": context_manager_resource} ) as context: do_something """ from dagster.core.execution.context.output import OutputContext from dagster.core.types.dagster_type import DagsterType from dagster.core.execution.context_creation_pipeline import initialize_console_manager experimental_fn_warning("build_input_context") name = check.opt_str_param(name, "name") metadata = check.opt_dict_param(metadata, "metadata", key_type=str) upstream_output = check.opt_inst_param(upstream_output, "upstream_output", OutputContext) dagster_type = check.opt_inst_param(dagster_type, "dagster_type", DagsterType) resource_config = check.opt_dict_param(resource_config, "resource_config", key_type=str) resources = check.opt_dict_param(resources, "resources", key_type=str) return InputContext( name=name, pipeline_name=None, solid_def=None, config=config, metadata=metadata, upstream_output=upstream_output, dagster_type=dagster_type, log_manager=initialize_console_manager(None), resource_config=resource_config, resources=resources, step_context=None, )
def my_experimental_function(): experimental_fn_warning("my_experimental_function")
def build_output_context( step_key: str, name: str, metadata: Optional[Dict[str, Any]] = None, mapping_key: Optional[str] = None, config: Optional[Any] = None, dagster_type: Optional["DagsterType"] = None, version: Optional[str] = None, resource_config: Optional[Dict[str, Any]] = None, resources: Optional[Dict[str, Any]] = None, ) -> "OutputContext": """Builds output context from provided parameters. ``build_output_context`` can be used as either a function, or a context manager. If resources that are also context managers are provided, then ``build_output_context`` must be used as a context manager. Args: step_key (str): The step_key for the compute step that produced the output. name (str): The name of the output that produced the output. metadata (Optional[Dict[str, Any]]): A dict of the metadata that is assigned to the OutputDefinition that produced the output. mapping_key (Optional[str]): The key that identifies a unique mapped output. None for regular outputs. config (Optional[Any]): The configuration for the output. dagster_type (Optional[DagsterType]): The type of this output. version (Optional[str]): (Experimental) The version of the output. resource_config (Optional[Dict[str, Any]]): The resource config to make available from the input context. This usually corresponds to the config provided to the resource that loads the output manager. resources (Optional[Resources]): The resources to make available from the context. For a given key, you can provide either an actual instance of an object, or a resource definition. Examples: .. code-block:: python build_output_context(step_key, name) with build_output_context( step_key, name, resources={"foo": context_manager_resource} ) as context: do_something """ from dagster.core.types.dagster_type import DagsterType from dagster.core.execution.context_creation_pipeline import initialize_console_manager experimental_fn_warning("build_output_context") step_key = check.str_param(step_key, "step_key") name = check.str_param(name, "name") metadata = check.opt_dict_param(metadata, "metadata", key_type=str) mapping_key = check.opt_str_param(mapping_key, "mapping_key") dagster_type = check.opt_inst_param(dagster_type, "dagster_type", DagsterType) version = check.opt_str_param(version, "version") resource_config = check.opt_dict_param(resource_config, "resource_config", key_type=str) resources = check.opt_dict_param(resources, "resources", key_type=str) return OutputContext( step_key=step_key, name=name, pipeline_name=None, run_id=None, metadata=metadata, mapping_key=mapping_key, config=config, solid_def=None, dagster_type=dagster_type, log_manager=initialize_console_manager(None), version=version, resource_config=resource_config, resources=resources, step_context=None, )