def build_sensor_context(
instance: DagsterInstance,
cursor: Optional[str] = None) -> SensorExecutionContext:
"""Builds sensor execution context using the provided parameters.
The instance provided to ``build_sensor_context`` must be persistent;
DagsterInstance.ephemeral() will result in an error.
Args:
instance (DagsterInstance): The dagster instance configured to run the sensor.
Examples:
.. code-block:: python
context = build_sensor_context(instance)
my_sensor.get_execution_data(context)
"""
experimental_fn_warning("build_sensor_context")
check.inst_param(instance, "instance", DagsterInstance)
check.opt_str_param(cursor, "cursor")
return SensorExecutionContext(
instance_ref=instance.get_ref(),
last_completion_time=None,
last_run_key=None,
cursor=cursor,
)
def sync_get_external_schedule_execution_data_grpc(
api_client: "DagsterGrpcClient",
instance: DagsterInstance,
repository_handle: RepositoryHandle,
schedule_name: str,
scheduled_execution_time: Any,
) -> ScheduleExecutionData:
check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
check.str_param(schedule_name, "schedule_name")
check.opt_inst_param(scheduled_execution_time, "scheduled_execution_time",
PendulumDateTime)
origin = repository_handle.get_external_origin()
result = deserialize_as(
api_client.external_schedule_execution(
external_schedule_execution_args=ExternalScheduleExecutionArgs(
repository_origin=origin,
instance_ref=instance.get_ref(),
schedule_name=schedule_name,
scheduled_execution_timestamp=scheduled_execution_time.
timestamp() if scheduled_execution_time else None,
scheduled_execution_timezone=scheduled_execution_time.timezone.
name if scheduled_execution_time else None,
)),
(ScheduleExecutionData, ExternalScheduleExecutionErrorData),
)
if isinstance(result, ExternalScheduleExecutionErrorData):
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def execute_list_versions_command(instance: DagsterInstance,
kwargs: Dict[str, Any]) -> None:
check.inst_param(instance, "instance", DagsterInstance)
config = list(
check.opt_tuple_param(kwargs.get("config"),
"config",
default=tuple(),
of_type=str))
preset = kwargs.get("preset")
mode = kwargs.get("mode")
if preset and config:
raise click.UsageError("Can not use --preset with --config.")
pipeline_origin = get_pipeline_or_job_python_origin_from_kwargs(kwargs)
pipeline = recon_pipeline_from_origin(pipeline_origin)
run_config = get_run_config_from_file_list(config)
memoized_plan = create_execution_plan(
pipeline,
run_config=run_config,
mode=mode,
instance_ref=instance.get_ref(),
tags={MEMOIZED_RUN_TAG: "true"},
)
add_step_to_table(memoized_plan)
def build_schedule_context(
instance: DagsterInstance,
scheduled_execution_time: Optional[datetime] = None
) -> ScheduleExecutionContext:
"""Builds schedule execution context using the provided parameters.
The instance provided to ``build_schedule_context`` must be persistent;
DagsterInstance.ephemeral() will result in an error.
Args:
instance (DagsterInstance): The dagster instance configured to run the schedule.
scheduled_execution_time (datetime): The time in which the execution was scheduled to
happen. May differ slightly from both the actual execution time and the time at which
the run config is computed.
Examples:
.. code-block:: python
context = build_schedule_context(instance)
daily_schedule.get_execution_data(context)
"""
experimental_fn_warning("build_schedule_context")
check.inst_param(instance, "instance", DagsterInstance)
return ScheduleExecutionContext(
instance_ref=instance.get_ref(),
scheduled_execution_time=check.opt_inst_param(
scheduled_execution_time, "scheduled_execution_time", datetime),
)
def get_external_sensor_execution_data(
self,
instance: DagsterInstance,
repository_handle: RepositoryHandle,
name: str,
last_completion_time: Optional[float],
last_run_key: Optional[str],
) -> Union["ExternalSensorExecutionData", "ExternalSensorExecutionErrorData"]:
return get_external_sensor_execution(
self._recon_repo, instance.get_ref(), name, last_completion_time, last_run_key
)
def start_run_cancellation_thread(
instance: DagsterInstance, run_id
) -> Tuple[threading.Thread, threading.Event]:
print("Starting run cancellation thread") # pylint: disable=print-call
shutdown_event = threading.Event()
thread = threading.Thread(
target=_kill_on_cancel,
args=(instance.get_ref(), run_id, shutdown_event),
name="kill-on-cancel",
)
thread.start()
return thread, shutdown_event
def _get_execution_plan_from_run(pipeline: IPipeline,
pipeline_run: PipelineRun,
instance: DagsterInstance) -> ExecutionPlan:
if (
# need to rebuild execution plan so it matches the subsetted graph
pipeline.solids_to_execute is None
and pipeline_run.execution_plan_snapshot_id):
execution_plan_snapshot = instance.get_execution_plan_snapshot(
pipeline_run.execution_plan_snapshot_id)
if execution_plan_snapshot.can_reconstruct_plan:
return ExecutionPlan.rebuild_from_snapshot(
pipeline_run.pipeline_name,
execution_plan_snapshot,
)
return create_execution_plan(
pipeline,
run_config=pipeline_run.run_config,
mode=pipeline_run.mode,
step_keys_to_execute=pipeline_run.step_keys_to_execute,
instance_ref=instance.get_ref() if instance.is_persistent else None,
)
def execute_list_versions_command(instance: DagsterInstance,
kwargs: Dict[str, object]):
check.inst_param(instance, "instance", DagsterInstance)
config = list(
check.opt_tuple_param(kwargs.get("config"),
"config",
default=(),
of_type=str))
job_origin = get_pipeline_or_job_python_origin_from_kwargs(kwargs, True)
job = recon_pipeline_from_origin(job_origin)
run_config = get_run_config_from_file_list(config)
memoized_plan = create_execution_plan(
job,
run_config=run_config,
mode="default",
instance_ref=instance.get_ref(),
tags={MEMOIZED_RUN_TAG: "true"},
)
add_step_to_table(memoized_plan)
def get_external_schedule_execution_data(
self,
instance: DagsterInstance,
repository_handle: RepositoryHandle,
schedule_name: str,
scheduled_execution_time,
) -> Union["ExternalScheduleExecutionData", "ExternalScheduleExecutionErrorData"]:
check.inst_param(instance, "instance", DagsterInstance)
check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
check.str_param(schedule_name, "schedule_name")
check.opt_inst_param(scheduled_execution_time, "scheduled_execution_time", PendulumDateTime)
return get_external_schedule_execution(
self._recon_repo,
instance_ref=instance.get_ref(),
schedule_name=schedule_name,
scheduled_execution_timestamp=scheduled_execution_time.timestamp()
if scheduled_execution_time
else None,
scheduled_execution_timezone=scheduled_execution_time.timezone.name
if scheduled_execution_time
else None,
)