예제 #1
0
def build_sensor_context(
        instance: DagsterInstance,
        cursor: Optional[str] = None) -> SensorExecutionContext:
    """Builds sensor execution context using the provided parameters.

    The instance provided to ``build_sensor_context`` must be persistent;
    DagsterInstance.ephemeral() will result in an error.

    Args:
        instance (DagsterInstance): The dagster instance configured to run the sensor.

    Examples:

        .. code-block:: python

            context = build_sensor_context(instance)
            my_sensor.get_execution_data(context)

    """

    experimental_fn_warning("build_sensor_context")

    check.inst_param(instance, "instance", DagsterInstance)
    check.opt_str_param(cursor, "cursor")
    return SensorExecutionContext(
        instance_ref=instance.get_ref(),
        last_completion_time=None,
        last_run_key=None,
        cursor=cursor,
    )
예제 #2
0
def sync_get_external_schedule_execution_data_grpc(
    api_client: "DagsterGrpcClient",
    instance: DagsterInstance,
    repository_handle: RepositoryHandle,
    schedule_name: str,
    scheduled_execution_time: Any,
) -> ScheduleExecutionData:
    check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
    check.str_param(schedule_name, "schedule_name")
    check.opt_inst_param(scheduled_execution_time, "scheduled_execution_time",
                         PendulumDateTime)

    origin = repository_handle.get_external_origin()
    result = deserialize_as(
        api_client.external_schedule_execution(
            external_schedule_execution_args=ExternalScheduleExecutionArgs(
                repository_origin=origin,
                instance_ref=instance.get_ref(),
                schedule_name=schedule_name,
                scheduled_execution_timestamp=scheduled_execution_time.
                timestamp() if scheduled_execution_time else None,
                scheduled_execution_timezone=scheduled_execution_time.timezone.
                name if scheduled_execution_time else None,
            )),
        (ScheduleExecutionData, ExternalScheduleExecutionErrorData),
    )
    if isinstance(result, ExternalScheduleExecutionErrorData):
        raise DagsterUserCodeProcessError.from_error_info(result.error)

    return result
예제 #3
0
def execute_list_versions_command(instance: DagsterInstance,
                                  kwargs: Dict[str, Any]) -> None:
    check.inst_param(instance, "instance", DagsterInstance)

    config = list(
        check.opt_tuple_param(kwargs.get("config"),
                              "config",
                              default=tuple(),
                              of_type=str))
    preset = kwargs.get("preset")
    mode = kwargs.get("mode")

    if preset and config:
        raise click.UsageError("Can not use --preset with --config.")

    pipeline_origin = get_pipeline_or_job_python_origin_from_kwargs(kwargs)
    pipeline = recon_pipeline_from_origin(pipeline_origin)
    run_config = get_run_config_from_file_list(config)

    memoized_plan = create_execution_plan(
        pipeline,
        run_config=run_config,
        mode=mode,
        instance_ref=instance.get_ref(),
        tags={MEMOIZED_RUN_TAG: "true"},
    )
    add_step_to_table(memoized_plan)
예제 #4
0
def build_schedule_context(
    instance: DagsterInstance,
    scheduled_execution_time: Optional[datetime] = None
) -> ScheduleExecutionContext:
    """Builds schedule execution context using the provided parameters.

    The instance provided to ``build_schedule_context`` must be persistent;
    DagsterInstance.ephemeral() will result in an error.

    Args:
        instance (DagsterInstance): The dagster instance configured to run the schedule.
        scheduled_execution_time (datetime): The time in which the execution was scheduled to
            happen. May differ slightly from both the actual execution time and the time at which
            the run config is computed.

    Examples:

        .. code-block:: python

            context = build_schedule_context(instance)
            daily_schedule.get_execution_data(context)

    """

    experimental_fn_warning("build_schedule_context")

    check.inst_param(instance, "instance", DagsterInstance)
    return ScheduleExecutionContext(
        instance_ref=instance.get_ref(),
        scheduled_execution_time=check.opt_inst_param(
            scheduled_execution_time, "scheduled_execution_time", datetime),
    )
예제 #5
0
 def get_external_sensor_execution_data(
     self,
     instance: DagsterInstance,
     repository_handle: RepositoryHandle,
     name: str,
     last_completion_time: Optional[float],
     last_run_key: Optional[str],
 ) -> Union["ExternalSensorExecutionData", "ExternalSensorExecutionErrorData"]:
     return get_external_sensor_execution(
         self._recon_repo, instance.get_ref(), name, last_completion_time, last_run_key
     )
예제 #6
0
def start_run_cancellation_thread(
    instance: DagsterInstance, run_id
) -> Tuple[threading.Thread, threading.Event]:
    print("Starting run cancellation thread")  # pylint: disable=print-call
    shutdown_event = threading.Event()
    thread = threading.Thread(
        target=_kill_on_cancel,
        args=(instance.get_ref(), run_id, shutdown_event),
        name="kill-on-cancel",
    )
    thread.start()
    return thread, shutdown_event
예제 #7
0
파일: api.py 프로젝트: amarrella/dagster
def _get_execution_plan_from_run(pipeline: IPipeline,
                                 pipeline_run: PipelineRun,
                                 instance: DagsterInstance) -> ExecutionPlan:
    if (
            # need to rebuild execution plan so it matches the subsetted graph
            pipeline.solids_to_execute is None
            and pipeline_run.execution_plan_snapshot_id):
        execution_plan_snapshot = instance.get_execution_plan_snapshot(
            pipeline_run.execution_plan_snapshot_id)
        if execution_plan_snapshot.can_reconstruct_plan:
            return ExecutionPlan.rebuild_from_snapshot(
                pipeline_run.pipeline_name,
                execution_plan_snapshot,
            )
    return create_execution_plan(
        pipeline,
        run_config=pipeline_run.run_config,
        mode=pipeline_run.mode,
        step_keys_to_execute=pipeline_run.step_keys_to_execute,
        instance_ref=instance.get_ref() if instance.is_persistent else None,
    )
예제 #8
0
def execute_list_versions_command(instance: DagsterInstance,
                                  kwargs: Dict[str, object]):
    check.inst_param(instance, "instance", DagsterInstance)

    config = list(
        check.opt_tuple_param(kwargs.get("config"),
                              "config",
                              default=(),
                              of_type=str))

    job_origin = get_pipeline_or_job_python_origin_from_kwargs(kwargs, True)
    job = recon_pipeline_from_origin(job_origin)
    run_config = get_run_config_from_file_list(config)

    memoized_plan = create_execution_plan(
        job,
        run_config=run_config,
        mode="default",
        instance_ref=instance.get_ref(),
        tags={MEMOIZED_RUN_TAG: "true"},
    )

    add_step_to_table(memoized_plan)
예제 #9
0
    def get_external_schedule_execution_data(
        self,
        instance: DagsterInstance,
        repository_handle: RepositoryHandle,
        schedule_name: str,
        scheduled_execution_time,
    ) -> Union["ExternalScheduleExecutionData", "ExternalScheduleExecutionErrorData"]:
        check.inst_param(instance, "instance", DagsterInstance)
        check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
        check.str_param(schedule_name, "schedule_name")
        check.opt_inst_param(scheduled_execution_time, "scheduled_execution_time", PendulumDateTime)

        return get_external_schedule_execution(
            self._recon_repo,
            instance_ref=instance.get_ref(),
            schedule_name=schedule_name,
            scheduled_execution_timestamp=scheduled_execution_time.timestamp()
            if scheduled_execution_time
            else None,
            scheduled_execution_timezone=scheduled_execution_time.timezone.name
            if scheduled_execution_time
            else None,
        )