def sync_get_external_schedule_execution_data_grpc(
api_client: "DagsterGrpcClient",
instance: DagsterInstance,
repository_handle: RepositoryHandle,
schedule_name: str,
scheduled_execution_time: Any,
) -> ScheduleExecutionData:
check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
check.str_param(schedule_name, "schedule_name")
check.opt_inst_param(scheduled_execution_time, "scheduled_execution_time",
PendulumDateTime)
origin = repository_handle.get_external_origin()
result = deserialize_as(
api_client.external_schedule_execution(
external_schedule_execution_args=ExternalScheduleExecutionArgs(
repository_origin=origin,
instance_ref=instance.get_ref(),
schedule_name=schedule_name,
scheduled_execution_timestamp=scheduled_execution_time.
timestamp() if scheduled_execution_time else None,
scheduled_execution_timezone=scheduled_execution_time.timezone.
name if scheduled_execution_time else None,
)),
(ScheduleExecutionData, ExternalScheduleExecutionErrorData),
)
if isinstance(result, ExternalScheduleExecutionErrorData):
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def _get_grpc_endpoint(self, repository_location_origin):
origin_id = repository_location_origin.get_id()
if not origin_id in self._active_grpc_processes_or_errors:
try:
new_server_id = str(uuid.uuid4())
server_process = GrpcServerProcess(
loadable_target_origin=repository_location_origin.
loadable_target_origin,
heartbeat=True,
heartbeat_timeout=self._heartbeat_ttl,
fixed_server_id=new_server_id,
)
self._all_processes.append(server_process)
except Exception: # pylint: disable=broad-except
server_process = serializable_error_info_from_exc_info(
sys.exc_info())
new_server_id = None
self._active_grpc_processes_or_errors[origin_id] = (
server_process,
pendulum.now("UTC").timestamp(),
new_server_id,
)
process, _creation_timestamp, server_id = self._active_grpc_processes_or_errors[
origin_id]
if isinstance(process, SerializableErrorInfo):
raise DagsterUserCodeProcessError(
process.to_string(), user_code_process_error_infos=[process])
return GrpcServerEndpoint(server_id=server_id,
host="localhost",
port=process.port,
socket=process.socket)
def sync_get_external_partition_set_execution_param_data_grpc(
api_client: "DagsterGrpcClient",
repository_handle: RepositoryHandle,
partition_set_name: str,
partition_names: List[str],
) -> ExternalPartitionSetExecutionParamData:
from dagster.grpc.client import DagsterGrpcClient
check.inst_param(api_client, "api_client", DagsterGrpcClient)
check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
check.str_param(partition_set_name, "partition_set_name")
check.list_param(partition_names, "partition_names", of_type=str)
repository_origin = repository_handle.get_external_origin()
result = deserialize_as(
api_client.external_partition_set_execution_params(
partition_set_execution_param_args=PartitionSetExecutionParamArgs(
repository_origin=repository_origin,
partition_set_name=partition_set_name,
partition_names=partition_names,
), ),
(ExternalPartitionSetExecutionParamData,
ExternalPartitionExecutionErrorData),
)
if isinstance(result, ExternalPartitionExecutionErrorData):
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def sync_get_external_pipeline_subset_grpc(
api_client: "DagsterGrpcClient",
pipeline_origin: ExternalPipelineOrigin,
solid_selection: Optional[List[str]] = None,
) -> ExternalPipelineSubsetResult:
from dagster.grpc.client import DagsterGrpcClient
check.inst_param(api_client, "api_client", DagsterGrpcClient)
pipeline_origin = check.inst_param(pipeline_origin, "pipeline_origin",
ExternalPipelineOrigin)
solid_selection = check.opt_list_param(solid_selection,
"solid_selection",
of_type=str)
result = deserialize_as(
api_client.external_pipeline_subset(
pipeline_subset_snapshot_args=PipelineSubsetSnapshotArgs(
pipeline_origin=pipeline_origin,
solid_selection=solid_selection), ),
ExternalPipelineSubsetResult,
)
if result.error:
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def sync_get_external_partition_tags_grpc(
api_client, repository_handle, partition_set_name, partition_name
):
from dagster.grpc.client import DagsterGrpcClient
check.inst_param(api_client, "api_client", DagsterGrpcClient)
check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
check.str_param(partition_set_name, "partition_set_name")
check.str_param(partition_name, "partition_name")
repository_origin = repository_handle.get_external_origin()
result = check.inst(
deserialize_json_to_dagster_namedtuple(
api_client.external_partition_tags(
partition_args=PartitionArgs(
repository_origin=repository_origin,
partition_set_name=partition_set_name,
partition_name=partition_name,
),
),
),
(ExternalPartitionTagsData, ExternalPartitionExecutionErrorData),
)
if isinstance(result, ExternalPartitionExecutionErrorData):
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def sync_get_external_sensor_execution_data_grpc(
api_client: "DagsterGrpcClient",
instance: "DagsterInstance",
repository_handle: RepositoryHandle,
sensor_name: str,
last_completion_time: Optional[float],
last_run_key: Optional[str],
cursor: Optional[str],
) -> SensorExecutionData:
check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
check.str_param(sensor_name, "sensor_name")
check.opt_float_param(last_completion_time, "last_completion_time")
check.opt_str_param(last_run_key, "last_run_key")
check.opt_str_param(cursor, "cursor")
origin = repository_handle.get_external_origin()
result = deserialize_as(
api_client.external_sensor_execution(
sensor_execution_args=SensorExecutionArgs(
repository_origin=origin,
instance_ref=instance.get_ref(),
sensor_name=sensor_name,
last_completion_time=last_completion_time,
last_run_key=last_run_key,
cursor=cursor,
)
),
(SensorExecutionData, ExternalSensorExecutionErrorData),
)
if isinstance(result, ExternalSensorExecutionErrorData):
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def sync_get_external_sensor_execution_data_grpc(api_client, instance,
repository_handle,
sensor_name,
last_completion_time,
last_run_key, cursor):
check.inst_param(repository_handle, "repository_handle", RepositoryHandle)
check.str_param(sensor_name, "sensor_name")
check.opt_float_param(last_completion_time, "last_completion_time")
check.opt_str_param(last_run_key, "last_run_key")
origin = repository_handle.get_external_origin()
result = check.inst(
deserialize_json_to_dagster_namedtuple(
api_client.external_sensor_execution(
sensor_execution_args=SensorExecutionArgs(
repository_origin=origin,
instance_ref=instance.get_ref(),
sensor_name=sensor_name,
last_completion_time=last_completion_time,
last_run_key=last_run_key,
cursor=cursor,
)), ),
(SensorExecutionData, ExternalSensorExecutionErrorData),
)
if isinstance(result, ExternalSensorExecutionErrorData):
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def _get_grpc_endpoint(
self,
repository_location_origin: ManagedGrpcPythonEnvRepositoryLocationOrigin
) -> GrpcServerEndpoint:
origin_id = repository_location_origin.get_id()
loadable_target_origin = self._get_loadable_target_origin(
repository_location_origin)
if not loadable_target_origin:
raise Exception(
f"No Python file/module information available for location {repository_location_origin.location_name}"
)
if not origin_id in self._active_entries:
refresh_server = True
else:
active_entry = self._active_entries[origin_id]
refresh_server = loadable_target_origin != active_entry.loadable_target_origin
server_process: Union[GrpcServerProcess, SerializableErrorInfo]
new_server_id: Optional[str]
if refresh_server:
try:
new_server_id = str(uuid.uuid4())
server_process = GrpcServerProcess(
loadable_target_origin=loadable_target_origin,
heartbeat=True,
heartbeat_timeout=self._heartbeat_ttl,
fixed_server_id=new_server_id,
startup_timeout=self._startup_timeout,
)
self._all_processes.append(server_process)
except Exception:
server_process = serializable_error_info_from_exc_info(
sys.exc_info())
new_server_id = None
self._active_entries[origin_id] = ProcessRegistryEntry(
process_or_error=server_process,
loadable_target_origin=loadable_target_origin,
creation_timestamp=pendulum.now("UTC").timestamp(),
server_id=new_server_id,
)
active_entry = self._active_entries[origin_id]
if isinstance(active_entry.process_or_error, SerializableErrorInfo):
raise DagsterUserCodeProcessError(
active_entry.process_or_error.to_string(),
user_code_process_error_infos=[active_entry.process_or_error],
)
return GrpcServerEndpoint(
server_id=active_entry.server_id,
host="localhost",
port=active_entry.process_or_error.port,
socket=active_entry.process_or_error.socket,
)
def sync_get_server_id(api_client: "DagsterGrpcClient") -> str:
from dagster.grpc.client import DagsterGrpcClient
check.inst_param(api_client, "api_client", DagsterGrpcClient)
result = check.inst(api_client.get_server_id(), (str, SerializableErrorInfo))
if isinstance(result, SerializableErrorInfo):
raise DagsterUserCodeProcessError(
result.to_string(), user_code_process_error_infos=[result]
)
else:
return result
def sync_list_repositories_grpc(api_client):
from dagster.grpc.client import DagsterGrpcClient
from dagster.grpc.types import ListRepositoriesResponse
check.inst_param(api_client, "api_client", DagsterGrpcClient)
result = check.inst(api_client.list_repositories(),
(ListRepositoriesResponse, SerializableErrorInfo))
if isinstance(result, SerializableErrorInfo):
raise DagsterUserCodeProcessError(
result.to_string(), user_code_process_error_infos=[result])
else:
return result
def sync_get_external_execution_plan_grpc(
api_client: "DagsterGrpcClient",
pipeline_origin: ExternalPipelineOrigin,
run_config: Mapping[str, Any],
mode: str,
pipeline_snapshot_id: str,
solid_selection: Optional[List[str]] = None,
step_keys_to_execute: Optional[List[str]] = None,
known_state: Optional[KnownExecutionState] = None,
instance: Optional[DagsterInstance] = None,
) -> ExecutionPlanSnapshot:
from dagster.grpc.client import DagsterGrpcClient
check.inst_param(api_client, "api_client", DagsterGrpcClient)
check.inst_param(pipeline_origin, "pipeline_origin",
ExternalPipelineOrigin)
solid_selection = check.opt_list_param(solid_selection,
"solid_selection",
of_type=str)
run_config = check.dict_param(run_config, "run_config", key_type=str)
check.str_param(mode, "mode")
check.opt_nullable_list_param(step_keys_to_execute,
"step_keys_to_execute",
of_type=str)
check.str_param(pipeline_snapshot_id, "pipeline_snapshot_id")
check.opt_inst_param(known_state, "known_state", KnownExecutionState)
check.opt_inst_param(instance, "instance", DagsterInstance)
result = deserialize_as(
api_client.execution_plan_snapshot(
execution_plan_snapshot_args=ExecutionPlanSnapshotArgs(
pipeline_origin=pipeline_origin,
solid_selection=solid_selection,
run_config=run_config,
mode=mode,
step_keys_to_execute=step_keys_to_execute,
pipeline_snapshot_id=pipeline_snapshot_id,
known_state=known_state,
instance_ref=instance.get_ref(
) if instance and instance.is_persistent else None,
)),
(ExecutionPlanSnapshot, ExecutionPlanSnapshotErrorData),
)
if isinstance(result, ExecutionPlanSnapshotErrorData):
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def wait_for_grpc_server(server_process, ipc_output_file, timeout=60):
event = read_unary_response(ipc_output_file,
timeout=timeout,
ipc_process=server_process)
if isinstance(event, GrpcServerFailedToBindEvent):
raise CouldNotBindGrpcServerToAddress()
elif isinstance(event, GrpcServerLoadErrorEvent):
raise DagsterUserCodeProcessError(
event.error_info.to_string(),
user_code_process_error_infos=[event.error_info])
elif isinstance(event, GrpcServerStartedEvent):
return True
else:
raise Exception(
"Received unexpected IPC event from gRPC Server: {event}".format(
event=event))
def sync_get_external_execution_plan_grpc(
api_client,
pipeline_origin,
run_config,
mode,
pipeline_snapshot_id,
solid_selection=None,
step_keys_to_execute=None,
known_state=None,
):
from dagster.grpc.client import DagsterGrpcClient
check.inst_param(api_client, "api_client", DagsterGrpcClient)
check.inst_param(pipeline_origin, "pipeline_origin",
ExternalPipelineOrigin)
check.opt_list_param(solid_selection, "solid_selection", of_type=str)
check.dict_param(run_config, "run_config")
check.str_param(mode, "mode")
check.opt_nullable_list_param(step_keys_to_execute,
"step_keys_to_execute",
of_type=str)
check.str_param(pipeline_snapshot_id, "pipeline_snapshot_id")
check.opt_inst_param(known_state, "known_state", KnownExecutionState)
result = check.inst(
deserialize_json_to_dagster_namedtuple(
api_client.execution_plan_snapshot(
execution_plan_snapshot_args=ExecutionPlanSnapshotArgs(
pipeline_origin=pipeline_origin,
solid_selection=solid_selection,
run_config=run_config,
mode=mode,
step_keys_to_execute=step_keys_to_execute,
pipeline_snapshot_id=pipeline_snapshot_id,
known_state=known_state,
)), ),
(ExecutionPlanSnapshot, ExecutionPlanSnapshotErrorData),
)
if isinstance(result, ExecutionPlanSnapshotErrorData):
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
def sync_list_repositories(executable_path, python_file, module_name,
working_directory, attribute):
from dagster.grpc.types import ListRepositoriesResponse, ListRepositoriesInput
result = check.inst(
execute_unary_api_cli_command(
executable_path,
'list_repositories',
ListRepositoriesInput(
module_name=module_name,
python_file=python_file,
working_directory=working_directory,
attribute=attribute,
),
),
(ListRepositoriesResponse, SerializableErrorInfo),
)
if isinstance(result, SerializableErrorInfo):
raise DagsterUserCodeProcessError(
result.to_string(), user_code_process_error_infos=[result])
else:
return result
def sync_get_external_pipeline_subset_grpc(api_client,
pipeline_origin,
solid_selection=None):
from dagster.grpc.client import DagsterGrpcClient
check.inst_param(api_client, "api_client", DagsterGrpcClient)
check.inst_param(pipeline_origin, "pipeline_origin",
ExternalPipelineOrigin)
check.opt_list_param(solid_selection, "solid_selection", of_type=str)
result = check.inst(
deserialize_json_to_dagster_namedtuple(
api_client.external_pipeline_subset(
pipeline_subset_snapshot_args=PipelineSubsetSnapshotArgs(
pipeline_origin=pipeline_origin,
solid_selection=solid_selection), ), ),
ExternalPipelineSubsetResult,
)
if result.error:
raise DagsterUserCodeProcessError.from_error_info(result.error)
return result
