def sync_get_external_schedule_execution_data_grpc( api_client: "DagsterGrpcClient", instance: DagsterInstance, repository_handle: RepositoryHandle, schedule_name: str, scheduled_execution_time: Any, ) -> ScheduleExecutionData: check.inst_param(repository_handle, "repository_handle", RepositoryHandle) check.str_param(schedule_name, "schedule_name") check.opt_inst_param(scheduled_execution_time, "scheduled_execution_time", PendulumDateTime) origin = repository_handle.get_external_origin() result = deserialize_as( api_client.external_schedule_execution( external_schedule_execution_args=ExternalScheduleExecutionArgs( repository_origin=origin, instance_ref=instance.get_ref(), schedule_name=schedule_name, scheduled_execution_timestamp=scheduled_execution_time. timestamp() if scheduled_execution_time else None, scheduled_execution_timezone=scheduled_execution_time.timezone. name if scheduled_execution_time else None, )), (ScheduleExecutionData, ExternalScheduleExecutionErrorData), ) if isinstance(result, ExternalScheduleExecutionErrorData): raise DagsterUserCodeProcessError.from_error_info(result.error) return result
def _get_grpc_endpoint(self, repository_location_origin): origin_id = repository_location_origin.get_id() if not origin_id in self._active_grpc_processes_or_errors: try: new_server_id = str(uuid.uuid4()) server_process = GrpcServerProcess( loadable_target_origin=repository_location_origin. loadable_target_origin, heartbeat=True, heartbeat_timeout=self._heartbeat_ttl, fixed_server_id=new_server_id, ) self._all_processes.append(server_process) except Exception: # pylint: disable=broad-except server_process = serializable_error_info_from_exc_info( sys.exc_info()) new_server_id = None self._active_grpc_processes_or_errors[origin_id] = ( server_process, pendulum.now("UTC").timestamp(), new_server_id, ) process, _creation_timestamp, server_id = self._active_grpc_processes_or_errors[ origin_id] if isinstance(process, SerializableErrorInfo): raise DagsterUserCodeProcessError( process.to_string(), user_code_process_error_infos=[process]) return GrpcServerEndpoint(server_id=server_id, host="localhost", port=process.port, socket=process.socket)
def sync_get_external_partition_set_execution_param_data_grpc( api_client: "DagsterGrpcClient", repository_handle: RepositoryHandle, partition_set_name: str, partition_names: List[str], ) -> ExternalPartitionSetExecutionParamData: from dagster.grpc.client import DagsterGrpcClient check.inst_param(api_client, "api_client", DagsterGrpcClient) check.inst_param(repository_handle, "repository_handle", RepositoryHandle) check.str_param(partition_set_name, "partition_set_name") check.list_param(partition_names, "partition_names", of_type=str) repository_origin = repository_handle.get_external_origin() result = deserialize_as( api_client.external_partition_set_execution_params( partition_set_execution_param_args=PartitionSetExecutionParamArgs( repository_origin=repository_origin, partition_set_name=partition_set_name, partition_names=partition_names, ), ), (ExternalPartitionSetExecutionParamData, ExternalPartitionExecutionErrorData), ) if isinstance(result, ExternalPartitionExecutionErrorData): raise DagsterUserCodeProcessError.from_error_info(result.error) return result
def sync_get_external_pipeline_subset_grpc( api_client: "DagsterGrpcClient", pipeline_origin: ExternalPipelineOrigin, solid_selection: Optional[List[str]] = None, ) -> ExternalPipelineSubsetResult: from dagster.grpc.client import DagsterGrpcClient check.inst_param(api_client, "api_client", DagsterGrpcClient) pipeline_origin = check.inst_param(pipeline_origin, "pipeline_origin", ExternalPipelineOrigin) solid_selection = check.opt_list_param(solid_selection, "solid_selection", of_type=str) result = deserialize_as( api_client.external_pipeline_subset( pipeline_subset_snapshot_args=PipelineSubsetSnapshotArgs( pipeline_origin=pipeline_origin, solid_selection=solid_selection), ), ExternalPipelineSubsetResult, ) if result.error: raise DagsterUserCodeProcessError.from_error_info(result.error) return result
def sync_get_external_partition_tags_grpc( api_client, repository_handle, partition_set_name, partition_name ): from dagster.grpc.client import DagsterGrpcClient check.inst_param(api_client, "api_client", DagsterGrpcClient) check.inst_param(repository_handle, "repository_handle", RepositoryHandle) check.str_param(partition_set_name, "partition_set_name") check.str_param(partition_name, "partition_name") repository_origin = repository_handle.get_external_origin() result = check.inst( deserialize_json_to_dagster_namedtuple( api_client.external_partition_tags( partition_args=PartitionArgs( repository_origin=repository_origin, partition_set_name=partition_set_name, partition_name=partition_name, ), ), ), (ExternalPartitionTagsData, ExternalPartitionExecutionErrorData), ) if isinstance(result, ExternalPartitionExecutionErrorData): raise DagsterUserCodeProcessError.from_error_info(result.error) return result
def sync_get_external_sensor_execution_data_grpc( api_client: "DagsterGrpcClient", instance: "DagsterInstance", repository_handle: RepositoryHandle, sensor_name: str, last_completion_time: Optional[float], last_run_key: Optional[str], cursor: Optional[str], ) -> SensorExecutionData: check.inst_param(repository_handle, "repository_handle", RepositoryHandle) check.str_param(sensor_name, "sensor_name") check.opt_float_param(last_completion_time, "last_completion_time") check.opt_str_param(last_run_key, "last_run_key") check.opt_str_param(cursor, "cursor") origin = repository_handle.get_external_origin() result = deserialize_as( api_client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=origin, instance_ref=instance.get_ref(), sensor_name=sensor_name, last_completion_time=last_completion_time, last_run_key=last_run_key, cursor=cursor, ) ), (SensorExecutionData, ExternalSensorExecutionErrorData), ) if isinstance(result, ExternalSensorExecutionErrorData): raise DagsterUserCodeProcessError.from_error_info(result.error) return result
def sync_get_external_sensor_execution_data_grpc(api_client, instance, repository_handle, sensor_name, last_completion_time, last_run_key, cursor): check.inst_param(repository_handle, "repository_handle", RepositoryHandle) check.str_param(sensor_name, "sensor_name") check.opt_float_param(last_completion_time, "last_completion_time") check.opt_str_param(last_run_key, "last_run_key") origin = repository_handle.get_external_origin() result = check.inst( deserialize_json_to_dagster_namedtuple( api_client.external_sensor_execution( sensor_execution_args=SensorExecutionArgs( repository_origin=origin, instance_ref=instance.get_ref(), sensor_name=sensor_name, last_completion_time=last_completion_time, last_run_key=last_run_key, cursor=cursor, )), ), (SensorExecutionData, ExternalSensorExecutionErrorData), ) if isinstance(result, ExternalSensorExecutionErrorData): raise DagsterUserCodeProcessError.from_error_info(result.error) return result
def _get_grpc_endpoint( self, repository_location_origin: ManagedGrpcPythonEnvRepositoryLocationOrigin ) -> GrpcServerEndpoint: origin_id = repository_location_origin.get_id() loadable_target_origin = self._get_loadable_target_origin( repository_location_origin) if not loadable_target_origin: raise Exception( f"No Python file/module information available for location {repository_location_origin.location_name}" ) if not origin_id in self._active_entries: refresh_server = True else: active_entry = self._active_entries[origin_id] refresh_server = loadable_target_origin != active_entry.loadable_target_origin server_process: Union[GrpcServerProcess, SerializableErrorInfo] new_server_id: Optional[str] if refresh_server: try: new_server_id = str(uuid.uuid4()) server_process = GrpcServerProcess( loadable_target_origin=loadable_target_origin, heartbeat=True, heartbeat_timeout=self._heartbeat_ttl, fixed_server_id=new_server_id, startup_timeout=self._startup_timeout, ) self._all_processes.append(server_process) except Exception: server_process = serializable_error_info_from_exc_info( sys.exc_info()) new_server_id = None self._active_entries[origin_id] = ProcessRegistryEntry( process_or_error=server_process, loadable_target_origin=loadable_target_origin, creation_timestamp=pendulum.now("UTC").timestamp(), server_id=new_server_id, ) active_entry = self._active_entries[origin_id] if isinstance(active_entry.process_or_error, SerializableErrorInfo): raise DagsterUserCodeProcessError( active_entry.process_or_error.to_string(), user_code_process_error_infos=[active_entry.process_or_error], ) return GrpcServerEndpoint( server_id=active_entry.server_id, host="localhost", port=active_entry.process_or_error.port, socket=active_entry.process_or_error.socket, )
def sync_get_server_id(api_client: "DagsterGrpcClient") -> str: from dagster.grpc.client import DagsterGrpcClient check.inst_param(api_client, "api_client", DagsterGrpcClient) result = check.inst(api_client.get_server_id(), (str, SerializableErrorInfo)) if isinstance(result, SerializableErrorInfo): raise DagsterUserCodeProcessError( result.to_string(), user_code_process_error_infos=[result] ) else: return result
def sync_list_repositories_grpc(api_client): from dagster.grpc.client import DagsterGrpcClient from dagster.grpc.types import ListRepositoriesResponse check.inst_param(api_client, "api_client", DagsterGrpcClient) result = check.inst(api_client.list_repositories(), (ListRepositoriesResponse, SerializableErrorInfo)) if isinstance(result, SerializableErrorInfo): raise DagsterUserCodeProcessError( result.to_string(), user_code_process_error_infos=[result]) else: return result
def sync_get_external_execution_plan_grpc( api_client: "DagsterGrpcClient", pipeline_origin: ExternalPipelineOrigin, run_config: Mapping[str, Any], mode: str, pipeline_snapshot_id: str, solid_selection: Optional[List[str]] = None, step_keys_to_execute: Optional[List[str]] = None, known_state: Optional[KnownExecutionState] = None, instance: Optional[DagsterInstance] = None, ) -> ExecutionPlanSnapshot: from dagster.grpc.client import DagsterGrpcClient check.inst_param(api_client, "api_client", DagsterGrpcClient) check.inst_param(pipeline_origin, "pipeline_origin", ExternalPipelineOrigin) solid_selection = check.opt_list_param(solid_selection, "solid_selection", of_type=str) run_config = check.dict_param(run_config, "run_config", key_type=str) check.str_param(mode, "mode") check.opt_nullable_list_param(step_keys_to_execute, "step_keys_to_execute", of_type=str) check.str_param(pipeline_snapshot_id, "pipeline_snapshot_id") check.opt_inst_param(known_state, "known_state", KnownExecutionState) check.opt_inst_param(instance, "instance", DagsterInstance) result = deserialize_as( api_client.execution_plan_snapshot( execution_plan_snapshot_args=ExecutionPlanSnapshotArgs( pipeline_origin=pipeline_origin, solid_selection=solid_selection, run_config=run_config, mode=mode, step_keys_to_execute=step_keys_to_execute, pipeline_snapshot_id=pipeline_snapshot_id, known_state=known_state, instance_ref=instance.get_ref( ) if instance and instance.is_persistent else None, )), (ExecutionPlanSnapshot, ExecutionPlanSnapshotErrorData), ) if isinstance(result, ExecutionPlanSnapshotErrorData): raise DagsterUserCodeProcessError.from_error_info(result.error) return result
def wait_for_grpc_server(server_process, ipc_output_file, timeout=60): event = read_unary_response(ipc_output_file, timeout=timeout, ipc_process=server_process) if isinstance(event, GrpcServerFailedToBindEvent): raise CouldNotBindGrpcServerToAddress() elif isinstance(event, GrpcServerLoadErrorEvent): raise DagsterUserCodeProcessError( event.error_info.to_string(), user_code_process_error_infos=[event.error_info]) elif isinstance(event, GrpcServerStartedEvent): return True else: raise Exception( "Received unexpected IPC event from gRPC Server: {event}".format( event=event))
def sync_get_external_execution_plan_grpc( api_client, pipeline_origin, run_config, mode, pipeline_snapshot_id, solid_selection=None, step_keys_to_execute=None, known_state=None, ): from dagster.grpc.client import DagsterGrpcClient check.inst_param(api_client, "api_client", DagsterGrpcClient) check.inst_param(pipeline_origin, "pipeline_origin", ExternalPipelineOrigin) check.opt_list_param(solid_selection, "solid_selection", of_type=str) check.dict_param(run_config, "run_config") check.str_param(mode, "mode") check.opt_nullable_list_param(step_keys_to_execute, "step_keys_to_execute", of_type=str) check.str_param(pipeline_snapshot_id, "pipeline_snapshot_id") check.opt_inst_param(known_state, "known_state", KnownExecutionState) result = check.inst( deserialize_json_to_dagster_namedtuple( api_client.execution_plan_snapshot( execution_plan_snapshot_args=ExecutionPlanSnapshotArgs( pipeline_origin=pipeline_origin, solid_selection=solid_selection, run_config=run_config, mode=mode, step_keys_to_execute=step_keys_to_execute, pipeline_snapshot_id=pipeline_snapshot_id, known_state=known_state, )), ), (ExecutionPlanSnapshot, ExecutionPlanSnapshotErrorData), ) if isinstance(result, ExecutionPlanSnapshotErrorData): raise DagsterUserCodeProcessError.from_error_info(result.error) return result
def sync_list_repositories(executable_path, python_file, module_name, working_directory, attribute): from dagster.grpc.types import ListRepositoriesResponse, ListRepositoriesInput result = check.inst( execute_unary_api_cli_command( executable_path, 'list_repositories', ListRepositoriesInput( module_name=module_name, python_file=python_file, working_directory=working_directory, attribute=attribute, ), ), (ListRepositoriesResponse, SerializableErrorInfo), ) if isinstance(result, SerializableErrorInfo): raise DagsterUserCodeProcessError( result.to_string(), user_code_process_error_infos=[result]) else: return result
def sync_get_external_pipeline_subset_grpc(api_client, pipeline_origin, solid_selection=None): from dagster.grpc.client import DagsterGrpcClient check.inst_param(api_client, "api_client", DagsterGrpcClient) check.inst_param(pipeline_origin, "pipeline_origin", ExternalPipelineOrigin) check.opt_list_param(solid_selection, "solid_selection", of_type=str) result = check.inst( deserialize_json_to_dagster_namedtuple( api_client.external_pipeline_subset( pipeline_subset_snapshot_args=PipelineSubsetSnapshotArgs( pipeline_origin=pipeline_origin, solid_selection=solid_selection), ), ), ExternalPipelineSubsetResult, ) if result.error: raise DagsterUserCodeProcessError.from_error_info(result.error) return result