def deserialize_objects(self, data_metadata_pairs, object_refs):
assert len(data_metadata_pairs) == len(object_refs)
results = []
for object_ref, (data, metadata) in zip(object_refs,
data_metadata_pairs):
assert self.get_outer_object_ref() is None
self.set_outer_object_ref(object_ref)
try:
obj = self._deserialize_object(data, metadata, object_ref)
except Exception as e:
logger.exception(e)
obj = RaySystemError(e, traceback.format_exc())
results.append(obj)
# Must clear ObjectRef to not hold a reference.
self.set_outer_object_ref(None)
return results
def deserialize_objects(self, data_metadata_pairs, object_refs):
assert len(data_metadata_pairs) == len(object_refs)
# initialize the thread-local field
if not hasattr(self._thread_local, "object_ref_stack"):
self._thread_local.object_ref_stack = []
results = []
for object_ref, (data, metadata) in zip(object_refs, data_metadata_pairs):
try:
# Push the object ref to the stack, so the object under
# the object ref knows where it comes from.
self._thread_local.object_ref_stack.append(object_ref)
obj = self._deserialize_object(data, metadata, object_ref)
except Exception as e:
logger.exception(e)
obj = RaySystemError(e, traceback.format_exc())
finally:
# Must clear ObjectRef to not hold a reference.
if self._thread_local.object_ref_stack:
self._thread_local.object_ref_stack.pop()
results.append(obj)
return results
def _deserialize_object(self, data, metadata, object_ref):
if metadata:
metadata_fields = metadata.split(b",")
if metadata_fields[0] in [
ray_constants.OBJECT_METADATA_TYPE_CROSS_LANGUAGE,
ray_constants.OBJECT_METADATA_TYPE_PYTHON,
]:
return self._deserialize_msgpack_data(data, metadata_fields)
# Check if the object should be returned as raw bytes.
if metadata_fields[0] == ray_constants.OBJECT_METADATA_TYPE_RAW:
if data is None:
return b""
return data.to_pybytes()
elif metadata_fields[0] == ray_constants.OBJECT_METADATA_TYPE_ACTOR_HANDLE:
obj = self._deserialize_msgpack_data(data, metadata_fields)
return _actor_handle_deserializer(obj)
# Otherwise, return an exception object based on
# the error type.
try:
error_type = int(metadata_fields[0])
except Exception:
raise Exception(
f"Can't deserialize object: {object_ref}, " f"metadata: {metadata}"
)
# RayTaskError is serialized with pickle5 in the data field.
# TODO (kfstorm): exception serialization should be language
# independent.
if error_type == ErrorType.Value("TASK_EXECUTION_EXCEPTION"):
obj = self._deserialize_msgpack_data(data, metadata_fields)
return RayError.from_bytes(obj)
elif error_type == ErrorType.Value("WORKER_DIED"):
return WorkerCrashedError()
elif error_type == ErrorType.Value("ACTOR_DIED"):
return self._deserialize_actor_died_error(data, metadata_fields)
elif error_type == ErrorType.Value("LOCAL_RAYLET_DIED"):
return LocalRayletDiedError()
elif error_type == ErrorType.Value("TASK_CANCELLED"):
return TaskCancelledError()
elif error_type == ErrorType.Value("OBJECT_LOST"):
return ObjectLostError(
object_ref.hex(), object_ref.owner_address(), object_ref.call_site()
)
elif error_type == ErrorType.Value("OBJECT_FETCH_TIMED_OUT"):
return ObjectFetchTimedOutError(
object_ref.hex(), object_ref.owner_address(), object_ref.call_site()
)
elif error_type == ErrorType.Value("OBJECT_DELETED"):
return ReferenceCountingAssertionError(
object_ref.hex(), object_ref.owner_address(), object_ref.call_site()
)
elif error_type == ErrorType.Value("OWNER_DIED"):
return OwnerDiedError(
object_ref.hex(), object_ref.owner_address(), object_ref.call_site()
)
elif error_type == ErrorType.Value("OBJECT_UNRECONSTRUCTABLE"):
return ObjectReconstructionFailedError(
object_ref.hex(), object_ref.owner_address(), object_ref.call_site()
)
elif error_type == ErrorType.Value(
"OBJECT_UNRECONSTRUCTABLE_MAX_ATTEMPTS_EXCEEDED"
):
return ObjectReconstructionFailedMaxAttemptsExceededError(
object_ref.hex(), object_ref.owner_address(), object_ref.call_site()
)
elif error_type == ErrorType.Value(
"OBJECT_UNRECONSTRUCTABLE_LINEAGE_EVICTED"
):
return ObjectReconstructionFailedLineageEvictedError(
object_ref.hex(), object_ref.owner_address(), object_ref.call_site()
)
elif error_type == ErrorType.Value("RUNTIME_ENV_SETUP_FAILED"):
error_info = self._deserialize_error_info(data, metadata_fields)
# TODO(sang): Assert instead once actor also reports error messages.
error_msg = ""
if error_info.HasField("runtime_env_setup_failed_error"):
error_msg = error_info.runtime_env_setup_failed_error.error_message
return RuntimeEnvSetupError(error_message=error_msg)
elif error_type == ErrorType.Value("TASK_PLACEMENT_GROUP_REMOVED"):
return TaskPlacementGroupRemoved()
elif error_type == ErrorType.Value("ACTOR_PLACEMENT_GROUP_REMOVED"):
return ActorPlacementGroupRemoved()
elif error_type == ErrorType.Value("TASK_UNSCHEDULABLE_ERROR"):
error_info = self._deserialize_error_info(data, metadata_fields)
return TaskUnschedulableError(error_info.error_message)
elif error_type == ErrorType.Value("ACTOR_UNSCHEDULABLE_ERROR"):
error_info = self._deserialize_error_info(data, metadata_fields)
return ActorUnschedulableError(error_info.error_message)
else:
return RaySystemError("Unrecognized error type " + str(error_type))
elif data:
raise ValueError("non-null object should always have metadata")
else:
# Object isn't available in plasma. This should never be returned
# to the user. We should only reach this line if this object was
# deserialized as part of a list, and another object in the list
# throws an exception.
return PlasmaObjectNotAvailable
def _deserialize_object(self, data, metadata, object_ref):
if metadata:
metadata_fields = metadata.split(b",")
if metadata_fields[0] in [
ray_constants.OBJECT_METADATA_TYPE_CROSS_LANGUAGE,
ray_constants.OBJECT_METADATA_TYPE_PYTHON
]:
return self._deserialize_msgpack_data(data, metadata_fields)
# Check if the object should be returned as raw bytes.
if metadata_fields[0] == ray_constants.OBJECT_METADATA_TYPE_RAW:
if data is None:
return b""
return data.to_pybytes()
elif metadata_fields[
0] == ray_constants.OBJECT_METADATA_TYPE_ACTOR_HANDLE:
obj = self._deserialize_msgpack_data(data, metadata_fields)
return _actor_handle_deserializer(obj)
# Otherwise, return an exception object based on
# the error type.
try:
error_type = int(metadata_fields[0])
except Exception:
raise Exception(f"Can't deserialize object: {object_ref}, "
f"metadata: {metadata}")
# RayTaskError is serialized with pickle5 in the data field.
# TODO (kfstorm): exception serialization should be language
# independent.
if error_type == ErrorType.Value("TASK_EXECUTION_EXCEPTION"):
obj = self._deserialize_msgpack_data(data, metadata_fields)
return RayError.from_bytes(obj)
elif error_type == ErrorType.Value("WORKER_DIED"):
return WorkerCrashedError()
elif error_type == ErrorType.Value("ACTOR_DIED"):
if data:
pb_bytes = self._deserialize_msgpack_data(
data, metadata_fields)
if pb_bytes:
return RayError.from_bytes(pb_bytes)
return RayActorError()
elif error_type == ErrorType.Value("TASK_CANCELLED"):
return TaskCancelledError()
elif error_type == ErrorType.Value("OBJECT_LOST"):
return ObjectLostError(object_ref.hex(),
object_ref.owner_address(),
object_ref.call_site())
elif error_type == ErrorType.Value("OBJECT_DELETED"):
return ReferenceCountingAssertionError(
object_ref.hex(), object_ref.owner_address(),
object_ref.call_site())
elif error_type == ErrorType.Value("OWNER_DIED"):
return OwnerDiedError(object_ref.hex(),
object_ref.owner_address(),
object_ref.call_site())
elif error_type == ErrorType.Value("OBJECT_UNRECONSTRUCTABLE"):
return ObjectReconstructionFailedError(
object_ref.hex(), object_ref.owner_address(),
object_ref.call_site())
elif error_type == ErrorType.Value("RUNTIME_ENV_SETUP_FAILED"):
return RuntimeEnvSetupError()
else:
return RaySystemError("Unrecognized error type " +
str(error_type))
elif data:
raise ValueError("non-null object should always have metadata")
else:
# Object isn't available in plasma. This should never be returned
# to the user. We should only reach this line if this object was
# deserialized as part of a list, and another object in the list
# throws an exception.
return PlasmaObjectNotAvailable