def dagnode_from_json(input_json: Any) -> Union[DAGNode, RayServeHandle, Any]:
"""
Decode a DAGNode from given input json dictionary. JSON serialization is
only used and enforced in ray serve from ray core API authored DAGNode(s).
Covers both RayServeHandle and DAGNode types.
Assumptions:
- User object's JSON dict does not have keys that collide with our
reserved DAGNODE_TYPE_KEY
- RayServeHandle and Deployment can be re-constructed without losing
states needed for their functionality or correctness.
- DAGNode type can be re-constructed with new stable_uuid upon each
deserialization without effective correctness of execution.
- Only exception is ClassNode used as parent of ClassMethodNode
that we perserve the same parent node.
- .options() does not contain any DAGNode type
"""
node_type_to_cls = {
# Ray DAG Inputs
InputNode.__name__: InputNode,
InputAttributeNode.__name__: InputAttributeNode,
# Deployment graph execution nodes
DeploymentExecutorNode.__name__: DeploymentExecutorNode,
DeploymentMethodExecutorNode.__name__: DeploymentMethodExecutorNode,
DeploymentFunctionExecutorNode.__name__:
DeploymentFunctionExecutorNode,
}
# Deserialize RayServeHandle type
if SERVE_HANDLE_JSON_KEY in input_json:
return serve_handle_from_json_dict(input_json)
# Base case for plain objects
elif DAGNODE_TYPE_KEY not in input_json:
return input_json
elif input_json[DAGNODE_TYPE_KEY] == RayServeDAGHandle.__name__:
return RayServeDAGHandle(input_json["dag_node_json"])
elif input_json[DAGNODE_TYPE_KEY] == "DeploymentSchema":
return DeploymentSchema.parse_obj(input_json["schema"])
elif input_json[DAGNODE_TYPE_KEY] == RayServeLazySyncHandle.__name__:
return RayServeLazySyncHandle(
input_json["deployment_name"],
HandleOptions(input_json["handle_options_method_name"]),
)
# Deserialize DAGNode type
elif input_json[DAGNODE_TYPE_KEY] in node_type_to_cls:
return node_type_to_cls[input_json[DAGNODE_TYPE_KEY]].from_json(
input_json)
else:
# Class and Function nodes require original module as body.
module_name, attr_name = parse_import_path(input_json["import_path"])
module = getattr(import_module(module_name), attr_name)
if input_json[DAGNODE_TYPE_KEY] == FunctionNode.__name__:
return FunctionNode.from_json(input_json, module)
elif input_json[DAGNODE_TYPE_KEY] == ClassNode.__name__:
return ClassNode.from_json(input_json, module)
def dagnode_from_json(input_json: Any) -> Union[DAGNode, RayServeHandle, Any]:
"""
Decode a DAGNode from given input json dictionary. JSON serialization is
only used and enforced in ray serve from ray core API authored DAGNode(s).
Covers both RayServeHandle and DAGNode types.
Assumptions:
- User object's JSON dict does not have keys that collide with our
reserved DAGNODE_TYPE_KEY
- RayServeHandle and Deployment can be re-constructed without losing
states needed for their functionality or correctness.
- DAGNode type can be re-constructed with new stable_uuid upon each
deserialization without effective correctness of execution.
- Only exception is ClassNode used as parent of ClassMethodNode
that we perserve the same parent node.
- .options() does not contain any DAGNode type
"""
# Deserialize RayServeHandle type
if SERVE_HANDLE_JSON_KEY in input_json:
return json.loads(input_json, object_hook=serve_handle_object_hook)
# Base case for plain objects
elif DAGNODE_TYPE_KEY not in input_json:
try:
return json.loads(input_json)
except Exception:
return input_json
# Deserialize DAGNode type
elif input_json[DAGNODE_TYPE_KEY] == InputNode.__name__:
return InputNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == InputAtrributeNode.__name__:
return InputAtrributeNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == PipelineInputNode.__name__:
return PipelineInputNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == ClassMethodNode.__name__:
return ClassMethodNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == DeploymentNode.__name__:
return DeploymentNode.from_json(input_json, object_hook=dagnode_from_json)
elif input_json[DAGNODE_TYPE_KEY] == DeploymentMethodNode.__name__:
return DeploymentMethodNode.from_json(input_json, object_hook=dagnode_from_json)
else:
# Class and Function nodes require original module as body.
print(f"import_path: {input_json['import_path']}")
module_name, attr_name = parse_import_path(input_json["import_path"])
module = getattr(import_module(module_name), attr_name)
if input_json[DAGNODE_TYPE_KEY] == FunctionNode.__name__:
return FunctionNode.from_json(
input_json, module, object_hook=dagnode_from_json
)
elif input_json[DAGNODE_TYPE_KEY] == ClassNode.__name__:
return ClassNode.from_json(
input_json, module, object_hook=dagnode_from_json
)
async def __init__(
self,
deployment_name,
replica_tag,
serialized_deployment_def: bytes,
serialized_init_args: bytes,
serialized_init_kwargs: bytes,
deployment_config_proto_bytes: bytes,
version: DeploymentVersion,
controller_name: str,
detached: bool,
):
configure_component_logger(
component_type="deployment",
component_name=deployment_name,
component_id=replica_tag,
)
deployment_def = cloudpickle.loads(serialized_deployment_def)
if isinstance(deployment_def, str):
import_path = deployment_def
module_name, attr_name = parse_import_path(import_path)
deployment_def = getattr(import_module(module_name), attr_name)
# For ray or serve decorated class or function, strip to return
# original body
if isinstance(deployment_def, RemoteFunction):
deployment_def = deployment_def._function
elif isinstance(deployment_def, ActorClass):
deployment_def = deployment_def.__ray_metadata__.modified_class
elif isinstance(deployment_def, Deployment):
logger.warning(
f'The import path "{import_path}" contains a '
"decorated Serve deployment. The decorator's settings "
"are ignored when deploying via import path.")
deployment_def = deployment_def.func_or_class
init_args = cloudpickle.loads(serialized_init_args)
init_kwargs = cloudpickle.loads(serialized_init_kwargs)
deployment_config = DeploymentConfig.from_proto_bytes(
deployment_config_proto_bytes)
if inspect.isfunction(deployment_def):
is_function = True
elif inspect.isclass(deployment_def):
is_function = False
else:
assert False, (
"deployment_def must be function, class, or "
"corresponding import path. Instead, it's type was "
f"{type(deployment_def)}.")
# Set the controller name so that serve.connect() in the user's
# code will connect to the instance that this deployment is running
# in.
ray.serve.context.set_internal_replica_context(
deployment_name,
replica_tag,
controller_name,
servable_object=None,
)
assert controller_name, "Must provide a valid controller_name"
controller_handle = ray.get_actor(controller_name,
namespace=SERVE_NAMESPACE)
# This closure initializes user code and finalizes replica
# startup. By splitting the initialization step like this,
# we can already access this actor before the user code
# has finished initializing.
# The supervising state manager can then wait
# for allocation of this replica by using the `is_allocated`
# method. After that, it calls `reconfigure` to trigger
# user code initialization.
async def initialize_replica():
if is_function:
_callable = deployment_def
else:
# This allows deployments to define an async __init__
# method (required for FastAPI).
_callable = deployment_def.__new__(deployment_def)
await sync_to_async(_callable.__init__)(*init_args,
**init_kwargs)
# Setting the context again to update the servable_object.
ray.serve.context.set_internal_replica_context(
deployment_name,
replica_tag,
controller_name,
servable_object=_callable,
)
self.replica = RayServeReplica(
_callable,
deployment_name,
replica_tag,
deployment_config,
deployment_config.user_config,
version,
is_function,
controller_handle,
)
# Is it fine that replica is None here?
# Should we add a check in all methods that use self.replica
# or, alternatively, create an async get_replica() method?
self.replica = None
self._initialize_replica = initialize_replica
async def __init__(
self,
deployment_name,
replica_tag,
init_args,
init_kwargs,
deployment_config_proto_bytes: bytes,
version: DeploymentVersion,
controller_name: str,
controller_namespace: str,
detached: bool,
):
if import_path is not None:
module_name, attr_name = parse_import_path(import_path)
deployment_def = getattr(import_module(module_name), attr_name)
else:
deployment_def = cloudpickle.loads(serialized_deployment_def)
deployment_config = DeploymentConfig.from_proto_bytes(
deployment_config_proto_bytes)
if inspect.isfunction(deployment_def):
is_function = True
elif inspect.isclass(deployment_def):
is_function = False
else:
assert False, (
"deployment_def must be function, class, or "
"corresponding import path. Instead, it's type was "
f"{type(deployment_def)}.")
# Set the controller name so that serve.connect() in the user's
# code will connect to the instance that this deployment is running
# in.
ray.serve.api._set_internal_replica_context(
deployment_name,
replica_tag,
controller_name,
controller_namespace,
servable_object=None,
)
assert controller_name, "Must provide a valid controller_name"
controller_handle = ray.get_actor(controller_name,
namespace=controller_namespace)
# This closure initializes user code and finalizes replica
# startup. By splitting the initialization step like this,
# we can already access this actor before the user code
# has finished initializing.
# The supervising state manager can then wait
# for allocation of this replica by using the `is_allocated`
# method. After that, it calls `reconfigure` to trigger
# user code initialization.
async def initialize_replica():
if is_function:
_callable = deployment_def
else:
# This allows deployments to define an async __init__
# method (required for FastAPI).
_callable = deployment_def.__new__(deployment_def)
await sync_to_async(_callable.__init__)(*init_args,
**init_kwargs)
# Setting the context again to update the servable_object.
ray.serve.api._set_internal_replica_context(
deployment_name,
replica_tag,
controller_name,
controller_namespace,
servable_object=_callable,
)
self.replica = RayServeReplica(
_callable,
deployment_name,
replica_tag,
deployment_config,
deployment_config.user_config,
version,
is_function,
controller_handle,
)
# Is it fine that replica is None here?
# Should we add a check in all methods that use self.replica
# or, alternatively, create an async get_replica() method?
self.replica = None
self._initialize_replica = initialize_replica