def serialize(obj):
"""This is the callback that will be used by numbuf.
If numbuf does not know how to serialize an object, it will call this method.
Args:
obj (object): A Python object.
Returns:
A dictionary that has the key "_pyttype_" to identify the class, and
contains all information needed to reconstruct the object.
"""
class_id = class_identifier(type(obj))
if class_id not in whitelisted_classes:
raise Exception(
"Ray does not know how to serialize objects of type {}. "
"To fix this, call 'ray.register_class' with this class.".format(
type(obj)))
if class_id in classes_to_pickle:
serialized_obj = {"data": pickling.dumps(obj)}
elif class_id in custom_serializers.keys():
serialized_obj = {"data": custom_serializers[class_id](obj)}
else:
# Handle the namedtuple case.
if is_named_tuple(type(obj)):
serialized_obj = {}
serialized_obj["_ray_getnewargs_"] = obj.__getnewargs__()
elif hasattr(obj, "__dict__"):
serialized_obj = obj.__dict__
else:
raise Exception(
"We do not know how to serialize the object '{}'".format(obj))
result = dict(serialized_obj, **{"_pytype_": class_id})
return result
def export_actor_class(class_id, Class, actor_method_names, worker):
if worker.mode is None:
raise NotImplemented("TODO(pcm): Cache actors")
key = b"ActorClass:" + class_id
d = {
"driver_id": worker.task_driver_id.id(),
"class_name": Class.__name__,
"module": Class.__module__,
"class": pickling.dumps(Class),
"actor_method_names": json.dumps(list(actor_method_names))
}
worker.redis_client.hmset(key, d)
worker.redis_client.rpush("Exports", key)
def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus,
worker):
"""Export an actor to redis.
Args:
actor_id: The ID of the actor.
Class: Name of the class to be exported as an actor.
actor_method_names (list): A list of the names of this actor's methods.
num_cpus (int): The number of CPUs that this actor requires.
num_gpus (int): The number of GPUs that this actor requires.
"""
ray.worker.check_main_thread()
if worker.mode is None:
raise NotImplemented("TODO(pcm): Cache actors")
key = "Actor:{}".format(actor_id.id())
pickled_class = pickling.dumps(Class)
# For now, all actor methods have 1 return value.
driver_id = worker.task_driver_id.id()
for actor_method_name in actor_method_names:
function_id = get_actor_method_function_id(actor_method_name).id()
worker.function_properties[driver_id][function_id] = (1, num_cpus,
num_gpus)
# Get a list of the local schedulers from the client table.
client_table = ray.global_state.client_table()
local_schedulers = []
for ip_address, clients in client_table.items():
for client in clients:
if client["ClientType"] == "local_scheduler":
local_schedulers.append(client)
# Select a local scheduler for the actor.
local_scheduler_id, gpu_ids = select_local_scheduler(local_schedulers,
num_gpus, worker)
# Really we should encode this message as a flatbuffer object. However, we're
# having trouble getting that to work. It almost works, but in Python 2.7,
# builder.CreateString fails on byte strings that contain characters outside
# range(128).
worker.redis_client.publish("actor_notifications",
actor_id.id() + driver_id + local_scheduler_id)
d = {"driver_id": driver_id,
"actor_id": actor_id.id(),
"name": Class.__name__,
"module": Class.__module__,
"class": pickled_class,
"gpu_ids": json.dumps(gpu_ids),
"actor_method_names": json.dumps(list(actor_method_names))}
worker.redis_client.hmset(key, d)
worker.redis_client.rpush("Exports", key)
def export_actor(actor_id, Class, actor_method_names, num_cpus, num_gpus,
worker):
"""Export an actor to redis.
Args:
actor_id: The ID of the actor.
Class: Name of the class to be exported as an actor.
actor_method_names (list): A list of the names of this actor's methods.
num_cpus (int): The number of CPUs that this actor requires.
num_gpus (int): The number of GPUs that this actor requires.
"""
ray.worker.check_main_thread()
if worker.mode is None:
raise NotImplemented("TODO(pcm): Cache actors")
key = "Actor:{}".format(actor_id.id())
pickled_class = pickling.dumps(Class)
# For now, all actor methods have 1 return value.
driver_id = worker.task_driver_id.id()
for actor_method_name in actor_method_names:
function_id = get_actor_method_function_id(actor_method_name).id()
worker.function_properties[driver_id][function_id] = (1, num_cpus,
num_gpus)
# Select a local scheduler for the actor.
local_schedulers = state.get_local_schedulers(worker)
local_scheduler_id, gpu_ids = select_local_scheduler(
local_schedulers, num_gpus, worker)
worker.redis_client.publish("actor_notifications",
actor_id.id() + local_scheduler_id)
d = {
"driver_id": driver_id,
"actor_id": actor_id.id(),
"name": Class.__name__,
"module": Class.__module__,
"class": pickled_class,
"gpu_ids": json.dumps(gpu_ids),
"actor_method_names": json.dumps(list(actor_method_names))
}
worker.redis_client.hmset(key, d)
worker.redis_client.rpush("Exports", key)
def export_actor(actor_id, Class, worker):
"""Export an actor to redis.
Args:
actor_id: The ID of the actor.
Class: Name of the class to be exported as an actor.
worker: The worker class
"""
ray.worker.check_main_thread()
if worker.mode is None:
raise NotImplemented("TODO(pcm): Cache actors")
key = "Actor:{}".format(actor_id.id())
pickled_class = pickling.dumps(Class)
# Select a local scheduler for the actor.
local_schedulers = state.get_local_schedulers()
local_scheduler_id = random.choice(local_schedulers)
worker.redis_client.publish("actor_notifications",
actor_id.id() + local_scheduler_id)
# The export counter is computed differently depending on whether we are
# currently in a driver or a worker.
if worker.mode in [ray.SCRIPT_MODE, ray.SILENT_MODE]:
export_counter = worker.driver_export_counter
elif worker.mode == ray.WORKER_MODE:
# We don't actually need export counters for actors.
export_counter = 0
d = {
"driver_id": worker.task_driver_id.id(),
"actor_id": actor_id.id(),
"name": Class.__name__,
"module": Class.__module__,
"class": pickled_class,
"class_export_counter": export_counter
}
worker.redis_client.hmset(key, d)
worker.redis_client.rpush("Exports", key)
worker.driver_export_counter += 1
