def test_kill_pending_actor_with_no_restart_true():
cluster = ray.init()
global_state_accessor = GlobalStateAccessor(
cluster["redis_address"], ray.ray_constants.REDIS_DEFAULT_PASSWORD)
global_state_accessor.connect()
@ray.remote(resources={"WORKER": 1.0})
class PendingActor:
pass
# Kill actor with `no_restart=True`.
actor = PendingActor.remote()
# TODO(ffbin): The raylet doesn't guarantee the order when dealing with
# RequestWorkerLease and CancelWorkerLease. If we kill the actor
# immediately after creating the actor, we may not be able to clean up
# the request cached by the raylet.
# See https://github.com/ray-project/ray/issues/13545 for details.
time.sleep(1)
ray.kill(actor, no_restart=True)
def condition1():
message = global_state_accessor.get_all_resource_usage()
resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
if len(resource_usages.resource_load_by_shape.resource_demands) == 0:
return True
return False
# Actor is dead, so the infeasible task queue length is 0.
wait_for_condition(condition1, timeout=10)
global_state_accessor.disconnect()
ray.shutdown()
def test_actor_resource_demand(shutdown_only):
ray.shutdown()
cluster = ray.init(num_cpus=3)
global_state_accessor = GlobalStateAccessor(
cluster["redis_address"], ray.ray_constants.REDIS_DEFAULT_PASSWORD)
global_state_accessor.connect()
@ray.remote(num_cpus=2)
class Actor:
def foo(self):
return "ok"
a = Actor.remote()
ray.get(a.foo.remote())
time.sleep(1)
message = global_state_accessor.get_all_resource_usage()
resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
# The actor is scheduled so there should be no more demands left.
assert len(resource_usages.resource_load_by_shape.resource_demands) == 0
@ray.remote(num_cpus=80)
class Actor2:
pass
actors = []
actors.append(Actor2.remote())
time.sleep(1)
# This actor cannot be scheduled.
message = global_state_accessor.get_all_resource_usage()
resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
assert len(resource_usages.resource_load_by_shape.resource_demands) == 1
assert (
resource_usages.resource_load_by_shape.resource_demands[0].shape == {
"CPU": 80.0
})
assert (resource_usages.resource_load_by_shape.resource_demands[0].
num_infeasible_requests_queued == 1)
actors.append(Actor2.remote())
time.sleep(1)
# Two actors cannot be scheduled.
message = global_state_accessor.get_all_resource_usage()
resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
assert len(resource_usages.resource_load_by_shape.resource_demands) == 1
assert (resource_usages.resource_load_by_shape.resource_demands[0].
num_infeasible_requests_queued == 2)
global_state_accessor.disconnect()
def test_backlog_report(shutdown_only):
cluster = ray.init(num_cpus=1,
_system_config={
"report_worker_backlog": True,
})
global_state_accessor = GlobalStateAccessor(
cluster["redis_address"], ray.ray_constants.REDIS_DEFAULT_PASSWORD)
global_state_accessor.connect()
@ray.remote(num_cpus=1)
def foo(x):
print(".")
time.sleep(x)
return None
def backlog_size_set():
message = global_state_accessor.get_all_resource_usage()
if message is None:
return False
resource_usage = ray.gcs_utils.ResourceUsageBatchData.FromString(
message)
aggregate_resource_load = \
resource_usage.resource_load_by_shape.resource_demands
if len(aggregate_resource_load) == 1:
backlog_size = aggregate_resource_load[0].backlog_size
print(backlog_size)
# Ideally we'd want to assert backlog_size == 8, but guaranteeing
# the order the order that submissions will occur is too
# hard/flaky.
return backlog_size > 0
return False
# We want this first task to finish
refs = [foo.remote(0.5)]
# These tasks should all start _before_ the first one finishes.
refs.extend([foo.remote(1000) for _ in range(9)])
# Now there's 1 request running, 1 queued in the raylet, and 8 queued in
# the worker backlog.
ray.get(refs[0])
# First request finishes, second request is now running, third lease
# request is sent to the raylet with backlog=7
ray.test_utils.wait_for_condition(backlog_size_set, timeout=2)
global_state_accessor.disconnect()
def test_heartbeat_ip(shutdown_only):
cluster = ray.init(num_cpus=1)
global_state_accessor = GlobalStateAccessor(
cluster["redis_address"], ray.ray_constants.REDIS_DEFAULT_PASSWORD)
global_state_accessor.connect()
self_ip = ray.util.get_node_ip_address()
def self_ip_is_set():
message = global_state_accessor.get_all_resource_usage()
if message is None:
return False
resource_usage = gcs_utils.ResourceUsageBatchData.FromString(message)
resources_data = resource_usage.batch[0]
return resources_data.node_manager_address == self_ip
wait_for_condition(self_ip_is_set, timeout=2)
global_state_accessor.disconnect()
class Monitor:
"""A monitor for Ray processes.
The monitor is in charge of cleaning up the tables in the global state
after processes have died. The monitor is currently not responsible for
detecting component failures.
Attributes:
redis: A connection to the Redis server.
primary_subscribe_client: A pubsub client for the Redis server.
This is used to receive notifications about failed components.
"""
def __init__(self, redis_address, autoscaling_config, redis_password=None):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
self.global_state_accessor = GlobalStateAccessor(
redis_address, redis_password, False)
self.global_state_accessor.connect()
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.mode = 0
# Setup subscriptions to the primary Redis server and the Redis shards.
self.primary_subscribe_client = self.redis.pubsub(
ignore_subscribe_messages=True)
# Keep a mapping from raylet client ID to IP address to use
# for updating the load metrics.
self.raylet_id_to_ip_map = {}
head_node_ip = redis_address.split(":")[0]
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
if autoscaling_config:
self.autoscaler = StandardAutoscaler(autoscaling_config,
self.load_metrics)
self.autoscaling_config = autoscaling_config
else:
self.autoscaler = None
self.autoscaling_config = None
def __del__(self):
"""Destruct the monitor object."""
# We close the pubsub client to avoid leaking file descriptors.
try:
primary_subscribe_client = self.primary_subscribe_client
except AttributeError:
primary_subscribe_client = None
if primary_subscribe_client is not None:
primary_subscribe_client.close()
if self.global_state_accessor is not None:
self.global_state_accessor.disconnect()
self.global_state_accessor = None
def subscribe(self, channel):
"""Subscribe to the given channel on the primary Redis shard.
Args:
channel (str): The channel to subscribe to.
Raises:
Exception: An exception is raised if the subscription fails.
"""
self.primary_subscribe_client.subscribe(channel)
def update_load_metrics(self):
"""Fetches heartbeat data from GCS and updates load metrics."""
all_heartbeat = self.global_state_accessor.get_all_heartbeat()
heartbeat_batch_data = \
ray.gcs_utils.HeartbeatBatchTableData.FromString(all_heartbeat)
for heartbeat_message in heartbeat_batch_data.batch:
resource_load = dict(heartbeat_message.resource_load)
total_resources = dict(heartbeat_message.resources_total)
available_resources = dict(heartbeat_message.resources_available)
waiting_bundles, infeasible_bundles = parse_resource_demands(
heartbeat_batch_data.resource_load_by_shape)
pending_placement_groups = list(
heartbeat_batch_data.placement_group_load.placement_group_data)
# Update the load metrics for this raylet.
node_id = ray.utils.binary_to_hex(heartbeat_message.node_id)
ip = self.raylet_id_to_ip_map.get(node_id)
if ip:
self.load_metrics.update(ip, total_resources,
available_resources, resource_load,
waiting_bundles, infeasible_bundles,
pending_placement_groups)
else:
logger.warning(
f"Monitor: could not find ip for node {node_id}")
def autoscaler_resource_request_handler(self, _, data):
"""Handle a notification of a resource request for the autoscaler.
This channel and method are only used by the manual
`ray.autoscaler.sdk.request_resources` api.
Args:
channel: unused
data: a resource request as JSON, e.g. {"CPU": 1}
"""
if not self.autoscaler:
return
try:
self.autoscaler.request_resources(json.loads(data))
except Exception:
# We don't want this to kill the monitor.
traceback.print_exc()
def process_messages(self, max_messages=10000):
"""Process all messages ready in the subscription channels.
This reads messages from the subscription channels and calls the
appropriate handlers until there are no messages left.
Args:
max_messages: The maximum number of messages to process before
returning.
"""
subscribe_clients = [self.primary_subscribe_client]
for subscribe_client in subscribe_clients:
for _ in range(max_messages):
message = None
try:
message = subscribe_client.get_message()
except redis.exceptions.ConnectionError:
pass
if message is None:
# Continue on to the next subscribe client.
break
# Parse the message.
channel = message["channel"]
data = message["data"]
if (channel ==
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL):
message_handler = self.autoscaler_resource_request_handler
else:
assert False, "This code should be unreachable."
# Call the handler.
message_handler(channel, data)
def update_raylet_map(self, _append_port=False):
"""Updates internal raylet map.
Args:
_append_port (bool): Defaults to False. Appending the port is
useful in testing, as mock clusters have many nodes with
the same IP and cannot be uniquely identified.
"""
all_raylet_nodes = ray.nodes()
self.raylet_id_to_ip_map = {}
for raylet_info in all_raylet_nodes:
node_id = (raylet_info.get("DBClientID") or raylet_info["NodeID"])
ip_address = (raylet_info.get("AuxAddress")
or raylet_info["NodeManagerAddress"]).split(":")[0]
if _append_port:
ip_address += ":" + str(raylet_info["NodeManagerPort"])
self.raylet_id_to_ip_map[node_id] = ip_address
def _run(self):
"""Run the monitor.
This function loops forever, checking for messages about dead database
clients and cleaning up state accordingly.
"""
self.subscribe(ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL)
# Handle messages from the subscription channels.
while True:
# Process autoscaling actions
if self.autoscaler:
# Only used to update the load metrics for the autoscaler.
self.update_raylet_map()
self.update_load_metrics()
self.autoscaler.update()
# Process a round of messages.
self.process_messages()
# Wait for a autoscaler update interval before processing the next
# round of messages.
time.sleep(AUTOSCALER_UPDATE_INTERVAL_S)
def destroy_autoscaler_workers(self):
"""Cleanup the autoscaler, in case of an exception in the run() method.
We kill the worker nodes, but retain the head node in order to keep
logs around, keeping costs minimal. This monitor process runs on the
head node anyway, so this is more reliable."""
if self.autoscaler is None:
return # Nothing to clean up.
if self.autoscaling_config is None:
# This is a logic error in the program. Can't do anything.
logger.error(
"Monitor: Cleanup failed due to lack of autoscaler config.")
return
logger.info("Monitor: Exception caught. Taking down workers...")
clean = False
while not clean:
try:
teardown_cluster(
config_file=self.autoscaling_config,
yes=True, # Non-interactive.
workers_only=True, # Retain head node for logs.
override_cluster_name=None,
keep_min_workers=True, # Retain minimal amount of workers.
)
clean = True
logger.info("Monitor: Workers taken down.")
except Exception:
logger.error("Monitor: Cleanup exception. Trying again...")
time.sleep(2)
def run(self):
try:
self._run()
except Exception:
logger.exception("Error in monitor loop")
if self.autoscaler:
self.autoscaler.kill_workers()
raise
def test_placement_group_load_report(ray_start_cluster):
cluster = ray_start_cluster
# Add a head node that doesn't have gpu resource.
cluster.add_node(num_cpus=4)
ray.init(address=cluster.address)
global_state_accessor = GlobalStateAccessor(
cluster.address, ray.ray_constants.REDIS_DEFAULT_PASSWORD)
global_state_accessor.connect()
class PgLoadChecker:
def nothing_is_ready(self):
resource_usage = self._read_resource_usage()
if not resource_usage:
return False
if resource_usage.HasField("placement_group_load"):
pg_load = resource_usage.placement_group_load
return len(pg_load.placement_group_data) == 2
return False
def only_first_one_ready(self):
resource_usage = self._read_resource_usage()
if not resource_usage:
return False
if resource_usage.HasField("placement_group_load"):
pg_load = resource_usage.placement_group_load
return len(pg_load.placement_group_data) == 1
return False
def two_infeasible_pg(self):
resource_usage = self._read_resource_usage()
if not resource_usage:
return False
if resource_usage.HasField("placement_group_load"):
pg_load = resource_usage.placement_group_load
return len(pg_load.placement_group_data) == 2
return False
def _read_resource_usage(self):
message = global_state_accessor.get_all_resource_usage()
if message is None:
return False
resource_usage = ray.gcs_utils.ResourceUsageBatchData.FromString(
message)
return resource_usage
checker = PgLoadChecker()
# Create 2 placement groups that are infeasible.
pg_feasible = ray.util.placement_group([{"A": 1}])
pg_infeasible = ray.util.placement_group([{"B": 1}])
_, unready = ray.wait(
[pg_feasible.ready(), pg_infeasible.ready()], timeout=0)
assert len(unready) == 2
ray.test_utils.wait_for_condition(checker.nothing_is_ready)
# Add a node that makes pg feasible. Make sure load include this change.
cluster.add_node(resources={"A": 1})
ray.get(pg_feasible.ready())
ray.test_utils.wait_for_condition(checker.only_first_one_ready)
# Create one more infeasible pg and make sure load is properly updated.
pg_infeasible_second = ray.util.placement_group([{"C": 1}])
_, unready = ray.wait([pg_infeasible_second.ready()], timeout=0)
assert len(unready) == 1
ray.test_utils.wait_for_condition(checker.two_infeasible_pg)
global_state_accessor.disconnect()
def test_load_report(shutdown_only, max_shapes):
resource1 = "A"
resource2 = "B"
cluster = ray.init(num_cpus=1,
resources={resource1: 1},
_system_config={
"max_resource_shapes_per_load_report": max_shapes,
})
global_state_accessor = GlobalStateAccessor(
cluster["redis_address"], ray.ray_constants.REDIS_DEFAULT_PASSWORD)
global_state_accessor.connect()
@ray.remote
def sleep():
time.sleep(1000)
sleep.remote()
for _ in range(3):
sleep.remote()
sleep.options(resources={resource1: 1}).remote()
sleep.options(resources={resource2: 1}).remote()
class Checker:
def __init__(self):
self.report = None
def check_load_report(self):
message = global_state_accessor.get_all_resource_usage()
if message is None:
return False
resource_usage = ray.gcs_utils.ResourceUsageBatchData.FromString(
message)
self.report = \
resource_usage.resource_load_by_shape.resource_demands
if max_shapes == 0:
return True
elif max_shapes == 2:
return len(self.report) >= 2
else:
return len(self.report) >= 3
# Wait for load information to arrive.
checker = Checker()
ray.test_utils.wait_for_condition(checker.check_load_report)
# Check that we respect the max shapes limit.
if max_shapes != -1:
assert len(checker.report) <= max_shapes
print(checker.report)
if max_shapes > 0:
# Check that we always include the 1-CPU resource shape.
one_cpu_shape = {"CPU": 1}
one_cpu_found = False
for demand in checker.report:
if demand.shape == one_cpu_shape:
one_cpu_found = True
assert one_cpu_found
# Check that we differentiate between infeasible and ready tasks.
for demand in checker.report:
if resource2 in demand.shape:
assert demand.num_infeasible_requests_queued > 0
assert demand.num_ready_requests_queued == 0
else:
assert demand.num_ready_requests_queued > 0
assert demand.num_infeasible_requests_queued == 0
global_state_accessor.disconnect()
class Monitor:
"""Autoscaling monitor.
This process periodically collects stats from the GCS and triggers
autoscaler updates.
Attributes:
redis: A connection to the Redis server.
"""
def __init__(self,
redis_address,
autoscaling_config,
redis_password=None,
prefix_cluster_info=False):
# Initialize the Redis clients.
ray.state.state._initialize_global_state(redis_address,
redis_password=redis_password)
self.redis = ray._private.services.create_redis_client(
redis_address, password=redis_password)
self.global_state_accessor = GlobalStateAccessor(
redis_address, redis_password, False)
self.global_state_accessor.connect()
# Set the redis client and mode so _internal_kv works for autoscaler.
worker = ray.worker.global_worker
worker.redis_client = self.redis
worker.mode = 0
# Keep a mapping from raylet client ID to IP address to use
# for updating the load metrics.
self.raylet_id_to_ip_map = {}
head_node_ip = redis_address.split(":")[0]
self.load_metrics = LoadMetrics(local_ip=head_node_ip)
if autoscaling_config:
self.autoscaler = StandardAutoscaler(
autoscaling_config,
self.load_metrics,
prefix_cluster_info=prefix_cluster_info)
self.autoscaling_config = autoscaling_config
else:
self.autoscaler = None
self.autoscaling_config = None
def __del__(self):
"""Destruct the monitor object."""
# We close the pubsub client to avoid leaking file descriptors.
if self.global_state_accessor is not None:
self.global_state_accessor.disconnect()
self.global_state_accessor = None
def update_load_metrics(self):
"""Fetches resource usage data from GCS and updates load metrics."""
all_resources = self.global_state_accessor.get_all_resource_usage()
resources_batch_data = \
ray.gcs_utils.ResourceUsageBatchData.FromString(all_resources)
for resource_message in resources_batch_data.batch:
resource_load = dict(resource_message.resource_load)
total_resources = dict(resource_message.resources_total)
available_resources = dict(resource_message.resources_available)
waiting_bundles, infeasible_bundles = parse_resource_demands(
resources_batch_data.resource_load_by_shape)
pending_placement_groups = list(
resources_batch_data.placement_group_load.placement_group_data)
# Update the load metrics for this raylet.
node_id = ray.utils.binary_to_hex(resource_message.node_id)
ip = self.raylet_id_to_ip_map.get(node_id)
if ip:
self.load_metrics.update(ip, total_resources,
available_resources, resource_load,
waiting_bundles, infeasible_bundles,
pending_placement_groups)
else:
logger.warning(
f"Monitor: could not find ip for node {node_id}")
def update_resource_requests(self):
"""Fetches resource requests from the internal KV and updates load."""
if not _internal_kv_initialized():
return
data = _internal_kv_get(
ray.ray_constants.AUTOSCALER_RESOURCE_REQUEST_CHANNEL)
if data:
try:
resource_request = json.loads(data)
self.load_metrics.set_resource_requests(resource_request)
except Exception:
logger.exception("Error parsing resource requests")
def autoscaler_resource_request_handler(self, _, data):
"""Handle a notification of a resource request for the autoscaler.
This channel and method are only used by the manual
`ray.autoscaler.sdk.request_resources` api.
Args:
channel: unused
data: a resource request as JSON, e.g. {"CPU": 1}
"""
resource_request = json.loads(data)
self.load_metrics.set_resource_requests(resource_request)
def update_raylet_map(self, _append_port=False):
"""Updates internal raylet map.
Args:
_append_port (bool): Defaults to False. Appending the port is
useful in testing, as mock clusters have many nodes with
the same IP and cannot be uniquely identified.
"""
all_raylet_nodes = ray.nodes()
self.raylet_id_to_ip_map = {}
for raylet_info in all_raylet_nodes:
node_id = (raylet_info.get("DBClientID") or raylet_info["NodeID"])
ip_address = (raylet_info.get("AuxAddress")
or raylet_info["NodeManagerAddress"]).split(":")[0]
if _append_port:
ip_address += ":" + str(raylet_info["NodeManagerPort"])
self.raylet_id_to_ip_map[node_id] = ip_address
def _run(self):
"""Run the monitor loop."""
while True:
self.update_raylet_map()
self.update_load_metrics()
self.update_resource_requests()
status = {
"load_metrics_report": self.load_metrics.summary()._asdict()
}
# Process autoscaling actions
if self.autoscaler:
# Only used to update the load metrics for the autoscaler.
self.autoscaler.update()
status["autoscaler_report"] = self.autoscaler.summary(
)._asdict()
as_json = json.dumps(status)
if _internal_kv_initialized():
_internal_kv_put(DEBUG_AUTOSCALING_STATUS,
as_json,
overwrite=True)
# Wait for a autoscaler update interval before processing the next
# round of messages.
time.sleep(AUTOSCALER_UPDATE_INTERVAL_S)
def destroy_autoscaler_workers(self):
"""Cleanup the autoscaler, in case of an exception in the run() method.
We kill the worker nodes, but retain the head node in order to keep
logs around, keeping costs minimal. This monitor process runs on the
head node anyway, so this is more reliable."""
if self.autoscaler is None:
return # Nothing to clean up.
if self.autoscaling_config is None:
# This is a logic error in the program. Can't do anything.
logger.error(
"Monitor: Cleanup failed due to lack of autoscaler config.")
return
logger.info("Monitor: Exception caught. Taking down workers...")
clean = False
while not clean:
try:
teardown_cluster(
config_file=self.autoscaling_config,
yes=True, # Non-interactive.
workers_only=True, # Retain head node for logs.
override_cluster_name=None,
keep_min_workers=True, # Retain minimal amount of workers.
)
clean = True
logger.info("Monitor: Workers taken down.")
except Exception:
logger.error("Monitor: Cleanup exception. Trying again...")
time.sleep(2)
def run(self):
try:
self._run()
except Exception:
logger.exception("Error in monitor loop")
if self.autoscaler:
self.autoscaler.kill_workers()
raise
def initialize(num_cpus: int,
num_gpus: int,
log_root_path: str,
log_name: Optional[str] = None,
logger_cls: type = TBXLogger,
launch_tensorboard: bool = True,
debug: bool = False,
verbose: bool = True) -> Callable[[Dict[str, Any]], Logger]:
"""Initialize Ray and Tensorboard daemons.
It will be used later for almost everything from dashboard, remote/client
management, to multithreaded environment.
.. note:
The default Tensorboard port will be used, namely 6006 if available,
using 0.0.0.0 (binding to all IPv4 addresses on local machine).
Similarly, Ray dashboard port is 8265 if available. In both cases, the
port will be increased interatively until to find one available.
:param num_cpus: Maximum number of CPU threads that can be executed in
parallel. Note that it does not actually reserve part of
the CPU, so that several processes can reserve the number
of threads available on the system at the same time.
:param num_gpu: Maximum number of GPU unit that can be used, which can be
fractional to only allocate part of the resource. Note that
contrary to CPU resource, the memory is likely to actually
be reserve and allocated by the process, in particular
using Tensorflow backend.
:param log_root_path: Fullpath of root log directory.
:param log_name: Name of the subdirectory where to save data. `None` to
use default name, empty string '' to set it interactively
in command prompt. It must be a valid Python identifier.
Optional: full date _ hostname by default.
:param logger_cls: Custom logger class type deriving from `TBXLogger`.
Optional: `TBXLogger` by default.
:param launch_tensorboard: Whether or not to launch tensorboard
automatically.
Optional: Enabled by default.
:param debug: Whether or not to display debugging trace.
Optional: Disabled by default.
:param verbose: Whether or not to print information about what is going on.
Optional: True by default.
:returns: lambda function to pass a `ray.Trainer` to monitor learning
progress in Tensorboard.
"""
# Make sure provided logger class derives from ray.tune.logger.Logger
assert issubclass(logger_cls, Logger), (
"Logger class must derive from `ray.tune.logger.Logger`")
# Check if cluster servers are already running, and if requested resources
# are available.
is_cluster_running = False
redis_addresses = services.find_redis_address()
if redis_addresses:
for redis_address in redis_addresses:
# Connect to redis global state accessor
global_state_accessor = GlobalStateAccessor(
redis_address, ray_constants.REDIS_DEFAULT_PASSWORD)
global_state_accessor.connect()
# Get available resources
resources: Dict[str, int] = defaultdict(int)
for info in global_state_accessor.get_all_available_resources():
# pylint: disable=no-member
message = ray.gcs_utils.AvailableResources.FromString(info)
for field, capacity in message.resources_available.items():
resources[field] += capacity
# Disconnect global state accessor
time.sleep(0.1)
global_state_accessor.disconnect()
# Check if enough computation resources are available
is_cluster_running = (resources["CPU"] >= num_cpus and
resources["GPU"] >= num_gpus)
# Stop looking as soon as a cluster with enough resources is found
if is_cluster_running:
break
# Connect to Ray server if necessary, starting one if not already running
if not ray.is_initialized():
if not is_cluster_running:
# Start new Ray server, if not already running
ray.init(
# Address of Ray cluster to connect to, if any
address=None,
# Number of CPUs assigned to each raylet
num_cpus=num_cpus,
# Number of GPUs assigned to each raylet
num_gpus=num_gpus,
# Enable object eviction in LRU order under memory pressure
_lru_evict=False,
# Whether or not to execute the code serially (for debugging)
local_mode=debug,
# Logging level
logging_level=logging.DEBUG if debug else logging.ERROR,
# Whether to redirect outputs from every worker to the driver
log_to_driver=debug,
# Whether to start Ray dashboard, to monitor cluster's status
include_dashboard=True,
# The host to bind the dashboard server to
dashboard_host="0.0.0.0")
else:
# Connect to existing Ray cluster
ray.init(
address="auto",
_lru_evict=False,
local_mode=debug,
logging_level=logging.DEBUG if debug else logging.ERROR,
log_to_driver=debug,
include_dashboard=False)
# Configure Tensorboard
if launch_tensorboard:
tb = TensorBoard()
tb.configure(host="0.0.0.0", logdir=os.path.abspath(log_root_path))
url = tb.launch()
if verbose:
print(f"Started Tensorboard {url}.",
f"Root directory: {log_root_path}")
# Define log filename interactively if requested
if log_name == "":
while True:
log_name = input(
"Enter desired log subdirectory name (empty for default)...")
if not log_name or re.match(r'^[A-Za-z0-9_]+$', log_name):
break
print("Unvalid name. Only Python identifiers are supported.")
# Handling of default log name and sanity checks
if not log_name:
log_name = "_".join((
datetime.now().strftime("%Y_%m_%d_%H_%M_%S"),
re.sub(r'[^A-Za-z0-9_]', "_", socket.gethostname())))
else:
assert re.match(r'^[A-Za-z0-9_]+$', log_name), (
"Log name must be a valid Python identifier.")
# Create log directory
log_path = os.path.join(log_root_path, log_name)
pathlib.Path(log_path).mkdir(parents=True, exist_ok=True)
if verbose:
print(f"Tensorboard logfiles directory: {log_path}")
# Define Ray logger
def logger_creator(config: Dict[str, Any]) -> Logger:
return logger_cls(config, log_path)
return logger_creator
