def create_cluster(num_nodes):
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(resources={str(i): 100}, object_store_memory=10**9)
ray.init(redis_address=cluster.redis_address)
return cluster
def test_connect_with_disconnected_node(shutdown_only):
config = json.dumps({
"num_heartbeats_timeout": 50,
"heartbeat_timeout_milliseconds": 10,
})
cluster = Cluster()
cluster.add_node(num_cpus=0, _internal_config=config)
ray.init(redis_address=cluster.redis_address)
info = relevant_errors(ray_constants.REMOVED_NODE_ERROR)
assert len(info) == 0
# This node is killed by SIGKILL, ray_monitor will mark it to dead.
dead_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(dead_node, allow_graceful=False)
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 1, timeout=2)
# This node is killed by SIGKILL, ray_monitor will mark it to dead.
dead_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(dead_node, allow_graceful=False)
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 2, timeout=2)
# This node is killed by SIGTERM, ray_monitor will not mark it again.
removing_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(removing_node, allow_graceful=True)
with pytest.raises(Exception, match=("Timing out of wait.")):
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 3, timeout=2)
# There is no connection error to a dead node.
info = relevant_errors(ray_constants.RAYLET_CONNECTION_ERROR)
assert len(info) == 0
def test_connect_with_disconnected_node(shutdown_only):
config = json.dumps({
"num_heartbeats_timeout": 50,
"heartbeat_timeout_milliseconds": 10,
})
cluster = Cluster()
cluster.add_node(num_cpus=0, _internal_config=config)
ray.init(redis_address=cluster.redis_address)
info = relevant_errors(ray_constants.REMOVED_NODE_ERROR)
assert len(info) == 0
# This node is killed by SIGKILL, ray_monitor will mark it to dead.
dead_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(dead_node, allow_graceful=False)
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 1, timeout=2)
# This node is killed by SIGKILL, ray_monitor will mark it to dead.
dead_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(dead_node, allow_graceful=False)
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 2, timeout=2)
# This node is killed by SIGTERM, ray_monitor will not mark it again.
removing_node = cluster.add_node(num_cpus=0, _internal_config=config)
cluster.remove_node(removing_node, allow_graceful=True)
with pytest.raises(Exception, match=('Timing out of wait.')):
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 3, timeout=2)
# There is no connection error to a dead node.
info = relevant_errors(ray_constants.RAYLET_CONNECTION_ERROR)
assert len(info) == 0
def ray_start_reconstruction(request):
num_nodes = request.param
plasma_store_memory = int(0.1 * 10**9)
cluster = Cluster(
initialize_head=True,
head_node_args={
"num_cpus":
1,
"object_store_memory":
plasma_store_memory // num_nodes,
"redis_max_memory":
10**7,
"_internal_config":
json.dumps({"initial_reconstruction_timeout_milliseconds": 200})
})
for i in range(num_nodes - 1):
cluster.add_node(num_cpus=1,
object_store_memory=plasma_store_memory // num_nodes,
_internal_config=json.dumps({
"initial_reconstruction_timeout_milliseconds":
200
}))
ray.init(redis_address=cluster.redis_address)
yield plasma_store_memory, num_nodes, cluster
# Clean up the Ray cluster.
ray.shutdown()
cluster.shutdown()
def create_cluster(num_nodes):
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(resources={str(i): 100}, object_store_memory=10**9)
ray.init(address=cluster.address)
return cluster
def start_local_cluster(num_nodes, object_store_memory):
"""Start a local Ray cluster.
The ith node in the cluster will have a resource named "i".
Args:
num_nodes: The number of nodes to start in the cluster.
Returns:
The cluster object.
"""
num_redis_shards = 2
redis_max_memory = 10**8
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(
redis_port=6379 if i == 0 else None,
num_redis_shards=num_redis_shards if i == 0 else None,
num_cpus=8 if i == 0 else 2,
num_gpus=0,
resources={str(i): 500},
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory)
ray.init(address=cluster.address)
return cluster
def main():
cluster = Cluster(initialize_head=True,
connect=True,
head_node_args={
"object_store_memory": 20 * 1024 * 1024 * 1024,
"num_cpus": 16
})
cluster.add_node(object_store_memory=20 * 1024 * 1024 * 1024,
num_gpus=1,
num_cpus=16)
object_id_list = []
for i in range(0, 10):
object_id = ray.put(np.random.rand(1024 * 128, 1024))
object_id_list.append(object_id)
@ray.remote(num_gpus=1)
def f(object_id_list):
diffs = []
for object_id in object_id_list:
before = time.time()
ray.get(object_id)
after = time.time()
diffs.append(after - before)
time.sleep(1)
return np.mean(diffs), np.std(diffs)
time_diff, time_diff_std = ray.get(f.remote(object_id_list))
print("latency to get an 1G object over network", round(time_diff, 2),
"+-", round(time_diff_std, 2))
ray.shutdown()
cluster.shutdown()
def ray_start_reconstruction(request):
num_nodes = request.param
plasma_store_memory = int(0.5 * 10**9)
cluster = Cluster(
initialize_head=True,
head_node_args={
"num_cpus": 1,
"object_store_memory": plasma_store_memory // num_nodes,
"redis_max_memory": 10**7,
"_internal_config": json.dumps({
"initial_reconstruction_timeout_milliseconds": 200
})
})
for i in range(num_nodes - 1):
cluster.add_node(
num_cpus=1,
object_store_memory=plasma_store_memory // num_nodes,
_internal_config=json.dumps({
"initial_reconstruction_timeout_milliseconds": 200
}))
ray.init(redis_address=cluster.redis_address)
yield plasma_store_memory, num_nodes, cluster
# Clean up the Ray cluster.
ray.shutdown()
cluster.shutdown()
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.eager:
exp["config"]["eager"] = True
if args.trace:
if not exp["config"].get("eager"):
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(address=cluster.address)
else:
ray.init(address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume)
def test_cluster():
"""Basic test for adding and removing nodes in cluster."""
g = Cluster(initialize_head=False)
node = g.add_node()
node2 = g.add_node()
assert node.remaining_processes_alive()
assert node2.remaining_processes_alive()
g.remove_node(node2)
g.remove_node(node)
assert not any(n.any_processes_alive() for n in [node, node2])
def test_cluster():
"""Basic test for adding and removing nodes in cluster."""
g = Cluster(initialize_head=False)
node = g.add_node()
node2 = g.add_node()
assert node.remaining_processes_alive()
assert node2.remaining_processes_alive()
g.remove_node(node2)
g.remove_node(node)
assert not any(n.any_processes_alive() for n in [node, node2])
def test_redis_password_cluster(self, password, shutdown_only):
@ray.remote
def f():
return 1
node_args = {"redis_password": password}
cluster = Cluster(
initialize_head=True, connect=True, head_node_args=node_args)
cluster.add_node(**node_args)
object_id = f.remote()
ray.get(object_id)
def test_redis_password_cluster(self, password, shutdown_only):
@ray.remote
def f():
return 1
node_args = {"redis_password": password}
cluster = Cluster(
initialize_head=True, connect=True, head_node_args=node_args)
cluster.add_node(**node_args)
object_id = f.remote()
ray.get(object_id)
def ray_start_workers_separate_multinode(request):
num_nodes = request.param[0]
num_initial_workers = request.param[1]
# Start the Ray processes.
cluster = Cluster()
for _ in range(num_nodes):
cluster.add_node(num_cpus=num_initial_workers)
ray.init(address=cluster.address)
yield num_nodes, num_initial_workers
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def ray_start_workers_separate_multinode(request):
num_nodes = request.param[0]
num_initial_workers = request.param[1]
# Start the Ray processes.
cluster = Cluster()
for _ in range(num_nodes):
cluster.add_node(num_cpus=num_initial_workers)
ray.init(redis_address=cluster.redis_address)
yield num_nodes, num_initial_workers
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def ray_start_two_nodes():
# Start the Ray processes.
cluster = Cluster()
for _ in range(2):
cluster.add_node(num_cpus=0,
_internal_config=json.dumps(
{"num_heartbeats_timeout": 40}))
ray.init(redis_address=cluster.redis_address)
yield cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(redis_address=cluster.redis_address)
else:
ray.init(
redis_address=args.redis_address,
object_store_memory=args.ray_object_store_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(
experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume)
def ray_start_two_nodes():
# Start the Ray processes.
cluster = Cluster()
for _ in range(2):
cluster.add_node(
num_cpus=0,
_internal_config=json.dumps({
"num_heartbeats_timeout": 40
}))
ray.init(redis_address=cluster.redis_address)
yield cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def ray_start_combination(request):
num_nodes = request.param[0]
num_workers_per_scheduler = request.param[1]
# Start the Ray processes.
cluster = Cluster(initialize_head=True,
head_node_args={
"num_cpus": 10,
"redis_max_memory": 10**7
})
for i in range(num_nodes - 1):
cluster.add_node(num_cpus=10)
ray.init(redis_address=cluster.redis_address)
yield num_nodes, num_workers_per_scheduler, cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def ray_start_combination(request):
num_nodes = request.param[0]
num_workers_per_scheduler = request.param[1]
# Start the Ray processes.
cluster = Cluster(
initialize_head=True,
head_node_args={
"num_cpus": 10,
"redis_max_memory": 10**7
})
for i in range(num_nodes - 1):
cluster.add_node(num_cpus=10)
ray.init(redis_address=cluster.redis_address)
yield num_nodes, num_workers_per_scheduler, cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def test_temp_plasma_store_socket():
ray.init(plasma_store_socket_name="/tmp/i_am_a_temp_socket")
assert os.path.exists(
"/tmp/i_am_a_temp_socket"), "Specified socket path not found."
ray.shutdown()
try:
os.remove("/tmp/i_am_a_temp_socket")
except OSError:
pass # It could have been removed by Ray.
cluster = Cluster(True)
cluster.add_node(plasma_store_socket_name="/tmp/i_am_a_temp_socket_2")
assert os.path.exists(
"/tmp/i_am_a_temp_socket_2"), "Specified socket path not found."
cluster.shutdown()
try:
os.remove("/tmp/i_am_a_temp_socket_2")
except OSError:
pass # It could have been removed by Ray.
def ray_initialize_cluster():
# Start with 4 workers and 4 cores.
num_nodes = 4
num_workers_per_scheduler = 8
cluster = Cluster()
for _ in range(num_nodes):
cluster.add_node(
num_cpus=num_workers_per_scheduler,
_internal_config=json.dumps({
"initial_reconstruction_timeout_milliseconds": 1000,
"num_heartbeats_timeout": 10,
}))
ray.init(redis_address=cluster.redis_address)
yield cluster
ray.shutdown()
cluster.shutdown()
def ray_start_cluster():
node_args = {
"num_cpus": 4,
"_internal_config": json.dumps({
"initial_reconstruction_timeout_milliseconds": 1000,
"num_heartbeats_timeout": 10
})
}
# Start with 3 worker nodes and 4 cores each.
cluster = Cluster(
initialize_head=True, connect=True, head_node_args=node_args)
workers = []
for _ in range(3):
workers.append(cluster.add_node(**node_args))
cluster.wait_for_nodes()
yield cluster
ray.shutdown()
cluster.shutdown()
def _ray_start_cluster(**kwargs):
init_kwargs = get_default_fixture_ray_kwargs()
num_nodes = 0
do_init = False
# num_nodes & do_init are not arguments for ray.init, so delete them.
if "num_nodes" in kwargs:
num_nodes = kwargs["num_nodes"]
del kwargs["num_nodes"]
if "do_init" in kwargs:
do_init = kwargs["do_init"]
del kwargs["do_init"]
elif num_nodes > 0:
do_init = True
init_kwargs.update(kwargs)
cluster = Cluster()
remote_nodes = []
for _ in range(num_nodes):
remote_nodes.append(cluster.add_node(**init_kwargs))
if do_init:
ray.init(redis_address=cluster.redis_address)
yield cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
object_store_memory = 10**8
num_nodes = 4
message = ("Make sure there is enough memory on this machine to run this "
"workload. We divide the system memory by 2 to provide a buffer.")
assert (num_nodes * object_store_memory + num_redis_shards * redis_max_memory <
ray.utils.get_system_memory() / 2)
# Simulate a cluster on one machine.
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(redis_port=6379 if i == 0 else None,
num_redis_shards=num_redis_shards if i == 0 else None,
num_cpus=4,
num_gpus=0,
resources={str(i): 5},
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory)
ray.init(address=cluster.address)
# Run the workload.
# Define a driver script that runs a few tasks and actors on each node in the
# cluster.
driver_script = """
import ray
ray.init(address="{}")
num_nodes = {}
def run_experiment(args, parser):
# args.ray_object_store_memory = int(1e10)
args.ray_redis_max_memory = int(2e9)
if args.config_file:
with open(args.config_file) as f:
exp = yaml.load(f)
else:
raise Exception('No config file!')
exp = merge_dicts(exp, args.config)
log.info('Num workers: %d, num_envs_per_worker: %d',
exp['config']['num_workers'],
exp['config']['num_envs_per_worker'])
if args.cfg_mixins is not None:
for cfg_mixin_file in args.cfg_mixins:
with open(cfg_mixin_file, 'r') as f:
override_cfg = yaml.load(f)
log.info('Overriding parameters from %s: %r', cfg_mixin_file,
override_cfg)
exp = merge_dicts(exp, override_cfg)
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
redis_max_memory=args.ray_redis_max_memory,
)
ray.init(redis_address=cluster.redis_address,
local_mode=args.local_mode)
else:
ray.init(
redis_address=args.redis_address,
object_store_memory=args.ray_object_store_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus,
local_mode=args.local_mode,
)
exp = Experiment.from_json(args.experiment_name, exp)
exp.spec['checkpoint_freq'] = 20
if args.pbt:
exp.spec['checkpoint_freq'] = 3
exp.spec['checkpoint_at_end'] = True
# exp.spec['checkpoint_score_attr'] = 'episode_reward_mean'
exp.spec['keep_checkpoints_num'] = 5
if args.stop_seconds > 0:
exp.spec['stop'] = {'time_total_s': args.stop_seconds}
# if 'multiagent' in exp.spec['config']:
# # noinspection PyProtectedMember
# make_env = ray.tune.registry._global_registry.get(ENV_CREATOR, exp.spec['config']['env'])
# temp_env = make_env(None)
# obs_space, action_space = temp_env.observation_space, temp_env.action_space
# temp_env.close()
# del temp_env
#
# policies = dict(
# main=(None, obs_space, action_space, {}),
# dummy=(None, obs_space, action_space, {}),
# )
#
# exp.spec['config']['multiagent'] = {
# 'policies': policies,
# 'policy_mapping_fn': function(lambda agent_id: 'main'),
# 'policies_to_train': ['main'],
# }
#
# if args.dbg:
# exp.spec['config']['num_workers'] = 1
# exp.spec['config']['num_gpus'] = 1
# exp.spec['config']['num_envs_per_worker'] = 1
#
# if 'callbacks' not in exp.spec['config']:
# exp.spec['config']['callbacks'] = {}
#
# fps_helper = FpsHelper()
#
# def on_train_result(info):
# if 'APPO' in exp.spec['run']:
# samples = info['result']['info']['num_steps_sampled']
# else:
# samples = info['trainer'].optimizer.num_steps_trained
#
# fps_helper.record(samples)
# fps = fps_helper.get_fps()
# info['result']['custom_metrics']['fps'] = fps
#
# # remove this as currently
# skip_frames = exp.spec['config']['env_config']['skip_frames']
# info['result']['custom_metrics']['fps_frameskip'] = fps * skip_frames
#
# exp.spec['config']['callbacks']['on_train_result'] = function(on_train_result)
#
# def on_episode_end(info):
# episode = info['episode']
# stats = {
# 'DEATHCOUNT': 0,
# 'FRAGCOUNT': 0,
# 'HITCOUNT': 0,
# 'DAMAGECOUNT': 0,
# 'KDR': 0,
# 'FINAL_PLACE': 0,
# 'LEADER_GAP': 0,
# 'PLAYER_COUNT': 0,
# 'BOT_DIFFICULTY': 0,
# }
#
# # noinspection PyProtectedMember
# agent_to_last_info = episode._agent_to_last_info
# for agent in agent_to_last_info.keys():
# agent_info = agent_to_last_info[agent]
# for stats_key in stats.keys():
# stats[stats_key] += agent_info.get(stats_key, 0.0)
#
# for stats_key in stats.keys():
# stats[stats_key] /= len(agent_to_last_info.keys())
#
# episode.custom_metrics.update(stats)
#
# exp.spec['config']['callbacks']['on_episode_end'] = function(on_episode_end)
extra_kwargs = {}
if args.pbt:
extra_kwargs['reuse_actors'] = False
run(exp,
name=args.experiment_name,
scheduler=make_custom_scheduler(args),
resume=args.resume,
queue_trials=args.queue_trials,
**extra_kwargs)
def test_shutdown():
g = Cluster(initialize_head=False)
node = g.add_node()
node2 = g.add_node()
g.shutdown()
assert not any(n.any_processes_alive() for n in [node, node2])
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
# add callbacks for self-defined metric
# and save successful transitions from RL agents
experiment_name = next(iter(experiments))
experiments[experiment_name]["config"]["optimizer"]["robot_demo_path"] = dir_path
experiments[experiment_name]["config"]["callbacks"] = {
"on_episode_start": on_episode_start,
"on_episode_step": on_episode_step,
"on_episode_end": on_episode_end,
"on_sample_end": on_sample_end,
"on_train_result": on_train_result,
"on_postprocess_traj": on_postprocess_traj
}
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.eager:
exp["config"]["eager"] = True
if args.trace:
if not exp["config"].get("eager"):
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(address=cluster.address) #, log_to_driver=False)
else:
ray.init(
address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
# log_to_driver=False) # disable the loggings
# https://github.com/ray-project/ray/issues/5048
run_experiments(
experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume)
class RayExecutor(Executor):
def __init__(self, **kwargs):
super(RayExecutor, self).__init__()
mode = kwargs.get('ray_mode', 'local')
self.resource_idx = 0
if mode == 'local':
node_kwargs = {
'num_cpus': 4,
'object_store_memory': 10**9,
'resources': {
'Node_0': 100
}
}
self.cluster = Cluster(initialize_head=True,
head_node_args=node_kwargs)
self.num_nodes = kwargs.get('ray_num_nodes', 4)
self.nodes = []
self.resources = []
i = 1
for _ in range(self.num_nodes):
node, resource = self._create_local_node(i, node_kwargs)
self.nodes.append(node)
self.resources.append(resource)
self._create_local_node(i, node_kwargs)
redis_address = self.cluster.redis_address
ray.init(redis_address=redis_address)
else:
redis_address = kwargs.get('redis_address', '127.0.0.1')
ray.init(redis_address=redis_address)
self.resources = []
self.nodes = ray.global_state.client_table()
for node in self.nodes:
for resource in node['Resources']:
if 'Node' in resource and resource != 'Node_0':
self.resources.append(resource)
def __del__(self):
self.cluster.shutdown()
ray.disconnect()
def _create_local_node(self, i, node_kwargs):
resource = 'Node_{}'.format(i)
node_kwargs['resources'] = {resource: 100}
node = self.cluster.add_node(**node_kwargs)
return node, resource
def get_next_resource(self):
resource = self.resources[self.resource_idx % self.num_nodes]
self.resource_idx += 1
return resource
def exec(self, dag):
num_stages = len(dag)
actors = []
task_handles = []
for i in range(num_stages):
stage = dag.pop()
for operator in stage:
actor = OperatorActor._remote(
args=[operator],
kwargs={},
resources={self.get_next_resource(): 1})
actors.append(actor)
task_handles.append(actor.run.remote())
ray.get(task_handles)
def test_shutdown():
g = Cluster(initialize_head=False)
node = g.add_node()
node2 = g.add_node()
g.shutdown()
assert not any(n.any_processes_alive() for n in [node, node2])
object_store_memory = 10**8
num_nodes = 1
message = ("Make sure there is enough memory on this machine to run this "
"workload. We divide the system memory by 2 to provide a buffer.")
assert (num_nodes * object_store_memory + num_redis_shards * redis_max_memory <
ray.utils.get_system_memory() / 2), message
# Simulate a cluster on one machine.
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(
redis_port=6379 if i == 0 else None,
num_redis_shards=num_redis_shards if i == 0 else None,
num_cpus=10,
num_gpus=0,
resources={str(i): 2},
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory)
ray.init(redis_address=cluster.redis_address)
# Run the workload.
run_experiments({
"impala": {
"run": "IMPALA",
"env": "CartPole-v0",
"config": {
"num_workers": 8,
"num_gpus": 0,
"num_envs_per_worker": 5,
class RayExecutorQueueTest(unittest.TestCase):
def setUp(self):
self.trial_executor = RayTrialExecutor(queue_trials=True,
refresh_period=0)
self.cluster = Cluster(initialize_head=True,
connect=True,
head_node_args={
"num_cpus":
1,
"_internal_config":
json.dumps({"num_heartbeats_timeout": 10})
})
# Pytest doesn't play nicely with imports
_register_all()
def tearDown(self):
ray.shutdown()
self.cluster.shutdown()
_register_all() # re-register the evicted objects
def testQueueTrial(self):
"""Tests that reset handles NotImplemented properly."""
def create_trial(cpu, gpu=0):
return Trial("__fake", resources=Resources(cpu=cpu, gpu=gpu))
cpu_only = create_trial(1, 0)
self.assertTrue(self.trial_executor.has_resources(cpu_only.resources))
self.trial_executor.start_trial(cpu_only)
gpu_only = create_trial(0, 1)
self.assertTrue(self.trial_executor.has_resources(gpu_only.resources))
def testHeadBlocking(self):
def create_trial(cpu, gpu=0):
return Trial("__fake", resources=Resources(cpu=cpu, gpu=gpu))
gpu_trial = create_trial(1, 1)
self.assertTrue(self.trial_executor.has_resources(gpu_trial.resources))
self.trial_executor.start_trial(gpu_trial)
# TODO(rliaw): This behavior is probably undesirable, but right now
# trials with different resource requirements is not often used.
cpu_only_trial = create_trial(1, 0)
self.assertFalse(
self.trial_executor.has_resources(cpu_only_trial.resources))
self.cluster.add_node(num_cpus=1, num_gpus=1)
self.cluster.wait_for_nodes()
self.assertTrue(
self.trial_executor.has_resources(cpu_only_trial.resources))
self.trial_executor.start_trial(cpu_only_trial)
cpu_only_trial2 = create_trial(1, 0)
self.assertTrue(
self.trial_executor.has_resources(cpu_only_trial2.resources))
self.trial_executor.start_trial(cpu_only_trial2)
cpu_only_trial3 = create_trial(1, 0)
self.assertFalse(
self.trial_executor.has_resources(cpu_only_trial3.resources))
def run(args, parser):
# create exps from configs
if args.config_file:
# load configs from yaml
with open(args.config_file) as f:
exps = yaml.safe_load(f)
else:
exps = create_exps(args=args, )
arena_exps = create_arena_exps(
exps=exps,
args=args,
parser=parser,
)
# config ray cluster
if args.ray_num_nodes:
cluster = Cluster()
for ray_node in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
)
ray.init(address=cluster.redis_address, )
else:
ray.init(
address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus,
)
if len(arena_exps.keys()) > 1:
logger.warning(
"There are multiple experiments scheduled, ray==0.7.4 will run them one by one, instead of cocurrently. "
"However, recent ray can run them cocurrently. But the recent ray has failed our test (the rllib is broken)"
"This is mainly due to there are grid search used in configs that is not supported by original rllib. "
)
if args.eval:
# evaluate policies
if len(arena_exps.keys()) < 1:
raise ValueError
elif len(arena_exps.keys()) >= 1:
if len(arena_exps.keys()) > 1:
arena_exp_key = inquire_select(
choices=list(arena_exps.keys()),
key="arena_exp_key",
)
else:
# if there is just one arena_exps
arena_exp_key = list(arena_exps.keys())[0]
logger.info("Evaluating arena_exp_key: {}".format(arena_exp_key, ))
arena_exp = arena_exps[arena_exp_key]
answers = prompt(
[{
'type': 'input',
'name': 'eval_log_path',
'message':
'Where do you want to log the results of this evaluation?',
'default': '../eval_log_path/'
}],
style=custom_style_2,
)
prepare_path(answers['eval_log_path'])
from ray.rllib.evaluation.rollout_worker import RolloutWorker
# worker = ArenaRolloutWorker(
# TODO: RolloutWorker does not support monitor for multi-agent envs
worker = RolloutWorker(
env_creator=lambda _: ArenaRllibEnv(
env=arena_exp["env"],
env_config=arena_exp["config"]["env_config"],
),
policy=arena_exp["config"]["multiagent"]["policies"],
policy_mapping_fn=arena_exp["config"]["multiagent"]
["policy_mapping_fn"],
batch_mode="complete_episodes",
batch_steps=500,
num_envs=1,
monitor_path=answers['eval_log_path'],
)
logger.info("Testing worker...")
sample_start = time.time()
worker.sample()
sample_time = time.time() - sample_start
logger.info("Finish testing worker.")
policy_ids = list(worker.policy_map.keys())
checkpoints = inquire_checkpoints(
local_dir=arena_exp["local_dir"],
policy_ids=policy_ids,
)
checkpoint_paths = checkpoints_2_checkpoint_paths(checkpoints)
num_checkpoint_paths = {}
for policy_id, checkpoint_paths_per_policy_id in checkpoint_paths.items(
):
num_checkpoint_paths[policy_id] = len(
checkpoint_paths_per_policy_id)
num_sampling = np.prod(list(num_checkpoint_paths.values()))
confirm = inquire_confirm(
"You have scheduled {} sampling, each sampling will take {} minutes, which means {} hours in total."
.format(
num_sampling,
sample_time / 60.0,
num_sampling * sample_time / 60.0 / 60.0,
))
if not confirm:
os.exit()
result_matrix = run_result_matrix(
checkpoint_paths=checkpoint_paths,
worker=worker,
)
result_matrix = np.asarray(result_matrix)
vis_result_matrix(
result_matrix=result_matrix,
log_path=answers['eval_log_path'],
)
else:
run_experiments(
arena_exps,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume,
)
