def ray_start_empty_cluster(): cluster = Cluster() yield cluster # The code after the yield will run as teardown code. ray.shutdown() cluster.shutdown()
def main(): cluster = Cluster(initialize_head=True, connect=True, head_node_args={ "object_store_memory": 20 * 1024 * 1024 * 1024, "num_cpus": 16 }) cluster.add_node(object_store_memory=20 * 1024 * 1024 * 1024, num_gpus=1, num_cpus=16) object_id_list = [] for i in range(0, 10): object_id = ray.put(np.random.rand(1024 * 128, 1024)) object_id_list.append(object_id) @ray.remote(num_gpus=1) def f(object_id_list): diffs = [] for object_id in object_id_list: before = time.time() ray.get(object_id) after = time.time() diffs.append(after - before) time.sleep(1) return np.mean(diffs), np.std(diffs) time_diff, time_diff_std = ray.get(f.remote(object_id_list)) print("latency to get an 1G object over network", round(time_diff, 2), "+-", round(time_diff_std, 2)) ray.shutdown() cluster.shutdown()
def start_local_cluster(num_nodes, object_store_memory): """Start a local Ray cluster. The ith node in the cluster will have a resource named "i". Args: num_nodes: The number of nodes to start in the cluster. Returns: The cluster object. """ num_redis_shards = 2 redis_max_memory = 10**8 cluster = Cluster() for i in range(num_nodes): cluster.add_node( redis_port=6379 if i == 0 else None, num_redis_shards=num_redis_shards if i == 0 else None, num_cpus=8 if i == 0 else 2, num_gpus=0, resources={str(i): 500}, object_store_memory=object_store_memory, redis_max_memory=redis_max_memory) ray.init(address=cluster.address) return cluster
def __init__(self, **kwargs): super(RayExecutor, self).__init__() mode = kwargs.get('ray_mode', 'local') self.resource_idx = 0 if mode == 'local': node_kwargs = { 'num_cpus': 4, 'object_store_memory': 10**9, 'resources': { 'Node_0': 100 } } self.cluster = Cluster(initialize_head=True, head_node_args=node_kwargs) self.num_nodes = kwargs.get('ray_num_nodes', 4) self.nodes = [] self.resources = [] i = 1 for _ in range(self.num_nodes): node, resource = self._create_local_node(i, node_kwargs) self.nodes.append(node) self.resources.append(resource) self._create_local_node(i, node_kwargs) redis_address = self.cluster.redis_address ray.init(redis_address=redis_address) else: redis_address = kwargs.get('redis_address', '127.0.0.1') ray.init(redis_address=redis_address) self.resources = [] self.nodes = ray.global_state.client_table() for node in self.nodes: for resource in node['Resources']: if 'Node' in resource and resource != 'Node_0': self.resources.append(resource)
def ray_start_reconstruction(request): num_nodes = request.param plasma_store_memory = int(0.1 * 10**9) cluster = Cluster( initialize_head=True, head_node_args={ "num_cpus": 1, "object_store_memory": plasma_store_memory // num_nodes, "redis_max_memory": 10**7, "_internal_config": json.dumps({"initial_reconstruction_timeout_milliseconds": 200}) }) for i in range(num_nodes - 1): cluster.add_node(num_cpus=1, object_store_memory=plasma_store_memory // num_nodes, _internal_config=json.dumps({ "initial_reconstruction_timeout_milliseconds": 200 })) ray.init(redis_address=cluster.redis_address) yield plasma_store_memory, num_nodes, cluster # Clean up the Ray cluster. ray.shutdown() cluster.shutdown()
def create_cluster(num_nodes): cluster = Cluster() for i in range(num_nodes): cluster.add_node(resources={str(i): 100}, object_store_memory=10**9) ray.init(address=cluster.address) return cluster
def test_connect_with_disconnected_node(shutdown_only): config = json.dumps({ "num_heartbeats_timeout": 50, "heartbeat_timeout_milliseconds": 10, }) cluster = Cluster() cluster.add_node(num_cpus=0, _internal_config=config) ray.init(redis_address=cluster.redis_address) info = relevant_errors(ray_constants.REMOVED_NODE_ERROR) assert len(info) == 0 # This node is killed by SIGKILL, ray_monitor will mark it to dead. dead_node = cluster.add_node(num_cpus=0, _internal_config=config) cluster.remove_node(dead_node, allow_graceful=False) wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 1, timeout=2) # This node is killed by SIGKILL, ray_monitor will mark it to dead. dead_node = cluster.add_node(num_cpus=0, _internal_config=config) cluster.remove_node(dead_node, allow_graceful=False) wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 2, timeout=2) # This node is killed by SIGTERM, ray_monitor will not mark it again. removing_node = cluster.add_node(num_cpus=0, _internal_config=config) cluster.remove_node(removing_node, allow_graceful=True) with pytest.raises(Exception, match=("Timing out of wait.")): wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 3, timeout=2) # There is no connection error to a dead node. info = relevant_errors(ray_constants.RAYLET_CONNECTION_ERROR) assert len(info) == 0
def run(args, parser): if args.config_file: with open(args.config_file) as f: experiments = yaml.safe_load(f) else: # Note: keep this in sync with tune/config_parser.py experiments = { args.experiment_name: { # i.e. log to ~/ray_results/default "run": args.run, "checkpoint_freq": args.checkpoint_freq, "keep_checkpoints_num": args.keep_checkpoints_num, "checkpoint_score_attr": args.checkpoint_score_attr, "local_dir": args.local_dir, "resources_per_trial": ( args.resources_per_trial and resources_to_json(args.resources_per_trial)), "stop": args.stop, "config": dict(args.config, env=args.env), "restore": args.restore, "num_samples": args.num_samples, "upload_dir": args.upload_dir, } } for exp in experiments.values(): if not exp.get("run"): parser.error("the following arguments are required: --run") if not exp.get("env") and not exp.get("config", {}).get("env"): parser.error("the following arguments are required: --env") if args.eager: exp["config"]["eager"] = True if args.trace: if not exp["config"].get("eager"): raise ValueError("Must enable --eager to enable tracing.") exp["config"]["eager_tracing"] = True if args.ray_num_nodes: cluster = Cluster() for _ in range(args.ray_num_nodes): cluster.add_node(num_cpus=args.ray_num_cpus or 1, num_gpus=args.ray_num_gpus or 0, object_store_memory=args.ray_object_store_memory, memory=args.ray_memory, redis_max_memory=args.ray_redis_max_memory) ray.init(address=cluster.address) else: ray.init(address=args.ray_address, object_store_memory=args.ray_object_store_memory, memory=args.ray_memory, redis_max_memory=args.ray_redis_max_memory, num_cpus=args.ray_num_cpus, num_gpus=args.ray_num_gpus) run_experiments(experiments, scheduler=_make_scheduler(args), queue_trials=args.queue_trials, resume=args.resume)
def test_cluster(): """Basic test for adding and removing nodes in cluster.""" g = Cluster(initialize_head=False) node = g.add_node() node2 = g.add_node() assert node.remaining_processes_alive() assert node2.remaining_processes_alive() g.remove_node(node2) g.remove_node(node) assert not any(n.any_processes_alive() for n in [node, node2])
def _start_new_cluster(): cluster = Cluster(initialize_head=True, connect=True, head_node_args={ "num_cpus": 1, "_internal_config": json.dumps({"num_heartbeats_timeout": 10}) }) # Pytest doesn't play nicely with imports _register_all() return cluster
def test_redis_password_cluster(self, password, shutdown_only): @ray.remote def f(): return 1 node_args = {"redis_password": password} cluster = Cluster( initialize_head=True, connect=True, head_node_args=node_args) cluster.add_node(**node_args) object_id = f.remote() ray.get(object_id)
def ray_start_workers_separate_multinode(request): num_nodes = request.param[0] num_initial_workers = request.param[1] # Start the Ray processes. cluster = Cluster() for _ in range(num_nodes): cluster.add_node(num_cpus=num_initial_workers) ray.init(address=cluster.address) yield num_nodes, num_initial_workers # The code after the yield will run as teardown code. ray.shutdown() cluster.shutdown()
def cluster_start(): # Start the Ray processes. cluster = Cluster(initialize_head=True, connect=True, head_node_args={ "num_cpus": 1, "_internal_config": json.dumps({"num_heartbeats_timeout": 10}) }) yield cluster ray.shutdown() cluster.shutdown()
def start_connected_longer_cluster(): """Creates a cluster with a longer timeout.""" g = Cluster(initialize_head=True, connect=True, head_node_args={ "num_cpus": 1, "_internal_config": json.dumps({"num_heartbeats_timeout": 20}) }) yield g # The code after the yield will run as teardown code. ray.shutdown() g.shutdown()
def start_connected_cluster(): # Start the Ray processes. g = Cluster(initialize_head=True, connect=True, head_node_args={ "num_cpus": 1, "_internal_config": json.dumps({"num_heartbeats_timeout": 10}) }) yield g # The code after the yield will run as teardown code. ray.shutdown() g.shutdown()
def ray_start_two_nodes(): # Start the Ray processes. cluster = Cluster() for _ in range(2): cluster.add_node(num_cpus=0, _internal_config=json.dumps( {"num_heartbeats_timeout": 40})) ray.init(redis_address=cluster.redis_address) yield cluster # The code after the yield will run as teardown code. ray.shutdown() cluster.shutdown()
def setUp(self): self.trial_executor = RayTrialExecutor(queue_trials=True, refresh_period=0) self.cluster = Cluster(initialize_head=True, connect=True, head_node_args={ "num_cpus": 1, "_internal_config": json.dumps({"num_heartbeats_timeout": 10}) }) # Pytest doesn't play nicely with imports _register_all()
def ray_start_combination(request): num_nodes = request.param[0] num_workers_per_scheduler = request.param[1] # Start the Ray processes. cluster = Cluster(initialize_head=True, head_node_args={ "num_cpus": 10, "redis_max_memory": 10**7 }) for i in range(num_nodes - 1): cluster.add_node(num_cpus=10) ray.init(redis_address=cluster.redis_address) yield num_nodes, num_workers_per_scheduler, cluster # The code after the yield will run as teardown code. ray.shutdown() cluster.shutdown()
def start_connected_emptyhead_cluster(): """Starts head with no resources.""" cluster = Cluster(initialize_head=True, connect=True, head_node_args={ "num_cpus": 0, "_internal_config": json.dumps({"num_heartbeats_timeout": 10}) }) # Pytest doesn't play nicely with imports _register_all() yield cluster # The code after the yield will run as teardown code. ray.shutdown() cluster.shutdown()
def ray_start_cluster(): node_args = { "num_cpus": 4, "_internal_config": json.dumps({ "initial_reconstruction_timeout_milliseconds": 1000, "num_heartbeats_timeout": 10 }) } # Start with 3 worker nodes and 4 cores each. cluster = Cluster( initialize_head=True, connect=True, head_node_args=node_args) workers = [] for _ in range(3): workers.append(cluster.add_node(**node_args)) cluster.wait_for_nodes() yield cluster ray.shutdown() cluster.shutdown()
def test_temp_plasma_store_socket(): ray.init(plasma_store_socket_name="/tmp/i_am_a_temp_socket") assert os.path.exists( "/tmp/i_am_a_temp_socket"), "Specified socket path not found." ray.shutdown() try: os.remove("/tmp/i_am_a_temp_socket") except OSError: pass # It could have been removed by Ray. cluster = Cluster(True) cluster.add_node(plasma_store_socket_name="/tmp/i_am_a_temp_socket_2") assert os.path.exists( "/tmp/i_am_a_temp_socket_2"), "Specified socket path not found." cluster.shutdown() try: os.remove("/tmp/i_am_a_temp_socket_2") except OSError: pass # It could have been removed by Ray.
def ray_initialize_cluster(): # Start with 4 workers and 4 cores. num_nodes = 4 num_workers_per_scheduler = 8 cluster = Cluster() for _ in range(num_nodes): cluster.add_node( num_cpus=num_workers_per_scheduler, _internal_config=json.dumps({ "initial_reconstruction_timeout_milliseconds": 1000, "num_heartbeats_timeout": 10, })) ray.init(redis_address=cluster.redis_address) yield cluster ray.shutdown() cluster.shutdown()
def _ray_start_cluster(**kwargs): init_kwargs = get_default_fixture_ray_kwargs() num_nodes = 0 do_init = False # num_nodes & do_init are not arguments for ray.init, so delete them. if "num_nodes" in kwargs: num_nodes = kwargs["num_nodes"] del kwargs["num_nodes"] if "do_init" in kwargs: do_init = kwargs["do_init"] del kwargs["do_init"] elif num_nodes > 0: do_init = True init_kwargs.update(kwargs) cluster = Cluster() remote_nodes = [] for _ in range(num_nodes): remote_nodes.append(cluster.add_node(**init_kwargs)) if do_init: ray.init(redis_address=cluster.redis_address) yield cluster # The code after the yield will run as teardown code. ray.shutdown() cluster.shutdown()
from ray.tests.cluster_utils import Cluster from ray.test_utils import run_string_as_driver num_redis_shards = 5 redis_max_memory = 10**8 object_store_memory = 10**8 num_nodes = 4 message = ("Make sure there is enough memory on this machine to run this " "workload. We divide the system memory by 2 to provide a buffer.") assert (num_nodes * object_store_memory + num_redis_shards * redis_max_memory < ray.utils.get_system_memory() / 2) # Simulate a cluster on one machine. cluster = Cluster() for i in range(num_nodes): cluster.add_node(redis_port=6379 if i == 0 else None, num_redis_shards=num_redis_shards if i == 0 else None, num_cpus=4, num_gpus=0, resources={str(i): 5}, object_store_memory=object_store_memory, redis_max_memory=redis_max_memory) ray.init(address=cluster.address) # Run the workload. # Define a driver script that runs a few tasks and actors on each node in the # cluster. driver_script = """
def run(args, parser): if args.config_file: with open(args.config_file) as f: experiments = yaml.safe_load(f) # add callbacks for self-defined metric # and save successful transitions from RL agents experiment_name = next(iter(experiments)) experiments[experiment_name]["config"]["optimizer"]["robot_demo_path"] = dir_path experiments[experiment_name]["config"]["callbacks"] = { "on_episode_start": on_episode_start, "on_episode_step": on_episode_step, "on_episode_end": on_episode_end, "on_sample_end": on_sample_end, "on_train_result": on_train_result, "on_postprocess_traj": on_postprocess_traj } else: # Note: keep this in sync with tune/config_parser.py experiments = { args.experiment_name: { # i.e. log to ~/ray_results/default "run": args.run, "checkpoint_freq": args.checkpoint_freq, "keep_checkpoints_num": args.keep_checkpoints_num, "checkpoint_score_attr": args.checkpoint_score_attr, "local_dir": args.local_dir, "resources_per_trial": ( args.resources_per_trial and resources_to_json(args.resources_per_trial)), "stop": args.stop, "config": dict(args.config, env=args.env), "restore": args.restore, "num_samples": args.num_samples, "upload_dir": args.upload_dir, } } for exp in experiments.values(): if not exp.get("run"): parser.error("the following arguments are required: --run") if not exp.get("env") and not exp.get("config", {}).get("env"): parser.error("the following arguments are required: --env") if args.eager: exp["config"]["eager"] = True if args.trace: if not exp["config"].get("eager"): raise ValueError("Must enable --eager to enable tracing.") exp["config"]["eager_tracing"] = True if args.ray_num_nodes: cluster = Cluster() for _ in range(args.ray_num_nodes): cluster.add_node( num_cpus=args.ray_num_cpus or 1, num_gpus=args.ray_num_gpus or 0, object_store_memory=args.ray_object_store_memory, memory=args.ray_memory, redis_max_memory=args.ray_redis_max_memory) ray.init(address=cluster.address) #, log_to_driver=False) else: ray.init( address=args.ray_address, object_store_memory=args.ray_object_store_memory, memory=args.ray_memory, redis_max_memory=args.ray_redis_max_memory, num_cpus=args.ray_num_cpus, num_gpus=args.ray_num_gpus) # log_to_driver=False) # disable the loggings # https://github.com/ray-project/ray/issues/5048 run_experiments( experiments, scheduler=_make_scheduler(args), queue_trials=args.queue_trials, resume=args.resume)
def run_experiment(args, parser): # args.ray_object_store_memory = int(1e10) args.ray_redis_max_memory = int(2e9) if args.config_file: with open(args.config_file) as f: exp = yaml.load(f) else: raise Exception('No config file!') exp = merge_dicts(exp, args.config) log.info('Num workers: %d, num_envs_per_worker: %d', exp['config']['num_workers'], exp['config']['num_envs_per_worker']) if args.cfg_mixins is not None: for cfg_mixin_file in args.cfg_mixins: with open(cfg_mixin_file, 'r') as f: override_cfg = yaml.load(f) log.info('Overriding parameters from %s: %r', cfg_mixin_file, override_cfg) exp = merge_dicts(exp, override_cfg) if not exp.get("run"): parser.error("the following arguments are required: --run") if not exp.get("env") and not exp.get("config", {}).get("env"): parser.error("the following arguments are required: --env") if args.ray_num_nodes: cluster = Cluster() for _ in range(args.ray_num_nodes): cluster.add_node( num_cpus=args.ray_num_cpus or 1, num_gpus=args.ray_num_gpus or 0, object_store_memory=args.ray_object_store_memory, redis_max_memory=args.ray_redis_max_memory, ) ray.init(redis_address=cluster.redis_address, local_mode=args.local_mode) else: ray.init( redis_address=args.redis_address, object_store_memory=args.ray_object_store_memory, redis_max_memory=args.ray_redis_max_memory, num_cpus=args.ray_num_cpus, num_gpus=args.ray_num_gpus, local_mode=args.local_mode, ) exp = Experiment.from_json(args.experiment_name, exp) exp.spec['checkpoint_freq'] = 20 if args.pbt: exp.spec['checkpoint_freq'] = 3 exp.spec['checkpoint_at_end'] = True # exp.spec['checkpoint_score_attr'] = 'episode_reward_mean' exp.spec['keep_checkpoints_num'] = 5 if args.stop_seconds > 0: exp.spec['stop'] = {'time_total_s': args.stop_seconds} # if 'multiagent' in exp.spec['config']: # # noinspection PyProtectedMember # make_env = ray.tune.registry._global_registry.get(ENV_CREATOR, exp.spec['config']['env']) # temp_env = make_env(None) # obs_space, action_space = temp_env.observation_space, temp_env.action_space # temp_env.close() # del temp_env # # policies = dict( # main=(None, obs_space, action_space, {}), # dummy=(None, obs_space, action_space, {}), # ) # # exp.spec['config']['multiagent'] = { # 'policies': policies, # 'policy_mapping_fn': function(lambda agent_id: 'main'), # 'policies_to_train': ['main'], # } # # if args.dbg: # exp.spec['config']['num_workers'] = 1 # exp.spec['config']['num_gpus'] = 1 # exp.spec['config']['num_envs_per_worker'] = 1 # # if 'callbacks' not in exp.spec['config']: # exp.spec['config']['callbacks'] = {} # # fps_helper = FpsHelper() # # def on_train_result(info): # if 'APPO' in exp.spec['run']: # samples = info['result']['info']['num_steps_sampled'] # else: # samples = info['trainer'].optimizer.num_steps_trained # # fps_helper.record(samples) # fps = fps_helper.get_fps() # info['result']['custom_metrics']['fps'] = fps # # # remove this as currently # skip_frames = exp.spec['config']['env_config']['skip_frames'] # info['result']['custom_metrics']['fps_frameskip'] = fps * skip_frames # # exp.spec['config']['callbacks']['on_train_result'] = function(on_train_result) # # def on_episode_end(info): # episode = info['episode'] # stats = { # 'DEATHCOUNT': 0, # 'FRAGCOUNT': 0, # 'HITCOUNT': 0, # 'DAMAGECOUNT': 0, # 'KDR': 0, # 'FINAL_PLACE': 0, # 'LEADER_GAP': 0, # 'PLAYER_COUNT': 0, # 'BOT_DIFFICULTY': 0, # } # # # noinspection PyProtectedMember # agent_to_last_info = episode._agent_to_last_info # for agent in agent_to_last_info.keys(): # agent_info = agent_to_last_info[agent] # for stats_key in stats.keys(): # stats[stats_key] += agent_info.get(stats_key, 0.0) # # for stats_key in stats.keys(): # stats[stats_key] /= len(agent_to_last_info.keys()) # # episode.custom_metrics.update(stats) # # exp.spec['config']['callbacks']['on_episode_end'] = function(on_episode_end) extra_kwargs = {} if args.pbt: extra_kwargs['reuse_actors'] = False run(exp, name=args.experiment_name, scheduler=make_custom_scheduler(args), resume=args.resume, queue_trials=args.queue_trials, **extra_kwargs)
def test_shutdown(): g = Cluster(initialize_head=False) node = g.add_node() node2 = g.add_node() g.shutdown() assert not any(n.any_processes_alive() for n in [node, node2])
def run(args, parser): # create exps from configs if args.config_file: # load configs from yaml with open(args.config_file) as f: exps = yaml.safe_load(f) else: exps = create_exps(args=args, ) arena_exps = create_arena_exps( exps=exps, args=args, parser=parser, ) # config ray cluster if args.ray_num_nodes: cluster = Cluster() for ray_node in range(args.ray_num_nodes): cluster.add_node( num_cpus=args.ray_num_cpus or 1, num_gpus=args.ray_num_gpus or 0, object_store_memory=args.ray_object_store_memory, memory=args.ray_memory, redis_max_memory=args.ray_redis_max_memory, ) ray.init(address=cluster.redis_address, ) else: ray.init( address=args.ray_address, object_store_memory=args.ray_object_store_memory, memory=args.ray_memory, redis_max_memory=args.ray_redis_max_memory, num_cpus=args.ray_num_cpus, num_gpus=args.ray_num_gpus, ) if len(arena_exps.keys()) > 1: logger.warning( "There are multiple experiments scheduled, ray==0.7.4 will run them one by one, instead of cocurrently. " "However, recent ray can run them cocurrently. But the recent ray has failed our test (the rllib is broken)" "This is mainly due to there are grid search used in configs that is not supported by original rllib. " ) if args.eval: # evaluate policies if len(arena_exps.keys()) < 1: raise ValueError elif len(arena_exps.keys()) >= 1: if len(arena_exps.keys()) > 1: arena_exp_key = inquire_select( choices=list(arena_exps.keys()), key="arena_exp_key", ) else: # if there is just one arena_exps arena_exp_key = list(arena_exps.keys())[0] logger.info("Evaluating arena_exp_key: {}".format(arena_exp_key, )) arena_exp = arena_exps[arena_exp_key] answers = prompt( [{ 'type': 'input', 'name': 'eval_log_path', 'message': 'Where do you want to log the results of this evaluation?', 'default': '../eval_log_path/' }], style=custom_style_2, ) prepare_path(answers['eval_log_path']) from ray.rllib.evaluation.rollout_worker import RolloutWorker # worker = ArenaRolloutWorker( # TODO: RolloutWorker does not support monitor for multi-agent envs worker = RolloutWorker( env_creator=lambda _: ArenaRllibEnv( env=arena_exp["env"], env_config=arena_exp["config"]["env_config"], ), policy=arena_exp["config"]["multiagent"]["policies"], policy_mapping_fn=arena_exp["config"]["multiagent"] ["policy_mapping_fn"], batch_mode="complete_episodes", batch_steps=500, num_envs=1, monitor_path=answers['eval_log_path'], ) logger.info("Testing worker...") sample_start = time.time() worker.sample() sample_time = time.time() - sample_start logger.info("Finish testing worker.") policy_ids = list(worker.policy_map.keys()) checkpoints = inquire_checkpoints( local_dir=arena_exp["local_dir"], policy_ids=policy_ids, ) checkpoint_paths = checkpoints_2_checkpoint_paths(checkpoints) num_checkpoint_paths = {} for policy_id, checkpoint_paths_per_policy_id in checkpoint_paths.items( ): num_checkpoint_paths[policy_id] = len( checkpoint_paths_per_policy_id) num_sampling = np.prod(list(num_checkpoint_paths.values())) confirm = inquire_confirm( "You have scheduled {} sampling, each sampling will take {} minutes, which means {} hours in total." .format( num_sampling, sample_time / 60.0, num_sampling * sample_time / 60.0 / 60.0, )) if not confirm: os.exit() result_matrix = run_result_matrix( checkpoint_paths=checkpoint_paths, worker=worker, ) result_matrix = np.asarray(result_matrix) vis_result_matrix( result_matrix=result_matrix, log_path=answers['eval_log_path'], ) else: run_experiments( arena_exps, scheduler=_make_scheduler(args), queue_trials=args.queue_trials, resume=args.resume, )