def testIgnoresCorruptedConfig(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() autoscaler = StandardAutoscaler(config_path, LoadMetrics(), max_concurrent_launches=10, max_failures=0, update_interval_s=0) autoscaler.update() # Write a corrupted config self.write_config("asdf") for _ in range(10): autoscaler.update() self.assertEqual(len(self.provider.nodes({})), 2) # New a good config again new_config = SMALL_CLUSTER.copy() new_config["min_workers"] = 10 new_config["max_workers"] = 10 self.write_config(new_config) autoscaler.update() self.assertEqual(len(self.provider.nodes({})), 10)
def testUpdateThrottling(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() autoscaler = StandardAutoscaler(config_path, LoadMetrics(), max_launch_batch=5, max_concurrent_launches=5, max_failures=0, process_runner=runner, update_interval_s=10) autoscaler.update() self.waitForNodes(2) assert autoscaler.num_launches_pending.value == 0 new_config = SMALL_CLUSTER.copy() new_config["max_workers"] = 1 self.write_config(new_config) autoscaler.update() # not updated yet # note that node termination happens in the main thread, so # we do not need to add any delay here before checking assert len(self.provider.non_terminated_nodes({})) == 2 assert autoscaler.num_launches_pending.value == 0
def testTerminateOutdatedNodesGracefully(self): config = SMALL_CLUSTER.copy() config["min_workers"] = 5 config["max_workers"] = 5 config_path = self.write_config(config) self.provider = MockProvider() self.provider.create_node({}, {TAG_RAY_NODE_TYPE: "worker"}, 10) runner = MockProcessRunner() autoscaler = StandardAutoscaler(config_path, LoadMetrics(), max_failures=0, process_runner=runner, update_interval_s=0) self.waitForNodes(10) # Gradually scales down to meet target size, never going too low for _ in range(10): autoscaler.update() self.waitForNodes(5, comparison=self.assertLessEqual) self.waitForNodes(4, comparison=self.assertGreaterEqual) # Eventually reaches steady state self.waitForNodes(5)
def testConfiguresOutdatedNodes(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() autoscaler = StandardAutoscaler(config_path, LoadMetrics(), max_failures=0, process_runner=runner, update_interval_s=0) autoscaler.update() autoscaler.update() self.waitForNodes(2) for node in self.provider.mock_nodes.values(): node.state = "running" autoscaler.update() self.waitForNodes(2, tag_filters={TAG_RAY_NODE_STATUS: "up-to-date"}) runner.calls = [] new_config = SMALL_CLUSTER.copy() new_config["worker_setup_commands"] = ["cmdX", "cmdY"] self.write_config(new_config) autoscaler.update() autoscaler.update() self.waitFor(lambda: len(runner.calls) > 0)
def testConfiguresNewNodes(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() autoscaler = StandardAutoscaler( config_path, LoadMetrics(), max_failures=0, process_runner=runner, verbose_updates=True, node_updater_cls=NodeUpdaterThread, update_interval_s=0) autoscaler.update() autoscaler.update() self.waitForNodes(2) for node in self.provider.mock_nodes.values(): node.state = "running" assert len( self.provider.nodes({ TAG_RAY_NODE_STATUS: "uninitialized" })) == 2 autoscaler.update() self.waitForNodes(2, tag_filters={TAG_RAY_NODE_STATUS: "up-to-date"})
def testScaleUpIgnoreUsed(self): config = MULTI_WORKER_CLUSTER.copy() # Commenting out this line causes the test case to fail?!?! config["min_workers"] = 0 config["target_utilization_fraction"] = 1.0 config_path = self.write_config(config) self.provider = MockProvider() self.provider.create_node({}, { TAG_RAY_NODE_KIND: "head", TAG_RAY_USER_NODE_TYPE: "p2.xlarge" }, 1) head_ip = self.provider.non_terminated_node_ips({})[0] self.provider.finish_starting_nodes() runner = MockProcessRunner() lm = LoadMetrics(local_ip=head_ip) autoscaler = StandardAutoscaler( config_path, lm, max_failures=0, process_runner=runner, update_interval_s=0) autoscaler.update() self.waitForNodes(1) lm.update(head_ip, {"CPU": 4, "GPU": 1}, {}, {}) self.waitForNodes(1) lm.update( head_ip, { "CPU": 4, "GPU": 1 }, {"GPU": 1}, {}, waiting_bundles=[{ "GPU": 1 }]) autoscaler.update() self.waitForNodes(2) assert self.provider.mock_nodes[1].node_type == "p2.xlarge"
def testDontScaleBelowTarget(self): config = SMALL_CLUSTER.copy() config["min_workers"] = 0 config["max_workers"] = 2 config["target_utilization_fraction"] = 0.5 config_path = self.write_config(config) self.provider = MockProvider() lm = LoadMetrics() runner = MockProcessRunner() autoscaler = StandardAutoscaler(config_path, lm, max_failures=0, process_runner=runner, update_interval_s=0) assert len(self.provider.non_terminated_nodes({})) == 0 autoscaler.update() assert autoscaler.pending_launches.value == 0 assert len(self.provider.non_terminated_nodes({})) == 0 # Scales up as nodes are reported as used local_ip = services.get_node_ip_address() lm.update(local_ip, {"CPU": 2}, {"CPU": 0}, {}) # head # 1.0 nodes used => target nodes = 2 => target workers = 1 autoscaler.update() self.waitForNodes(1) # Make new node idle, and never used. # Should hold steady as target is still 2. lm.update("172.0.0.0", {"CPU": 0}, {"CPU": 0}, {}) lm.last_used_time_by_ip["172.0.0.0"] = 0 autoscaler.update() assert len(self.provider.non_terminated_nodes({})) == 1 # Reduce load on head => target nodes = 1 => target workers = 0 lm.update(local_ip, {"CPU": 2}, {"CPU": 1}, {}) autoscaler.update() assert len(self.provider.non_terminated_nodes({})) == 0
def testSetupCommandsWithNoNodeCaching(self): config = SMALL_CLUSTER.copy() config["min_workers"] = 1 config["max_workers"] = 1 config_path = self.write_config(config) self.provider = MockProvider(cache_stopped=False) runner = MockProcessRunner() lm = LoadMetrics() autoscaler = StandardAutoscaler(config_path, lm, max_failures=0, process_runner=runner, update_interval_s=0) autoscaler.update() self.waitForNodes(1) self.provider.finish_starting_nodes() autoscaler.update() self.waitForNodes(1, tag_filters={TAG_RAY_NODE_STATUS: STATUS_UP_TO_DATE}) runner.assert_has_call("172.0.0.0", "init_cmd") runner.assert_has_call("172.0.0.0", "setup_cmd") runner.assert_has_call("172.0.0.0", "worker_setup_cmd") runner.assert_has_call("172.0.0.0", "start_ray_worker") # Check the node was not reused self.provider.terminate_node(0) autoscaler.update() self.waitForNodes(1) runner.clear_history() self.provider.finish_starting_nodes() autoscaler.update() self.waitForNodes(1, tag_filters={TAG_RAY_NODE_STATUS: STATUS_UP_TO_DATE}) runner.assert_has_call("172.0.0.1", "init_cmd") runner.assert_has_call("172.0.0.1", "setup_cmd") runner.assert_has_call("172.0.0.1", "worker_setup_cmd") runner.assert_has_call("172.0.0.1", "start_ray_worker")
def __init__(self, redis_address, redis_port, autoscaling_config): # Initialize the Redis clients. self.state = ray.experimental.state.GlobalState() self.state._initialize_global_state(redis_address, redis_port) self.redis = redis.StrictRedis( host=redis_address, port=redis_port, db=0) # TODO(swang): Update pubsub client to use ray.experimental.state once # subscriptions are implemented there. self.subscribe_client = self.redis.pubsub() self.subscribed = {} # Initialize data structures to keep track of the active database # clients. self.dead_local_schedulers = set() self.live_plasma_managers = Counter() self.dead_plasma_managers = set() # Keep a mapping from local scheduler client ID to IP address to use # for updating the load metrics. self.local_scheduler_id_to_ip_map = dict() self.load_metrics = LoadMetrics() if autoscaling_config: self.autoscaler = StandardAutoscaler( autoscaling_config, self.load_metrics) else: self.autoscaler = None
def testRecoverUnhealthyWorkers(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() lm = LoadMetrics() autoscaler = StandardAutoscaler(config_path, lm, max_failures=0, process_runner=runner, verbose_updates=True, node_updater_cls=NodeUpdaterThread, update_interval_s=0) autoscaler.update() self.waitForNodes(2) for node in self.provider.mock_nodes.values(): node.state = "running" autoscaler.update() self.waitForNodes(2, tag_filters={TAG_RAY_NODE_STATUS: "up-to-date"}) # Mark a node as unhealthy lm.last_heartbeat_time_by_ip["172.0.0.0"] = 0 num_calls = len(runner.calls) autoscaler.update() self.waitFor(lambda: len(runner.calls) > num_calls, num_retries=150)
def __init__(self, redis_address, autoscaling_config, redis_password=None): # Initialize the Redis clients. ray.state.state._initialize_global_state(redis_address, redis_password=redis_password) self.redis = ray.services.create_redis_client(redis_address, password=redis_password) # Set the redis client and mode so _internal_kv works for autoscaler. worker = ray.worker.global_worker worker.redis_client = self.redis worker.mode = 0 # Setup subscriptions to the primary Redis server and the Redis shards. self.primary_subscribe_client = self.redis.pubsub( ignore_subscribe_messages=True) # Keep a mapping from raylet client ID to IP address to use # for updating the load metrics. self.raylet_id_to_ip_map = {} self.load_metrics = LoadMetrics() if autoscaling_config: self.autoscaler = StandardAutoscaler(autoscaling_config, self.load_metrics) self.autoscaling_config = autoscaling_config else: self.autoscaler = None self.autoscaling_config = None
def testRequestBundlesAccountsForHeadNode(self): config = MULTI_WORKER_CLUSTER.copy() config["head_node_type"] = "p2.8xlarge" config["min_workers"] = 0 config["max_workers"] = 50 config_path = self.write_config(config) self.provider = MockProvider() self.provider.create_node({}, { TAG_RAY_USER_NODE_TYPE: "p2.8xlarge", TAG_RAY_NODE_KIND: "head" }, 1) runner = MockProcessRunner() autoscaler = StandardAutoscaler( config_path, LoadMetrics(), max_failures=0, process_runner=runner, update_interval_s=0) assert len(self.provider.non_terminated_nodes({})) == 1 # These requests fit on the head node. autoscaler.update() self.waitForNodes(1) autoscaler.request_resources([{"CPU": 1}]) autoscaler.update() self.waitForNodes(1) assert len(self.provider.mock_nodes) == 1 autoscaler.request_resources([{"GPU": 8}]) autoscaler.update() self.waitForNodes(1) # This request requires an additional worker node. autoscaler.request_resources([{"GPU": 8}] * 2) autoscaler.update() self.waitForNodes(2) assert self.provider.mock_nodes[1].node_type == "p2.8xlarge"
def testConfiguresOutdatedNodes(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() runner = MockProcessRunner() autoscaler = StandardAutoscaler(config_path, max_failures=0, process_runner=runner, verbose_updates=True, node_updater_cls=NodeUpdaterThread) autoscaler.update() autoscaler.update() self.assertEqual(len(self.provider.nodes({})), 2) for node in self.provider.mock_nodes.values(): node.state = "running" autoscaler.update() self.waitFor(lambda: len( self.provider.nodes({TAG_RAY_NODE_STATUS: "Up-to-date"})) == 2) runner.calls = [] new_config = SMALL_CLUSTER.copy() new_config["worker_init_commands"] = ["cmdX", "cmdY"] self.write_config(new_config) autoscaler.update() autoscaler.update() self.waitFor(lambda: len(runner.calls) > 0)
def testDynamicScaling(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() autoscaler = StandardAutoscaler(config_path, max_concurrent_launches=5, max_failures=0) self.assertEqual(len(self.provider.nodes({})), 0) autoscaler.update() self.assertEqual(len(self.provider.nodes({})), 2) # Update the config to reduce the cluster size new_config = SMALL_CLUSTER.copy() new_config["max_workers"] = 1 self.write_config(new_config) autoscaler.update() self.assertEqual(len(self.provider.nodes({})), 1) # Update the config to reduce the cluster size new_config["max_workers"] = 10 self.write_config(new_config) autoscaler.update() self.assertEqual(len(self.provider.nodes({})), 6) autoscaler.update() self.assertEqual(len(self.provider.nodes({})), 10)
def testEmptyDocker(self): config = MULTI_WORKER_CLUSTER.copy() del config["docker"] config["min_workers"] = 0 config["max_workers"] = 10 config_path = self.write_config(config) self.provider = MockProvider() runner = MockProcessRunner() autoscaler = StandardAutoscaler(config_path, LoadMetrics(), max_failures=0, process_runner=runner, update_interval_s=0) assert len(self.provider.non_terminated_nodes({})) == 0 autoscaler.update() self.waitForNodes(0) autoscaler.request_resources([{"CPU": 1}]) autoscaler.update() self.waitForNodes(1) assert self.provider.mock_nodes[0].node_type == "m4.large" autoscaler.request_resources([{"GPU": 8}]) autoscaler.update() self.waitForNodes(2) assert self.provider.mock_nodes[1].node_type == "p2.8xlarge"
def testInvalidConfig(self): invalid_config = "/dev/null" self.assertRaises(ValueError, lambda: StandardAutoscaler(invalid_config))
def __init__(self, redis_address, redis_port, autoscaling_config, redis_password=None): # Initialize the Redis clients. self.state = ray.experimental.state.GlobalState() self.state._initialize_global_state(redis_address, redis_port, redis_password=redis_password) self.use_raylet = self.state.use_raylet self.redis = redis.StrictRedis(host=redis_address, port=redis_port, db=0, password=redis_password) # Setup subscriptions to the primary Redis server and the Redis shards. self.primary_subscribe_client = self.redis.pubsub( ignore_subscribe_messages=True) if self.use_raylet: self.shard_subscribe_clients = [] for redis_client in self.state.redis_clients: subscribe_client = redis_client.pubsub( ignore_subscribe_messages=True) self.shard_subscribe_clients.append(subscribe_client) else: # We don't need to subscribe to the shards in legacy Ray. self.shard_subscribe_clients = [] # Initialize data structures to keep track of the active database # clients. self.dead_local_schedulers = set() self.live_plasma_managers = Counter() self.dead_plasma_managers = set() # Keep a mapping from local scheduler client ID to IP address to use # for updating the load metrics. self.local_scheduler_id_to_ip_map = {} self.load_metrics = LoadMetrics() if autoscaling_config: self.autoscaler = StandardAutoscaler(autoscaling_config, self.load_metrics) else: self.autoscaler = None # Experimental feature: GCS flushing. self.issue_gcs_flushes = "RAY_USE_NEW_GCS" in os.environ self.gcs_flush_policy = None if self.issue_gcs_flushes: # Data is stored under the first data shard, so we issue flushes to # that redis server. addr_port = self.redis.lrange("RedisShards", 0, -1) if len(addr_port) > 1: logger.warning("TODO: if launching > 1 redis shard, flushing " "needs to touch shards in parallel.") self.issue_gcs_flushes = False else: addr_port = addr_port[0].split(b":") self.redis_shard = redis.StrictRedis(host=addr_port[0], port=addr_port[1], password=redis_password) try: self.redis_shard.execute_command("HEAD.FLUSH 0") except redis.exceptions.ResponseError as e: logger.info( "Turning off flushing due to exception: {}".format( str(e))) self.issue_gcs_flushes = False
def testAbortOnCreationFailures(self): config_path = self.write_config(SMALL_CLUSTER) self.provider = MockProvider() self.provider.fail_creates = True autoscaler = StandardAutoscaler(config_path, max_failures=0) self.assertRaises(AssertionError, autoscaler.update)
def testInvalidConfig(self): invalid_config = os.devnull with pytest.raises(ValueError): StandardAutoscaler(invalid_config, LoadMetrics(), update_interval_s=0)
def testFileMountsNonContinuous(self): file_mount_dir = tempfile.mkdtemp() self.provider = MockProvider() config = SMALL_CLUSTER.copy() config["file_mounts"] = {"/home/test-folder": file_mount_dir} config["min_workers"] = 2 config["max_workers"] = 2 config_path = self.write_config(config) runner = MockProcessRunner() lm = LoadMetrics() autoscaler = StandardAutoscaler(config_path, lm, max_failures=0, process_runner=runner, update_interval_s=0) autoscaler.update() self.waitForNodes(2) self.provider.finish_starting_nodes() autoscaler.update() self.waitForNodes(2, tag_filters={TAG_RAY_NODE_STATUS: STATUS_UP_TO_DATE}) autoscaler.update() for i in [0, 1]: runner.assert_has_call("172.0.0.{}".format(i), "setup_cmd") runner.assert_has_call( "172.0.0.{}".format(i), "{}/ [email protected].{}:/home/test-folder/".format( file_mount_dir, i)) runner.clear_history() with open(os.path.join(file_mount_dir, "test.txt"), "wb") as temp_file: temp_file.write("hello".encode()) autoscaler.update() self.waitForNodes(2) self.provider.finish_starting_nodes() self.waitForNodes(2, tag_filters={TAG_RAY_NODE_STATUS: STATUS_UP_TO_DATE}) for i in [0, 1]: runner.assert_not_has_call("172.0.0.{}".format(i), "setup_cmd") runner.assert_not_has_call( "172.0.0.{}".format(i), "{}/ [email protected].{}:/home/test-folder/".format( file_mount_dir, i)) # Simulate a second `ray up` call from ray.autoscaler import util util._hash_cache = {} runner = MockProcessRunner() lm = LoadMetrics() autoscaler = StandardAutoscaler(config_path, lm, max_failures=0, process_runner=runner, update_interval_s=0) autoscaler.update() self.waitForNodes(2) self.provider.finish_starting_nodes() self.waitForNodes(2, tag_filters={TAG_RAY_NODE_STATUS: STATUS_UP_TO_DATE}) autoscaler.update() for i in [0, 1]: runner.assert_has_call("172.0.0.{}".format(i), "setup_cmd") runner.assert_has_call( "172.0.0.{}".format(i), "{}/ [email protected].{}:/home/test-folder/".format( file_mount_dir, i))
def testInvalidConfig(self): invalid_config = "/dev/null" self.assertRaises( ValueError, lambda: StandardAutoscaler( invalid_config, LoadMetrics(), update_interval_s=0))
def testDockerWorkers(self): config = MULTI_WORKER_CLUSTER.copy() config["available_node_types"]["p2.8xlarge"]["docker"] = { "worker_image": "p2.8x_image:latest", "worker_run_options": ["p2.8x-run-options"] } config["available_node_types"]["p2.xlarge"]["docker"] = { "worker_image": "p2x_image:nightly" } config["docker"]["worker_run_options"] = ["standard-run-options"] config["docker"]["image"] = "default-image:nightly" config["docker"]["worker_image"] = "default-image:nightly" # Commenting out this line causes the test case to fail?!?! config["min_workers"] = 0 config["max_workers"] = 10 config_path = self.write_config(config) self.provider = MockProvider() runner = MockProcessRunner() autoscaler = StandardAutoscaler(config_path, LoadMetrics(), max_failures=0, process_runner=runner, update_interval_s=0) assert len(self.provider.non_terminated_nodes({})) == 0 autoscaler.update() self.waitForNodes(0) autoscaler.request_resources([{"CPU": 1}]) autoscaler.update() self.waitForNodes(1) assert self.provider.mock_nodes[0].node_type == "m4.large" autoscaler.request_resources([{"GPU": 8}]) autoscaler.update() self.waitForNodes(2) assert self.provider.mock_nodes[1].node_type == "p2.8xlarge" autoscaler.request_resources([{"GPU": 1}] * 9) autoscaler.update() self.waitForNodes(3) assert self.provider.mock_nodes[2].node_type == "p2.xlarge" autoscaler.update() # Fill up m4, p2.8, p2 and request 2 more CPUs autoscaler.request_resources([{ "CPU": 2 }, { "CPU": 16 }, { "CPU": 32 }, { "CPU": 2 }]) autoscaler.update() self.waitForNodes(4) assert self.provider.mock_nodes[3].node_type == "m4.16xlarge" autoscaler.update() sleep(0.1) runner.assert_has_call(self.provider.mock_nodes[1].internal_ip, "p2.8x-run-options") runner.assert_has_call(self.provider.mock_nodes[1].internal_ip, "p2.8x_image:latest") runner.assert_not_has_call(self.provider.mock_nodes[1].internal_ip, "default-image:nightly") runner.assert_not_has_call(self.provider.mock_nodes[1].internal_ip, "standard-run-options") runner.assert_has_call(self.provider.mock_nodes[2].internal_ip, "p2x_image:nightly") runner.assert_has_call(self.provider.mock_nodes[2].internal_ip, "standard-run-options") runner.assert_not_has_call(self.provider.mock_nodes[2].internal_ip, "p2.8x-run-options") runner.assert_has_call(self.provider.mock_nodes[3].internal_ip, "default-image:nightly") runner.assert_has_call(self.provider.mock_nodes[3].internal_ip, "standard-run-options") runner.assert_not_has_call(self.provider.mock_nodes[3].internal_ip, "p2.8x-run-options") runner.assert_not_has_call(self.provider.mock_nodes[3].internal_ip, "p2x_image:nightly")