def random_driver_id(): return local_scheduler.ObjectID(np.random.bytes(ID_SIZE))
def random_function_id(): return local_scheduler.ObjectID(np.random.bytes(ID_SIZE))
def integration_many_tasks_helper(self, timesync=True): # There should be three db clients, the global scheduler, the local # scheduler, and the plasma manager. self.assertEqual(len(self.state.client_table()[self.node_ip_address]), 2 * NUM_CLUSTER_NODES + 1) num_return_vals = [0, 1, 2, 3, 5, 10] # Submit a bunch of tasks to Redis. num_tasks = 1000 for _ in range(num_tasks): # Create a new object for each task. data_size = np.random.randint(1 << 12) metadata_size = np.random.randint(1 << 9) plasma_client = self.plasma_clients[0] object_dep, memory_buffer, metadata = create_object(plasma_client, data_size, metadata_size, seal=True) if timesync: # Give 10ms for object info handler to fire (long enough to # yield CPU). time.sleep(0.010) task = local_scheduler.Task( random_driver_id(), random_function_id(), [local_scheduler.ObjectID(object_dep.binary())], num_return_vals[0], random_task_id(), 0) self.local_scheduler_clients[0].submit(task) # Check that there are the correct number of tasks in Redis and that # they all get assigned to the local scheduler. num_retries = 20 num_tasks_done = 0 while num_retries > 0: task_entries = self.state.task_table() self.assertLessEqual(len(task_entries), num_tasks) # First, check if all tasks made it to Redis. if len(task_entries) == num_tasks: task_statuses = [ task_entry["State"] for task_entry in task_entries.values() ] self.assertTrue( all([ status in [ state.TASK_STATUS_WAITING, state.TASK_STATUS_SCHEDULED, state.TASK_STATUS_QUEUED ] for status in task_statuses ])) num_tasks_done = task_statuses.count(state.TASK_STATUS_QUEUED) num_tasks_scheduled = task_statuses.count( state.TASK_STATUS_SCHEDULED) num_tasks_waiting = task_statuses.count( state.TASK_STATUS_WAITING) print("tasks in Redis = {}, tasks waiting = {}, " "tasks scheduled = {}, " "tasks queued = {}, retries left = {}".format( len(task_entries), num_tasks_waiting, num_tasks_scheduled, num_tasks_done, num_retries)) if all([ status == state.TASK_STATUS_QUEUED for status in task_statuses ]): # We're done, so pass. break num_retries -= 1 time.sleep(0.1) # Tasks can either be queued or in the global scheduler due to # spillback. self.assertEqual(num_tasks_done + num_tasks_waiting, num_tasks)
def integration_many_tasks_helper(self, timesync=True): # There should be three db clients, the global scheduler, the local # scheduler, and the plasma manager. self.assertEqual( len(self.redis_client.keys("{}*".format(DB_CLIENT_PREFIX))), 2 * NUM_CLUSTER_NODES + 1) num_return_vals = [0, 1, 2, 3, 5, 10] # Submit a bunch of tasks to Redis. num_tasks = 1000 for _ in range(num_tasks): # Create a new object for each task. data_size = np.random.randint(1 << 20) metadata_size = np.random.randint(1 << 10) plasma_client = self.plasma_clients[0] object_dep, memory_buffer, metadata = create_object(plasma_client, data_size, metadata_size, seal=True) if timesync: # Give 10ms for object info handler to fire (long enough to yield CPU). time.sleep(0.010) task = local_scheduler.Task(random_driver_id(), random_function_id(), [local_scheduler.ObjectID(object_dep)], num_return_vals[0], random_task_id(), 0) self.local_scheduler_clients[0].submit(task) # Check that there are the correct number of tasks in Redis and that they # all get assigned to the local scheduler. num_retries = 10 num_tasks_done = 0 while num_retries > 0: task_entries = self.redis_client.keys("{}*".format(TASK_PREFIX)) self.assertLessEqual(len(task_entries), num_tasks) # First, check if all tasks made it to Redis. if len(task_entries) == num_tasks: task_contents = [ self.redis_client.hgetall(task_entries[i]) for i in range(len(task_entries)) ] task_statuses = [ int(contents[b"state"]) for contents in task_contents ] self.assertTrue( all([ status in [ TASK_STATUS_WAITING, TASK_STATUS_SCHEDULED, TASK_STATUS_QUEUED ] for status in task_statuses ])) num_tasks_done = task_statuses.count(TASK_STATUS_QUEUED) num_tasks_scheduled = task_statuses.count( TASK_STATUS_SCHEDULED) num_tasks_waiting = task_statuses.count(TASK_STATUS_WAITING) print( "tasks in Redis = {}, tasks waiting = {}, tasks scheduled = {}, " "tasks queued = {}, retries left = {}".format( len(task_entries), num_tasks_waiting, num_tasks_scheduled, num_tasks_done, num_retries)) if all( [status == TASK_STATUS_QUEUED for status in task_statuses]): # We're done, so pass. break num_retries -= 1 time.sleep(0.1) if num_tasks_done != num_tasks: # At least one of the tasks failed to schedule. self.tearDown() sys.exit(2)
def random_task_id(): return local_scheduler.ObjectID(np.random.bytes(ray_constants.ID_SIZE))