def start_global_scheduler(redis_address, node_ip_address, stdout_file=None, stderr_file=None, cleanup=True): """Start a global scheduler process. Args: redis_address (str): The address of the Redis instance. node_ip_address: The IP address of the node that this scheduler will run on. stdout_file: A file handle opened for writing to redirect stdout to. If no redirection should happen, then this should be None. stderr_file: A file handle opened for writing to redirect stderr to. If no redirection should happen, then this should be None. cleanup (bool): True if using Ray in local mode. If cleanup is true, then this process will be killed by services.cleanup() when the Python process that imported services exits. """ p = global_scheduler.start_global_scheduler(redis_address, node_ip_address, stdout_file=stdout_file, stderr_file=stderr_file) if cleanup: all_processes[PROCESS_TYPE_GLOBAL_SCHEDULER].append(p) record_log_files_in_redis(redis_address, node_ip_address, [stdout_file, stderr_file])
def setUp(self): # Start one Redis server and N pairs of (plasma, local_scheduler) self.node_ip_address = "127.0.0.1" redis_address, redis_shards = services.start_redis( self.node_ip_address) redis_port = services.get_port(redis_address) time.sleep(0.1) # Create a client for the global state store. self.state = state.GlobalState() self.state._initialize_global_state(self.node_ip_address, redis_port) # Start one global scheduler. self.p1 = global_scheduler.start_global_scheduler( redis_address, self.node_ip_address, use_valgrind=USE_VALGRIND) self.plasma_store_pids = [] self.plasma_manager_pids = [] self.local_scheduler_pids = [] self.plasma_clients = [] self.local_scheduler_clients = [] for i in range(NUM_CLUSTER_NODES): # Start the Plasma store. Plasma store name is randomly generated. plasma_store_name, p2 = plasma.start_plasma_store() self.plasma_store_pids.append(p2) # Start the Plasma manager. # Assumption: Plasma manager name and port are randomly generated # by the plasma module. manager_info = plasma.start_plasma_manager(plasma_store_name, redis_address) plasma_manager_name, p3, plasma_manager_port = manager_info self.plasma_manager_pids.append(p3) plasma_address = "{}:{}".format(self.node_ip_address, plasma_manager_port) plasma_client = pa.plasma.connect(plasma_store_name, plasma_manager_name, 64) self.plasma_clients.append(plasma_client) # Start the local scheduler. local_scheduler_name, p4 = local_scheduler.start_local_scheduler( plasma_store_name, plasma_manager_name=plasma_manager_name, plasma_address=plasma_address, redis_address=redis_address, static_resources={"CPU": 10}) # Connect to the scheduler. local_scheduler_client = local_scheduler.LocalSchedulerClient( local_scheduler_name, NIL_WORKER_ID, False, random_task_id(), False) self.local_scheduler_clients.append(local_scheduler_client) self.local_scheduler_pids.append(p4)
def setUp(self): # Start one Redis server and N pairs of (plasma, local_scheduler) self.node_ip_address = "127.0.0.1" redis_address, redis_shards = services.start_redis( self.node_ip_address) redis_port = services.get_port(redis_address) time.sleep(0.1) # Create a client for the global state store. self.state = state.GlobalState() self.state._initialize_global_state(self.node_ip_address, redis_port) # Start one global scheduler. self.p1 = global_scheduler.start_global_scheduler( redis_address, self.node_ip_address, use_valgrind=USE_VALGRIND) self.plasma_store_pids = [] self.plasma_manager_pids = [] self.local_scheduler_pids = [] self.plasma_clients = [] self.local_scheduler_clients = [] for i in range(NUM_CLUSTER_NODES): # Start the Plasma store. Plasma store name is randomly generated. plasma_store_name, p2 = plasma.start_plasma_store() self.plasma_store_pids.append(p2) # Start the Plasma manager. # Assumption: Plasma manager name and port are randomly generated # by the plasma module. manager_info = plasma.start_plasma_manager(plasma_store_name, redis_address) plasma_manager_name, p3, plasma_manager_port = manager_info self.plasma_manager_pids.append(p3) plasma_address = "{}:{}".format(self.node_ip_address, plasma_manager_port) plasma_client = pa.plasma.connect(plasma_store_name, plasma_manager_name, 64) self.plasma_clients.append(plasma_client) # Start the local scheduler. local_scheduler_name, p4 = local_scheduler.start_local_scheduler( plasma_store_name, plasma_manager_name=plasma_manager_name, plasma_address=plasma_address, redis_address=redis_address, static_resources={"CPU": 10}) # Connect to the scheduler. local_scheduler_client = local_scheduler.LocalSchedulerClient( local_scheduler_name, NIL_WORKER_ID, NIL_ACTOR_ID, False, 0) self.local_scheduler_clients.append(local_scheduler_client) self.local_scheduler_pids.append(p4)
def setUp(self): # Start one Redis server and N pairs of (plasma, local_scheduler) node_ip_address = "127.0.0.1" redis_port, self.redis_process = services.start_redis(cleanup=False) redis_address = services.address(node_ip_address, redis_port) # Create a Redis client. self.redis_client = redis.StrictRedis(host=node_ip_address, port=redis_port) # Start one global scheduler. self.p1 = global_scheduler.start_global_scheduler( redis_address, use_valgrind=USE_VALGRIND) self.plasma_store_pids = [] self.plasma_manager_pids = [] self.local_scheduler_pids = [] self.plasma_clients = [] self.local_scheduler_clients = [] for i in range(NUM_CLUSTER_NODES): # Start the Plasma store. Plasma store name is randomly generated. plasma_store_name, p2 = plasma.start_plasma_store() self.plasma_store_pids.append(p2) # Start the Plasma manager. # Assumption: Plasma manager name and port are randomly generated by the # plasma module. manager_info = plasma.start_plasma_manager(plasma_store_name, redis_address) plasma_manager_name, p3, plasma_manager_port = manager_info self.plasma_manager_pids.append(p3) plasma_address = "{}:{}".format(node_ip_address, plasma_manager_port) plasma_client = plasma.PlasmaClient(plasma_store_name, plasma_manager_name) self.plasma_clients.append(plasma_client) # Start the local scheduler. local_scheduler_name, p4 = local_scheduler.start_local_scheduler( plasma_store_name, plasma_manager_name=plasma_manager_name, plasma_address=plasma_address, redis_address=redis_address, static_resource_list=[10, 0]) # Connect to the scheduler. local_scheduler_client = local_scheduler.LocalSchedulerClient( local_scheduler_name, NIL_WORKER_ID, NIL_ACTOR_ID, False) self.local_scheduler_clients.append(local_scheduler_client) self.local_scheduler_pids.append(p4)