def setUp(self): # Start Plasma store. plasma_store_name, self.p1 = plasma.start_plasma_store() self.plasma_client = plasma.PlasmaClient(plasma_store_name) # Start a local scheduler. scheduler_name, self.p2 = photon.start_local_scheduler( plasma_store_name, use_valgrind=USE_VALGRIND) # Connect to the scheduler. self.photon_client = photon.PhotonClient(scheduler_name)
def setUp(self): # Start one Redis server and N pairs of (plasma, photon) redis_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server") redis_module = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../../core/src/common/redis_module/libray_redis_module.so") assert os.path.isfile(redis_path) assert os.path.isfile(redis_module) node_ip_address = "127.0.0.1" redis_port = new_port() redis_address = "{}:{}".format(node_ip_address, redis_port) self.redis_process = subprocess.Popen([ redis_path, "--port", str(redis_port), "--loglevel", "warning", "--loadmodule", redis_module ]) time.sleep(0.1) # Create a Redis client. self.redis_client = redis.StrictRedis(host=node_ip_address, port=redis_port) # Start one global scheduler. self.p1 = global_scheduler.start_global_scheduler( redis_address, use_valgrind=USE_VALGRIND) self.plasma_store_pids = [] self.plasma_manager_pids = [] self.local_scheduler_pids = [] self.plasma_clients = [] self.photon_clients = [] for i in range(NUM_CLUSTER_NODES): # Start the Plasma store. Plasma store name is randomly generated. plasma_store_name, p2 = plasma.start_plasma_store() self.plasma_store_pids.append(p2) # Start the Plasma manager. # Assumption: Plasma manager name and port are randomly generated by the plasma module. plasma_manager_name, p3, plasma_manager_port = plasma.start_plasma_manager( plasma_store_name, redis_address) self.plasma_manager_pids.append(p3) plasma_address = "{}:{}".format(node_ip_address, plasma_manager_port) plasma_client = plasma.PlasmaClient(plasma_store_name, plasma_manager_name) self.plasma_clients.append(plasma_client) # Start the local scheduler. local_scheduler_name, p4 = photon.start_local_scheduler( plasma_store_name, plasma_manager_name=plasma_manager_name, plasma_address=plasma_address, redis_address=redis_address, static_resource_list=[10, 0]) # Connect to the scheduler. photon_client = photon.PhotonClient(local_scheduler_name, NIL_ACTOR_ID) self.photon_clients.append(photon_client) self.local_scheduler_pids.append(p4)
def start_local_scheduler(redis_address, node_ip_address, plasma_store_name, plasma_manager_name, worker_path, plasma_address=None, cleanup=True, redirect_output=False, num_cpus=None, num_gpus=None, num_workers=0): """Start a local scheduler process. Args: redis_address (str): The address of the Redis instance. node_ip_address (str): The IP address of the node that this local scheduler is running on. plasma_store_name (str): The name of the plasma store socket to connect to. plasma_manager_name (str): The name of the plasma manager socket to connect to. worker_path (str): The path of the script to use when the local scheduler starts up new workers. cleanup (bool): True if using Ray in local mode. If cleanup is true, then this process will be killed by serices.cleanup() when the Python process that imported services exits. redirect_output (bool): True if stdout and stderr should be redirected to /dev/null. num_cpus: The number of CPUs the local scheduler should be configured with. num_gpus: The number of GPUs the local scheduler should be configured with. num_workers (int): The number of workers that the local scheduler should start. Return: The name of the local scheduler socket. """ if num_cpus is None: # By default, use the number of hardware execution threads for the number of # cores. num_cpus = multiprocessing.cpu_count() if num_gpus is None: # By default, assume this node has no GPUs. num_gpus = 0 local_scheduler_name, p = photon.start_local_scheduler(plasma_store_name, plasma_manager_name, worker_path=worker_path, node_ip_address=node_ip_address, redis_address=redis_address, plasma_address=plasma_address, use_profiler=RUN_PHOTON_PROFILER, redirect_output=redirect_output, static_resource_list=[num_cpus, num_gpus], num_workers=num_workers) if cleanup: all_processes[PROCESS_TYPE_LOCAL_SCHEDULER].append(p) return local_scheduler_name
def setUp(self): # Start a Redis server. redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../common/thirdparty/redis/src/redis-server") node_ip_address = "127.0.0.1" redis_port = new_port() redis_address = "{}:{}".format(node_ip_address, redis_port) self.redis_process = subprocess.Popen([redis_path, "--port", str(redis_port), "--loglevel", "warning"]) time.sleep(0.1) # Create a Redis client. self.redis_client = redis.StrictRedis(host=node_ip_address, port=redis_port) # Start the global scheduler. self.p1 = global_scheduler.start_global_scheduler(redis_address, USE_VALGRIND) # Start the Plasma store. plasma_store_name, self.p2 = plasma.start_plasma_store() # Start the local scheduler. local_scheduler_name, self.p3 = photon.start_local_scheduler(plasma_store_name, redis_address=redis_address) # Connect to the scheduler. self.photon_client = photon.PhotonClient(local_scheduler_name)
def start_local_scheduler(redis_address, plasma_store_name, cleanup=True): """Start a local scheduler process. Args: redis_address (str): The address of the Redis instance. plasma_store_name (str): The name of the plasma store socket to connect to. cleanup (bool): True if using Ray in local mode. If cleanup is true, then this process will be killed by serices.cleanup() when the Python process that imported services exits. Return: The name of the local scheduler socket. """ local_scheduler_name, p = photon.start_local_scheduler( plasma_store_name, redis_address=redis_address, use_profiler=RUN_PHOTON_PROFILER) if cleanup: all_processes.append(p) return local_scheduler_name
def start_local_scheduler(redis_address, node_ip_address, plasma_store_name, plasma_manager_name, worker_path, plasma_address=None, cleanup=True, redirect_output=False): """Start a local scheduler process. Args: redis_address (str): The address of the Redis instance. node_ip_address (str): The IP address of the node that this local scheduler is running on. plasma_store_name (str): The name of the plasma store socket to connect to. plasma_manager_name (str): The name of the plasma manager socket to connect to. worker_path (str): The path of the script to use when the local scheduler starts up new workers. cleanup (bool): True if using Ray in local mode. If cleanup is true, then this process will be killed by serices.cleanup() when the Python process that imported services exits. redirect_output (bool): True if stdout and stderr should be redirected to /dev/null. Return: The name of the local scheduler socket. """ local_scheduler_name, p = photon.start_local_scheduler( plasma_store_name, plasma_manager_name, worker_path=worker_path, node_ip_address=node_ip_address, redis_address=redis_address, plasma_address=plasma_address, use_profiler=RUN_PHOTON_PROFILER, redirect_output=redirect_output) if cleanup: all_processes[PROCESS_TYPE_LOCAL_SCHEDULER].append(p) return local_scheduler_name