def _start_restricted_worker(self, redis_address, redis_password, object_store_memory):
num_cores = 0
command = "ray start --redis-address {} " \
"--redis-password {} --num-cpus {} --object-store-memory {}".\
format(redis_address, redis_password, num_cores, object_store_memory)
print("Executing command: {}".format(command))
process_info = session_execute(command=command, fail_fast=True)
ProcessMonitor.register_shutdown_hook(pgid=process_info.pgid)
def _detect_python_location(self):
from zoo.ray.util.process import session_execute
process_info = session_execute("command -v python")
if 0 != process_info.errorcode:
raise Exception(
"Cannot detect current python location. Please set it manually by python_location"
)
return process_info.out
def _start_ray_node(self, command, tag, wait_before=5, wait_after=5):
modified_env = self._prepare_env(self.mkl_cores)
print("Starting {} by running: {}".format(tag, command))
print("Wait for {} sec before launching {}".format(wait_before, tag))
time.sleep(wait_before)
process_info = session_execute(command=command, env=modified_env, tag=tag)
JVMGuard.registerPids(process_info.pids)
process_info.node_ip = rservices.get_node_ip_address()
print("Wait for {} sec before return process info for {}".format(wait_after, tag))
time.sleep(wait_after)
return process_info
def _start_restricted_worker(self,
object_store_memory,
num_cores=0,
labels="",
extra_params={}):
command = RayServiceFuncGenerator._get_raylet_command(
redis_address=self.redis_address,
ray_exec="ray ",
password=self.redis_password,
ray_node_cpu_cores=num_cores,
labels=labels,
object_store_memory=object_store_memory,
extra_params=extra_params)
print("Executing command: {}".format(command))
process_info = session_execute(command=command, fail_fast=True)
ProcessMonitor.register_shutdown_hook(pgid=process_info.pgid)
def _stop(iter):
command = "{} stop".format(self.ray_exec)
print("Start to end the ray services: {}".format(command))
session_execute(command=command, fail_fast=True)
return iter