def automl_fixture():
from zoo import init_spark_on_local
from zoo.ray.util.raycontext import RayContext
sc = init_spark_on_local(cores=4, spark_log_level="INFO")
ray_ctx = RayContext(sc=sc, object_store_memory="1g")
ray_ctx.init()
yield
ray_ctx.stop()
sc.stop()
def test_local(self):
node_num = 4
sc = init_spark_on_local(cores=node_num)
ray_ctx = RayContext(sc=sc)
ray_ctx.init()
actors = [TestRay.remote() for i in range(0, node_num)]
print([ray.get(actor.hostname.remote()) for actor in actors])
ray_ctx.stop()
sc.stop()
for process_info in ray_ctx.ray_processesMonitor.process_infos:
for pid in process_info.pids:
assert not psutil.pid_exists(pid)
def test_local(self):
node_num = 4
sc = init_spark_on_local(cores=node_num)
ray_ctx = RayContext(sc=sc, object_store_memory="1g")
ray_ctx.init()
actors = [TestRay.remote() for i in range(0, node_num)]
print(ray.get([actor.hostname.remote() for actor in actors]))
ray_ctx.stop()
time.sleep(3)
# repeat
print("-------------------first repeat begin!------------------")
ray_ctx = RayContext(sc=sc, object_store_memory="1g")
ray_ctx.init()
actors = [TestRay.remote() for i in range(0, node_num)]
print(ray.get([actor.hostname.remote() for actor in actors]))
ray_ctx.stop()
sc.stop()
time.sleep(3)
for process_info in ray_ctx.ray_processesMonitor.process_infos:
for pid in process_info.pids:
assert not psutil.pid_exists(pid)
type=str,
help="turn on yarn mode by passing the hadoop path"
"configuration folder. Otherwise, turn on local mode.")
args, _ = parser.parse_known_args()
if args.hadoop_conf:
sc = init_spark_on_yarn(hadoop_conf=args.hadoop_conf,
conda_name="rayexample",
num_executor=args.num_replicas,
executor_cores=88,
executor_memory="10g",
driver_memory="3g",
driver_cores=4,
extra_executor_memory_for_ray="2g")
ray_ctx = RayContext(sc=sc, object_store_memory="5g")
ray_ctx.init()
else:
ray.init(redis_address=args.redis_address)
# sc = init_spark_on_local(cores=44)
# ray_ctx = RayContext(sc=sc, object_store_memory="5g")
#ray.init(redis_address=args.redis_address)
if args.tune:
tune_example(num_replicas=args.num_replicas, use_gpu=args.use_gpu)
else:
train_example(num_replicas=args.num_replicas,
batch_size=args.batch_size,
use_gpu=args.use_gpu)
num_executor=slave_num,
executor_cores=28,
executor_memory="10g",
driver_memory="2g",
driver_cores=4,
extra_executor_memory_for_ray="30g",
spark_conf={"hello": "world"})
ray_ctx = RayContext(sc=sc,
object_store_memory="25g",
extra_params={"temp-dir": "/tmp/hello/"},
env={
"http_proxy": "http://child-prc.intel.com:913",
"http_proxys": "http://child-prc.intel.com:913"
})
ray_ctx.init(object_store_memory="2g", num_cores=0, labels="", extra_params={})
@ray.remote
class TestRay():
def hostname(self):
import socket
return socket.gethostname()
def check_cv2(self):
# conda install -c conda-forge opencv==3.4.2
import cv2
return cv2.__version__
def ip(self):
import ray.services as rservices