def _sync_web_session_test(web_address):
register_ray_serializers()
new_session(web_address, backend='oscar', default=True)
raw = np.random.RandomState(0).rand(10, 5)
a = mt.tensor(raw, chunk_size=5).sum(axis=1)
b = a.execute(show_progress=False)
assert b is a
return True
def test_sync_execute():
session = new_session(n_cpu=2, default=True,
web=False)
# web not started
assert session._session.client.web_address is None
with session:
raw = np.random.RandomState(0).rand(10, 5)
a = mt.tensor(raw, chunk_size=5).sum(axis=1)
b = a.execute(show_progress=False)
assert b is a
result = a.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1))
c = b + 1
c.execute(show_progress=False)
result = c.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1) + 1)
c = mt.tensor(raw, chunk_size=5).sum()
d = session.execute(c)
assert d is c
assert abs(session.fetch(d) - raw.sum()) < 0.001
session.stop_server()
assert get_default_session() is None
def test_decref():
session = new_session(n_cpu=2, default=True)
with session:
a = mt.ones((10, 10))
b = mt.ones((10, 10))
c = b + 1
d = mt.ones((5, 5))
a.execute(show_progress=False)
b.execute(show_progress=False)
c.execute(show_progress=False)
d.execute(show_progress=False)
del a
ref_counts = session._get_ref_counts()
assert len(ref_counts) == 3
del b
ref_counts = session._get_ref_counts()
assert len(ref_counts) == 3
del c
ref_counts = session._get_ref_counts()
assert len(ref_counts) == 1
session.stop_server()
def test_sync_execute():
start_method = os.environ.get('POOL_START_METHOD', 'forkserver') \
if sys.platform != 'win32' else None
session = new_session(n_cpu=2,
subprocess_start_method=start_method,
default=True)
with session:
raw = np.random.RandomState(0).rand(10, 5)
a = mt.tensor(raw, chunk_size=5).sum(axis=1)
b = a.execute(show_progress=False)
assert b is a
result = a.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1))
c = b + 1
c.execute(show_progress=False)
result = c.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1) + 1)
c = mt.tensor(raw, chunk_size=5).sum()
d = session.execute(c)
assert d is c
assert abs(session.fetch(d) - raw.sum()) < 0.001
session.stop_server()
assert get_default_session() is None
def setup_session():
session = new_session(n_cpu=2, default=True)
with session:
with option_context({'show_progress': False}):
yield session
session.stop_server()
def test_cmdline_run(supervisor_args, worker_args, use_web_addr):
sv_proc = w_proc = None
try:
sv_args = _reload_args(supervisor_args)
sv_proc = subprocess.Popen(sv_args, env=os.environ.copy())
oscar_port = _get_labelled_port('supervisor', create=False)
if not oscar_port:
oscar_ep = _wait_supervisor_ready(sv_proc)
else:
oscar_ep = f'127.0.0.1:{oscar_port}'
if use_web_addr:
host = oscar_ep.rsplit(':', 1)[0]
api_ep = f'http://{host}:{_get_labelled_port("web", create=False)}'
else:
api_ep = oscar_ep
w_proc = subprocess.Popen(_reload_args(worker_args),
env=os.environ.copy())
_wait_worker_ready(oscar_ep, w_proc)
new_session(api_ep, default=True)
data = np.random.rand(10, 10)
res = mt.tensor(data, chunk_size=5).sum().execute().fetch()
np.testing.assert_almost_equal(res, data.sum())
finally:
ep_file_name = OscarCommandRunner._build_endpoint_file_path(
pid=sv_proc.pid)
try:
os.unlink(ep_file_name)
except OSError:
pass
for proc in [w_proc, sv_proc]:
if not proc:
continue
proc.terminate()
try:
proc.wait(3)
except subprocess.TimeoutExpired:
kill_process_tree(proc.pid)
def test_distributed_ts_fresh(setup):
robot_execution_failures.download_robot_execution_failures()
df, y = robot_execution_failures.load_robot_execution_failures()
default_session = get_default_session()
sync_session = new_session(default_session.address)
dist = MarsDistributor(session=sync_session)
df = df.iloc[:200].copy()
extraction_settings = ComprehensiveFCParameters()
extract_features(df, column_id='id', column_sort='time',
default_fc_parameters=extraction_settings,
# we impute = remove all NaN features automatically
impute_function=impute, distributor=dist)
def test_sync_execute(ray_cluster, create_cluster):
assert create_cluster.session
session = new_session(address=create_cluster.address,
backend='oscar',
default=True)
with session:
raw = np.random.RandomState(0).rand(10, 5)
a = mt.tensor(raw, chunk_size=5).sum(axis=1)
b = a.execute(show_progress=False)
assert b is a
result = a.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1))
c = mt.tensor(raw, chunk_size=5).sum()
d = session.execute(c)
assert d is c
assert abs(session.fetch(d) - raw.sum()) < 0.001
assert get_default_session() is None