def _sync_web_session_test(web_address): register_ray_serializers() new_session(web_address, backend='oscar', default=True) raw = np.random.RandomState(0).rand(10, 5) a = mt.tensor(raw, chunk_size=5).sum(axis=1) b = a.execute(show_progress=False) assert b is a return True
def test_sync_execute(): session = new_session(n_cpu=2, default=True, web=False) # web not started assert session._session.client.web_address is None with session: raw = np.random.RandomState(0).rand(10, 5) a = mt.tensor(raw, chunk_size=5).sum(axis=1) b = a.execute(show_progress=False) assert b is a result = a.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1)) c = b + 1 c.execute(show_progress=False) result = c.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1) + 1) c = mt.tensor(raw, chunk_size=5).sum() d = session.execute(c) assert d is c assert abs(session.fetch(d) - raw.sum()) < 0.001 session.stop_server() assert get_default_session() is None
def test_decref(): session = new_session(n_cpu=2, default=True) with session: a = mt.ones((10, 10)) b = mt.ones((10, 10)) c = b + 1 d = mt.ones((5, 5)) a.execute(show_progress=False) b.execute(show_progress=False) c.execute(show_progress=False) d.execute(show_progress=False) del a ref_counts = session._get_ref_counts() assert len(ref_counts) == 3 del b ref_counts = session._get_ref_counts() assert len(ref_counts) == 3 del c ref_counts = session._get_ref_counts() assert len(ref_counts) == 1 session.stop_server()
def test_sync_execute(): start_method = os.environ.get('POOL_START_METHOD', 'forkserver') \ if sys.platform != 'win32' else None session = new_session(n_cpu=2, subprocess_start_method=start_method, default=True) with session: raw = np.random.RandomState(0).rand(10, 5) a = mt.tensor(raw, chunk_size=5).sum(axis=1) b = a.execute(show_progress=False) assert b is a result = a.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1)) c = b + 1 c.execute(show_progress=False) result = c.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1) + 1) c = mt.tensor(raw, chunk_size=5).sum() d = session.execute(c) assert d is c assert abs(session.fetch(d) - raw.sum()) < 0.001 session.stop_server() assert get_default_session() is None
def setup_session(): session = new_session(n_cpu=2, default=True) with session: with option_context({'show_progress': False}): yield session session.stop_server()
def test_cmdline_run(supervisor_args, worker_args, use_web_addr): sv_proc = w_proc = None try: sv_args = _reload_args(supervisor_args) sv_proc = subprocess.Popen(sv_args, env=os.environ.copy()) oscar_port = _get_labelled_port('supervisor', create=False) if not oscar_port: oscar_ep = _wait_supervisor_ready(sv_proc) else: oscar_ep = f'127.0.0.1:{oscar_port}' if use_web_addr: host = oscar_ep.rsplit(':', 1)[0] api_ep = f'http://{host}:{_get_labelled_port("web", create=False)}' else: api_ep = oscar_ep w_proc = subprocess.Popen(_reload_args(worker_args), env=os.environ.copy()) _wait_worker_ready(oscar_ep, w_proc) new_session(api_ep, default=True) data = np.random.rand(10, 10) res = mt.tensor(data, chunk_size=5).sum().execute().fetch() np.testing.assert_almost_equal(res, data.sum()) finally: ep_file_name = OscarCommandRunner._build_endpoint_file_path( pid=sv_proc.pid) try: os.unlink(ep_file_name) except OSError: pass for proc in [w_proc, sv_proc]: if not proc: continue proc.terminate() try: proc.wait(3) except subprocess.TimeoutExpired: kill_process_tree(proc.pid)
def test_distributed_ts_fresh(setup): robot_execution_failures.download_robot_execution_failures() df, y = robot_execution_failures.load_robot_execution_failures() default_session = get_default_session() sync_session = new_session(default_session.address) dist = MarsDistributor(session=sync_session) df = df.iloc[:200].copy() extraction_settings = ComprehensiveFCParameters() extract_features(df, column_id='id', column_sort='time', default_fc_parameters=extraction_settings, # we impute = remove all NaN features automatically impute_function=impute, distributor=dist)
def test_sync_execute(ray_cluster, create_cluster): assert create_cluster.session session = new_session(address=create_cluster.address, backend='oscar', default=True) with session: raw = np.random.RandomState(0).rand(10, 5) a = mt.tensor(raw, chunk_size=5).sum(axis=1) b = a.execute(show_progress=False) assert b is a result = a.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1)) c = mt.tensor(raw, chunk_size=5).sum() d = session.execute(c) assert d is c assert abs(session.fetch(d) - raw.sum()) < 0.001 assert get_default_session() is None