def test_sync_execute(): session = new_session(n_cpu=2, default=True, web=False) # web not started assert session._session.client.web_address is None with session: raw = np.random.RandomState(0).rand(10, 5) a = mt.tensor(raw, chunk_size=5).sum(axis=1) b = a.execute(show_progress=False) assert b is a result = a.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1)) c = b + 1 c.execute(show_progress=False) result = c.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1) + 1) c = mt.tensor(raw, chunk_size=5).sum() d = session.execute(c) assert d is c assert abs(session.fetch(d) - raw.sum()) < 0.001 session.stop_server() assert get_default_session() is None
def test_sync_execute(): start_method = os.environ.get('POOL_START_METHOD', 'forkserver') \ if sys.platform != 'win32' else None session = new_session(n_cpu=2, subprocess_start_method=start_method, default=True) with session: raw = np.random.RandomState(0).rand(10, 5) a = mt.tensor(raw, chunk_size=5).sum(axis=1) b = a.execute(show_progress=False) assert b is a result = a.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1)) c = b + 1 c.execute(show_progress=False) result = c.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1) + 1) c = mt.tensor(raw, chunk_size=5).sum() d = session.execute(c) assert d is c assert abs(session.fetch(d) - raw.sum()) < 0.001 session.stop_server() assert get_default_session() is None
async def test_execute_describe(create_cluster): s = np.random.RandomState(0) raw = pd.DataFrame(s.rand(100, 4), columns=list('abcd')) df = md.DataFrame(raw, chunk_size=30) session = get_default_session() r = df.describe() info = await session.execute(r) await info assert info.result() is None assert info.exception() is None assert info.progress() == 1 res = await session.fetch(r) pd.testing.assert_frame_equal(res[0], raw.describe())
def test_distributed_ts_fresh(setup): robot_execution_failures.download_robot_execution_failures() df, y = robot_execution_failures.load_robot_execution_failures() default_session = get_default_session() sync_session = new_session(default_session.address) dist = MarsDistributor(session=sync_session) df = df.iloc[:200].copy() extraction_settings = ComprehensiveFCParameters() extract_features(df, column_id='id', column_sort='time', default_fc_parameters=extraction_settings, # we impute = remove all NaN features automatically impute_function=impute, distributor=dist)
def test_sync_execute(ray_cluster, create_cluster): assert create_cluster.session session = new_session(address=create_cluster.address, backend='oscar', default=True) with session: raw = np.random.RandomState(0).rand(10, 5) a = mt.tensor(raw, chunk_size=5).sum(axis=1) b = a.execute(show_progress=False) assert b is a result = a.fetch() np.testing.assert_array_equal(result, raw.sum(axis=1)) c = mt.tensor(raw, chunk_size=5).sum() d = session.execute(c) assert d is c assert abs(session.fetch(d) - raw.sum()) < 0.001 assert get_default_session() is None
async def test_execute(create_cluster): session = get_default_session() assert session.address is not None raw = np.random.RandomState(0).rand(10, 10) a = mt.tensor(raw, chunk_size=5) b = a + 1 info = await session.execute(b) await info assert info.result() is None assert info.exception() is None assert info.progress() == 1 np.testing.assert_equal(raw + 1, (await session.fetch(b))[0]) with pytest.raises(ValueError): await session.fetch(b + 1) with pytest.raises(ValueError): await session.fetch(b[b < 0.6])
async def test_fault_inject_subtask_processor(fault_cluster): extra_config = {'fault_injection_manager_name': FaultInjectionManager.name} session = get_default_session() raw = np.random.RandomState(0).rand(10, 10) a = mt.tensor(raw, chunk_size=5) b = a + 1 # TODO(fyrestone): We can use b.execute() when the issue # https://github.com/mars-project/mars/issues/2165 is fixed with pytest.raises(RuntimeError, match='Fault Injection'): info = await session.execute(b, extra_config=extra_config) await info info = await session.execute(b, extra_config=extra_config) await info assert info.result() is None assert info.exception() is None r = await session.fetch(b) np.testing.assert_array_equal(r[0], raw + 1)
async def test_iterative_tiling(create_cluster): session = get_default_session() raw = np.random.RandomState(0).rand(30, 5) raw_df = pd.DataFrame(raw, index=np.arange(1, 31)) df = md.DataFrame(raw_df, chunk_size=10) df = df[df[0] < .7] df2 = df.shift(2) info = await session.execute(df2) await info assert info.result() is None result = (await session.fetch(df2))[0] expected = raw_df[raw_df[0] < .7].shift(2) pd.testing.assert_frame_equal(result, expected) # test meta assert df2.index_value.min_val >= 1 assert df2.index_value.max_val <= 30
def check_ref_counts(): yield sess = get_default_session() assert len(SyncSession(sess)._get_ref_counts()) == 0
def f(): assert get_default_session().session_id == session_id return mt.ones((2, 3)).sum().to_numpy()