def test_sync_execute():
session = new_session(n_cpu=2, default=True,
web=False)
# web not started
assert session._session.client.web_address is None
with session:
raw = np.random.RandomState(0).rand(10, 5)
a = mt.tensor(raw, chunk_size=5).sum(axis=1)
b = a.execute(show_progress=False)
assert b is a
result = a.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1))
c = b + 1
c.execute(show_progress=False)
result = c.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1) + 1)
c = mt.tensor(raw, chunk_size=5).sum()
d = session.execute(c)
assert d is c
assert abs(session.fetch(d) - raw.sum()) < 0.001
session.stop_server()
assert get_default_session() is None
def test_sync_execute():
start_method = os.environ.get('POOL_START_METHOD', 'forkserver') \
if sys.platform != 'win32' else None
session = new_session(n_cpu=2,
subprocess_start_method=start_method,
default=True)
with session:
raw = np.random.RandomState(0).rand(10, 5)
a = mt.tensor(raw, chunk_size=5).sum(axis=1)
b = a.execute(show_progress=False)
assert b is a
result = a.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1))
c = b + 1
c.execute(show_progress=False)
result = c.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1) + 1)
c = mt.tensor(raw, chunk_size=5).sum()
d = session.execute(c)
assert d is c
assert abs(session.fetch(d) - raw.sum()) < 0.001
session.stop_server()
assert get_default_session() is None
async def test_execute_describe(create_cluster):
s = np.random.RandomState(0)
raw = pd.DataFrame(s.rand(100, 4), columns=list('abcd'))
df = md.DataFrame(raw, chunk_size=30)
session = get_default_session()
r = df.describe()
info = await session.execute(r)
await info
assert info.result() is None
assert info.exception() is None
assert info.progress() == 1
res = await session.fetch(r)
pd.testing.assert_frame_equal(res[0], raw.describe())
def test_distributed_ts_fresh(setup):
robot_execution_failures.download_robot_execution_failures()
df, y = robot_execution_failures.load_robot_execution_failures()
default_session = get_default_session()
sync_session = new_session(default_session.address)
dist = MarsDistributor(session=sync_session)
df = df.iloc[:200].copy()
extraction_settings = ComprehensiveFCParameters()
extract_features(df, column_id='id', column_sort='time',
default_fc_parameters=extraction_settings,
# we impute = remove all NaN features automatically
impute_function=impute, distributor=dist)
def test_sync_execute(ray_cluster, create_cluster):
assert create_cluster.session
session = new_session(address=create_cluster.address,
backend='oscar',
default=True)
with session:
raw = np.random.RandomState(0).rand(10, 5)
a = mt.tensor(raw, chunk_size=5).sum(axis=1)
b = a.execute(show_progress=False)
assert b is a
result = a.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1))
c = mt.tensor(raw, chunk_size=5).sum()
d = session.execute(c)
assert d is c
assert abs(session.fetch(d) - raw.sum()) < 0.001
assert get_default_session() is None
async def test_execute(create_cluster):
session = get_default_session()
assert session.address is not None
raw = np.random.RandomState(0).rand(10, 10)
a = mt.tensor(raw, chunk_size=5)
b = a + 1
info = await session.execute(b)
await info
assert info.result() is None
assert info.exception() is None
assert info.progress() == 1
np.testing.assert_equal(raw + 1, (await session.fetch(b))[0])
with pytest.raises(ValueError):
await session.fetch(b + 1)
with pytest.raises(ValueError):
await session.fetch(b[b < 0.6])
async def test_fault_inject_subtask_processor(fault_cluster):
extra_config = {'fault_injection_manager_name': FaultInjectionManager.name}
session = get_default_session()
raw = np.random.RandomState(0).rand(10, 10)
a = mt.tensor(raw, chunk_size=5)
b = a + 1
# TODO(fyrestone): We can use b.execute() when the issue
# https://github.com/mars-project/mars/issues/2165 is fixed
with pytest.raises(RuntimeError, match='Fault Injection'):
info = await session.execute(b, extra_config=extra_config)
await info
info = await session.execute(b, extra_config=extra_config)
await info
assert info.result() is None
assert info.exception() is None
r = await session.fetch(b)
np.testing.assert_array_equal(r[0], raw + 1)
async def test_iterative_tiling(create_cluster):
session = get_default_session()
raw = np.random.RandomState(0).rand(30, 5)
raw_df = pd.DataFrame(raw, index=np.arange(1, 31))
df = md.DataFrame(raw_df, chunk_size=10)
df = df[df[0] < .7]
df2 = df.shift(2)
info = await session.execute(df2)
await info
assert info.result() is None
result = (await session.fetch(df2))[0]
expected = raw_df[raw_df[0] < .7].shift(2)
pd.testing.assert_frame_equal(result, expected)
# test meta
assert df2.index_value.min_val >= 1
assert df2.index_value.max_val <= 30
def check_ref_counts():
yield
sess = get_default_session()
assert len(SyncSession(sess)._get_ref_counts()) == 0
def f():
assert get_default_session().session_id == session_id
return mt.ones((2, 3)).sum().to_numpy()