def test_no_default_session():
raw = np.random.RandomState(0).rand(10, 10)
a = mt.tensor(raw, chunk_size=5)
b = a + 1
with pytest.warns(Warning):
execute(b, show_progress=False)
np.testing.assert_array_equal(fetch(b), raw + 1)
assert get_default_async_session() is not None
stop_server()
assert get_default_async_session() is None
async def test_rerun_subtask(fault_cluster, fault_config):
fault_type, fault_count, expect_raises = fault_config
name = await create_fault_injection_manager(
session_id=fault_cluster.session.session_id,
address=fault_cluster.session.address,
fault_count=fault_count,
fault_type=fault_type)
extra_config = {ExtraConfigKey.FAULT_INJECTION_MANAGER_NAME: name}
session = get_default_async_session()
raw = np.random.RandomState(0).rand(10, 10)
a = mt.tensor(raw, chunk_size=5)
b = a + 1
info = await session.execute(b, extra_config=extra_config)
await info
assert info.result() is None
assert info.exception() is None
r = await session.fetch(b)
np.testing.assert_array_equal(r, raw + 1)
fault_injection_manager = await session.get_remote_object(
fault_cluster.session.session_id, name)
await fault_injection_manager.set_fault_count(
{FaultPosition.ON_EXECUTE_OPERAND: 1})
# the extra config overwrites the default config.
extra_config['subtask_max_retries'] = 0
info = await session.execute(b, extra_config=extra_config)
with expect_raises:
await info
def test_sync_execute():
session = new_session(n_cpu=2, web=False, use_uvloop=False)
# web not started
assert session._session.client.web_address is None
assert session.get_web_endpoint() is None
with session:
raw = np.random.RandomState(0).rand(10, 5)
a = mt.tensor(raw, chunk_size=5).sum(axis=1)
b = a.execute(show_progress=False)
assert b is a
result = a.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1))
c = b + 1
c.execute(show_progress=False)
result = c.fetch()
np.testing.assert_array_equal(result, raw.sum(axis=1) + 1)
c = mt.tensor(raw, chunk_size=5).sum()
d = session.execute(c)
assert d is c
assert abs(session.fetch(d) - raw.sum()) < 0.001
with tempfile.TemporaryDirectory() as tempdir:
file_path = os.path.join(tempdir, 'test.csv')
pdf = pd.DataFrame(np.random.RandomState(0).rand(100, 10),
columns=[f'col{i}' for i in range(10)])
pdf.to_csv(file_path, index=False)
df = md.read_csv(file_path,
chunk_bytes=os.stat(file_path).st_size / 5)
result = df.sum(axis=1).execute().fetch()
expected = pd.read_csv(file_path).sum(axis=1)
pd.testing.assert_series_equal(result, expected)
df = md.read_csv(file_path,
chunk_bytes=os.stat(file_path).st_size / 5)
result = df.head(10).execute().fetch()
expected = pd.read_csv(file_path).head(10)
pd.testing.assert_frame_equal(result, expected)
for worker_pool in session._session.client._cluster._worker_pools:
_assert_storage_cleaned(session.session_id,
worker_pool.external_address,
StorageLevel.MEMORY)
session.stop_server()
assert get_default_async_session() is None
async def test_execute_describe(create_cluster):
s = np.random.RandomState(0)
raw = pd.DataFrame(s.rand(100, 4), columns=list('abcd'))
df = md.DataFrame(raw, chunk_size=30)
session = get_default_async_session()
r = df.describe()
info = await session.execute(r)
await info
assert info.result() is None
assert info.exception() is None
assert info.progress() == 1
res = await session.fetch(r)
pd.testing.assert_frame_equal(res, raw.describe())
async def test_iterative_tiling(create_cluster):
session = get_default_async_session()
raw = np.random.RandomState(0).rand(30, 5)
raw_df = pd.DataFrame(raw, index=np.arange(1, 31))
df = md.DataFrame(raw_df, chunk_size=10)
df = df[df[0] < .7]
df2 = df.shift(2)
info = await session.execute(df2)
await info
assert info.result() is None
result = await session.fetch(df2)
expected = raw_df[raw_df[0] < .7].shift(2)
pd.testing.assert_frame_equal(result, expected)
# test meta
assert df2.index_value.min_val >= 1
assert df2.index_value.max_val <= 30
async def test_execute(create_cluster):
session = get_default_async_session()
assert session.address is not None
assert session.session_id is not None
raw = np.random.RandomState(0).rand(10, 10)
a = mt.tensor(raw, chunk_size=5)
b = a + 1
info = await session.execute(b)
await info
assert info.result() is None
assert info.exception() is None
assert info.progress() == 1
np.testing.assert_equal(raw + 1, await session.fetch(b))
with pytest.raises(ValueError):
await session.fetch(b + 1)
with pytest.raises(ValueError):
await session.fetch(b[b < 0.6])
del a, b