class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
def testFromPandasDataFrameExecution(self):
pdf = pd.DataFrame(np.random.rand(20, 30),
index=[np.arange(20),
np.arange(20, 0, -1)])
df = from_pandas_df(pdf, chunk_size=(13, 21))
result = self.executor.execute_dataframe(df, concat=True)[0]
pd.testing.assert_frame_equal(pdf, result)
def testFromPandasSeriesExecution(self):
ps = pd.Series(np.random.rand(20),
index=[np.arange(20),
np.arange(20, 0, -1)],
name='a')
series = from_pandas_series(ps, chunk_size=13)
result = self.executor.execute_dataframe(series, concat=True)[0]
pd.testing.assert_series_equal(ps, result)
def testInitializerExecution(self):
pdf = pd.DataFrame(np.random.rand(20, 30),
index=[np.arange(20),
np.arange(20, 0, -1)])
df = md.DataFrame(pdf, chunk_size=(15, 10))
result = self.executor.execute_dataframe(df, concat=True)[0]
pd.testing.assert_frame_equal(pdf, result)
ps = pd.Series(np.random.rand(20),
index=[np.arange(20),
np.arange(20, 0, -1)],
name='a')
series = md.Series(ps, chunk_size=7)
result = self.executor.execute_dataframe(series, concat=True)[0]
pd.testing.assert_series_equal(ps, result)
def testSeriesFromTensor(self):
data = np.random.rand(10)
series = md.Series(mt.tensor(data), name='a')
pd.testing.assert_series_equal(series.execute(),
pd.Series(data, name='a'))
series = md.Series(mt.tensor(data, chunk_size=3))
pd.testing.assert_series_equal(series.execute(), pd.Series(data))
series = md.Series(mt.ones((10, ), chunk_size=4))
pd.testing.assert_series_equal(series.execute(),
pd.Series(np.ones(10, )))
def testFromTensorExecution(self):
tensor = mt.random.rand(10, 10, chunk_size=5)
df = dataframe_from_tensor(tensor)
tensor_res = self.executor.execute_tensor(tensor, concat=True)[0]
pdf_expected = pd.DataFrame(tensor_res)
df_result = self.executor.execute_dataframe(df, concat=True)[0]
pd.testing.assert_index_equal(df_result.index, pd.RangeIndex(0, 10))
pd.testing.assert_index_equal(df_result.columns, pd.RangeIndex(0, 10))
pd.testing.assert_frame_equal(df_result, pdf_expected)
# test converted with specified index_value and columns
tensor2 = mt.random.rand(2, 2, chunk_size=1)
df2 = dataframe_from_tensor(tensor2,
index=pd.Index(['a', 'b']),
columns=pd.Index([3, 4]))
df_result = self.executor.execute_dataframe(df2, concat=True)[0]
pd.testing.assert_index_equal(df_result.index, pd.Index(['a', 'b']))
pd.testing.assert_index_equal(df_result.columns, pd.Index([3, 4]))
# test converted from 1-d tensor
tensor3 = mt.array([1, 2, 3])
df3 = dataframe_from_tensor(tensor3)
result3 = self.executor.execute_dataframe(df3, concat=True)[0]
pdf_expected = pd.DataFrame(np.array([1, 2, 3]))
pd.testing.assert_frame_equal(pdf_expected, result3)
# test converted from identical chunks
tensor4 = mt.ones((10, 10), chunk_size=3)
df4 = dataframe_from_tensor(tensor4)
result4 = self.executor.execute_dataframe(df4, concat=True)[0]
pdf_expected = pd.DataFrame(
self.executor.execute_tensor(tensor4, concat=True)[0])
pd.testing.assert_frame_equal(pdf_expected, result4)
# from tensor with given index
tensor5 = mt.ones((10, 10), chunk_size=3)
df5 = dataframe_from_tensor(tensor5, index=np.arange(0, 20, 2))
result5 = self.executor.execute_dataframe(df5, concat=True)[0]
pdf_expected = pd.DataFrame(self.executor.execute_tensor(
tensor5, concat=True)[0],
index=np.arange(0, 20, 2))
pd.testing.assert_frame_equal(pdf_expected, result5)
# from tensor with given columns
tensor6 = mt.ones((10, 10), chunk_size=3)
df6 = dataframe_from_tensor(tensor6, columns=list('abcdefghij'))
result6 = self.executor.execute_dataframe(df6, concat=True)[0]
pdf_expected = pd.DataFrame(self.executor.execute_tensor(
tensor6, concat=True)[0],
columns=list('abcdefghij'))
pd.testing.assert_frame_equal(pdf_expected, result6)
def testFromRecordsExecution(self):
dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')])
ndarr = np.ones((10, ), dtype=dtype)
pdf_expected = pd.DataFrame.from_records(ndarr,
index=pd.RangeIndex(10))
# from structured array of mars
tensor = mt.ones((10, ), dtype=dtype, chunk_size=3)
df1 = from_records(tensor)
df1_result = self.executor.execute_dataframe(df1, concat=True)[0]
pd.testing.assert_frame_equal(df1_result, pdf_expected)
# from structured array of numpy
df2 = from_records(ndarr)
df2_result = self.executor.execute_dataframe(df2, concat=True)[0]
pd.testing.assert_frame_equal(df2_result, pdf_expected)
def testReadCSVExecution(self):
tempdir = tempfile.mkdtemp()
file_path = os.path.join(tempdir, 'test.csv')
try:
df = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
columns=['a', 'b', 'c'])
df.to_csv(file_path)
pdf = pd.read_csv(file_path, index_col=0)
mdf = self.executor.execute_dataframe(md.read_csv(file_path,
index_col=0),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf)
mdf2 = self.executor.execute_dataframe(md.read_csv(file_path,
index_col=0,
chunk_bytes=10),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf2)
finally:
shutil.rmtree(tempdir)
# test sep
tempdir = tempfile.mkdtemp()
file_path = os.path.join(tempdir, 'test.csv')
try:
df = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
columns=['a', 'b', 'c'])
df.to_csv(file_path, sep=';')
pdf = pd.read_csv(file_path, sep=';', index_col=0)
mdf = self.executor.execute_dataframe(md.read_csv(file_path,
sep=';',
index_col=0),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf)
mdf2 = self.executor.execute_dataframe(md.read_csv(file_path,
sep=';',
index_col=0,
chunk_bytes=10),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf2)
finally:
shutil.rmtree(tempdir)
# test missing value
tempdir = tempfile.mkdtemp()
file_path = os.path.join(tempdir, 'test.csv')
try:
df = pd.DataFrame({
'c1': [np.nan, 'a', 'b', 'c'],
'c2': [1, 2, 3, np.nan],
'c3': [np.nan, np.nan, 3.4, 2.2]
})
df.to_csv(file_path)
pdf = pd.read_csv(file_path, index_col=0)
mdf = self.executor.execute_dataframe(md.read_csv(file_path,
index_col=0),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf)
mdf2 = self.executor.execute_dataframe(md.read_csv(file_path,
index_col=0,
chunk_bytes=12),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf2)
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
file_path = os.path.join(tempdir, 'test.csv')
try:
index = pd.date_range(start='1/1/2018', periods=100)
df = pd.DataFrame(
{
'col1': np.random.rand(100),
'col2': np.random.choice(['a', 'b', 'c'], (100, )),
'col3': np.arange(100)
},
index=index)
df.to_csv(file_path)
pdf = pd.read_csv(file_path, index_col=0)
mdf = self.executor.execute_dataframe(md.read_csv(file_path,
index_col=0),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf)
mdf2 = self.executor.execute_dataframe(md.read_csv(
file_path, index_col=0, chunk_bytes=100),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf2)
finally:
shutil.rmtree(tempdir)
# test compression
tempdir = tempfile.mkdtemp()
file_path = os.path.join(tempdir, 'test.gzip')
try:
index = pd.date_range(start='1/1/2018', periods=100)
df = pd.DataFrame(
{
'col1': np.random.rand(100),
'col2': np.random.choice(['a', 'b', 'c'], (100, )),
'col3': np.arange(100)
},
index=index)
df.to_csv(file_path, compression='gzip')
pdf = pd.read_csv(file_path, compression='gzip', index_col=0)
mdf = self.executor.execute_dataframe(md.read_csv(
file_path, compression='gzip', index_col=0),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf)
mdf2 = self.executor.execute_dataframe(md.read_csv(
file_path, compression='gzip', index_col=0, chunk_bytes='1k'),
concat=True)[0]
pd.testing.assert_frame_equal(pdf, mdf2)
finally:
shutil.rmtree(tempdir)
# test multiply files
tempdir = tempfile.mkdtemp()
try:
df = pd.DataFrame(np.random.rand(300, 3), columns=['a', 'b', 'c'])
file_paths = [
os.path.join(tempdir, 'test{}.csv'.format(i)) for i in range(3)
]
df[:100].to_csv(file_paths[0])
df[100:200].to_csv(file_paths[1])
df[200:].to_csv(file_paths[2])
mdf = self.executor.execute_dataframe(md.read_csv(file_paths,
index_col=0),
concat=True)[0]
pd.testing.assert_frame_equal(df, mdf)
mdf2 = self.executor.execute_dataframe(md.read_csv(file_paths,
index_col=0,
chunk_bytes=50),
concat=True)[0]
pd.testing.assert_frame_equal(df, mdf2)
finally:
shutil.rmtree(tempdir)
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testConcatenateExecution(self):
a_data = np.random.rand(10, 20, 30)
b_data = np.random.rand(10, 20, 40)
c_data = np.random.rand(10, 20, 50)
a = tensor(a_data, chunk_size=5)
b = tensor(b_data, chunk_size=6)
c = tensor(c_data, chunk_size=7)
d = concatenate([a, b, c], axis=-1)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.concatenate([a_data, b_data, c_data], axis=-1)
self.assertTrue(np.array_equal(res, expected))
a_data = sps.random(10, 30)
b_data = sps.rand(10, 40)
c_data = sps.rand(10, 50)
a = tensor(a_data, chunk_size=5)
b = tensor(b_data, chunk_size=6)
c = tensor(c_data, chunk_size=7)
d = concatenate([a, b, c], axis=-1)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.concatenate([a_data.A, b_data.A, c_data.A], axis=-1)
self.assertTrue(np.array_equal(res.toarray(), expected))
def testStackExecution(self):
raw = [np.random.randn(3, 4) for _ in range(10)]
arrs = [tensor(a, chunk_size=3) for a in raw]
arr2 = stack(arrs)
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.array_equal(res[0], np.stack(raw)))
arr3 = stack(arrs, axis=1)
res = self.executor.execute_tensor(arr3, concat=True)
self.assertTrue(np.array_equal(res[0], np.stack(raw, axis=1)))
arr4 = stack(arrs, axis=2)
res = self.executor.execute_tensor(arr4, concat=True)
self.assertTrue(np.array_equal(res[0], np.stack(raw, axis=2)))
def testHStackExecution(self):
a_data = np.random.rand(10)
b_data = np.random.rand(20)
a = tensor(a_data, chunk_size=4)
b = tensor(b_data, chunk_size=4)
c = hstack([a, b])
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.hstack([a_data, b_data])
self.assertTrue(np.array_equal(res, expected))
a_data = np.random.rand(10, 20)
b_data = np.random.rand(10, 5)
a = tensor(a_data, chunk_size=3)
b = tensor(b_data, chunk_size=4)
c = hstack([a, b])
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.hstack([a_data, b_data])
self.assertTrue(np.array_equal(res, expected))
def testVStackExecution(self):
a_data = np.random.rand(10)
b_data = np.random.rand(10)
a = tensor(a_data, chunk_size=4)
b = tensor(b_data, chunk_size=4)
c = vstack([a, b])
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.vstack([a_data, b_data])
self.assertTrue(np.array_equal(res, expected))
a_data = np.random.rand(10, 20)
b_data = np.random.rand(5, 20)
a = tensor(a_data, chunk_size=3)
b = tensor(b_data, chunk_size=4)
c = vstack([a, b])
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.vstack([a_data, b_data])
self.assertTrue(np.array_equal(res, expected))
def testDStackExecution(self):
a_data = np.random.rand(10)
b_data = np.random.rand(10)
a = tensor(a_data, chunk_size=4)
b = tensor(b_data, chunk_size=4)
c = dstack([a, b])
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.dstack([a_data, b_data])
self.assertTrue(np.array_equal(res, expected))
a_data = np.random.rand(10, 20)
b_data = np.random.rand(10, 20)
a = tensor(a_data, chunk_size=3)
b = tensor(b_data, chunk_size=4)
c = dstack([a, b])
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.dstack([a_data, b_data])
self.assertTrue(np.array_equal(res, expected))
def testColumnStackExecution(self):
a_data = np.array((1, 2, 3))
b_data = np.array((2, 3, 4))
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=2)
c = column_stack((a, b))
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.column_stack((a_data, b_data))
np.testing.assert_equal(res, expected)
a_data = np.random.rand(4, 2, 3)
b_data = np.random.rand(4, 2, 3)
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=2)
c = column_stack((a, b))
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.column_stack((a_data, b_data))
np.testing.assert_equal(res, expected)
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
def testFromPandasDataFrameExecution(self):
pdf = pd.DataFrame(np.random.rand(20, 30),
index=[np.arange(20),
np.arange(20, 0, -1)])
df = from_pandas_df(pdf, chunk_size=(13, 21))
result = self.executor.execute_dataframe(df, concat=True)[0]
pd.testing.assert_frame_equal(pdf, result)
def testFromPandasSeriesExecution(self):
ps = pd.Series(np.random.rand(20),
index=[np.arange(20),
np.arange(20, 0, -1)],
name='a')
series = from_pandas_series(ps, chunk_size=13)
result = self.executor.execute_dataframe(series, concat=True)[0]
pd.testing.assert_series_equal(ps, result)
def testInitializerExecution(self):
pdf = pd.DataFrame(np.random.rand(20, 30),
index=[np.arange(20),
np.arange(20, 0, -1)])
df = md.DataFrame(pdf, chunk_size=(15, 10))
result = self.executor.execute_dataframe(df, concat=True)[0]
pd.testing.assert_frame_equal(pdf, result)
ps = pd.Series(np.random.rand(20),
index=[np.arange(20),
np.arange(20, 0, -1)],
name='a')
series = md.Series(ps, chunk_size=7)
result = self.executor.execute_dataframe(series, concat=True)[0]
pd.testing.assert_series_equal(ps, result)
def testFromTensorExecution(self):
tensor = mt.random.rand(10, 10, chunk_size=5)
df = from_tensor(tensor)
tensor_res = self.executor.execute_tensor(tensor, concat=True)[0]
pdf_expected = pd.DataFrame(tensor_res)
df_result = self.executor.execute_dataframe(df, concat=True)[0]
pd.testing.assert_index_equal(df_result.index, pd.RangeIndex(0, 10))
pd.testing.assert_index_equal(df_result.columns, pd.RangeIndex(0, 10))
pd.testing.assert_frame_equal(df_result, pdf_expected)
# test converted with specified index_value and columns
tensor2 = mt.random.rand(2, 2, chunk_size=1)
df2 = from_tensor(tensor2,
index=pd.Index(['a', 'b']),
columns=pd.Index([3, 4]))
df_result = self.executor.execute_dataframe(df2, concat=True)[0]
pd.testing.assert_index_equal(df_result.index, pd.Index(['a', 'b']))
pd.testing.assert_index_equal(df_result.columns, pd.Index([3, 4]))
# test converted from 1-d tensor
tensor3 = mt.array([1, 2, 3])
df3 = from_tensor(tensor3)
result3 = self.executor.execute_dataframe(df3, concat=True)[0]
pdf_expected = pd.DataFrame(np.array([1, 2, 3]))
pd.testing.assert_frame_equal(pdf_expected, result3)
def testFromRecordsExecution(self):
dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')])
ndarr = np.ones((10, ), dtype=dtype)
pdf_expected = pd.DataFrame.from_records(ndarr,
index=pd.RangeIndex(10))
# from structured array of mars
tensor = mt.ones((10, ), dtype=dtype, chunk_size=3)
df1 = from_records(tensor)
df1_result = self.executor.execute_dataframe(df1, concat=True)[0]
pd.testing.assert_frame_equal(df1_result, pdf_expected)
# from structured array of numpy
df2 = from_records(ndarr)
df2_result = self.executor.execute_dataframe(df2, concat=True)[0]
pd.testing.assert_frame_equal(df2_result, pdf_expected)
def __init__(self):
self._executor = Executor(
sync_provider_type=Executor.SyncProviderType.GEVENT)
class TestBinary(TestBase):
def setUp(self):
self.executor = Executor()
@property
def rfunc_name(self):
return 'r' + self.func_name
def testWithoutShuffleExecution(self):
# all the axes are monotonic
# data1 with index split into [0...4], [5...9],
# columns [3...7], [8...12]
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=np.arange(3, 13))
df1 = from_pandas(data1, chunk_size=5)
# data2 with index split into [6...11], [2, 5],
# columns [4...9], [10, 13]
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=np.arange(4, 14))
df2 = from_pandas(data2, chunk_size=6)
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testWithOneShuffleExecution(self):
# only 1 axis is monotonic
# data1 with index split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=5)
# data2 with index split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=6)
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# only 1 axis is monotonic
# data1 with columns split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7],
columns=np.arange(10))
df1 = from_pandas(data1, chunk_size=5)
# data2 with columns split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2],
columns=np.arange(11, 1, -1))
df2 = from_pandas(data2, chunk_size=6)
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testWithAllShuffleExecution(self):
# no axis is monotonic
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[0, 10, 2, 3, 4, 5, 6, 7, 8, 9],
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[11, 1, 2, 5, 7, 6, 8, 9, 10, 3],
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=6)
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testBothWithOneChunk(self):
# only 1 axis is monotonic
# data1 with index split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=10)
# data2 with index split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=10)
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# only 1 axis is monotonic
# data1 with columns split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7],
columns=np.arange(10))
df1 = from_pandas(data1, chunk_size=10)
# data2 with columns split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2],
columns=np.arange(11, 1, -1))
df2 = from_pandas(data2, chunk_size=10)
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testWithoutShuffleAndWithOneChunk(self):
# only 1 axis is monotonic
# data1 with index split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=(5, 10))
# data2 with index split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=(6, 10))
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# only 1 axis is monotonic
# data1 with columns split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7],
columns=np.arange(10))
df1 = from_pandas(data1, chunk_size=(10, 5))
# data2 with columns split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2],
columns=np.arange(11, 1, -1))
df2 = from_pandas(data2, chunk_size=(10, 6))
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testWithShuffleAndWithOneChunk(self):
# only 1 axis is monotonic
# data1 with index split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=(10, 5))
# data2 with index split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=(10, 6))
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# only 1 axis is monotonic
# data1 with columns split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7],
columns=np.arange(10))
df1 = from_pandas(data1, chunk_size=(5, 10))
# data2 with columns split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2],
columns=np.arange(11, 1, -1))
df2 = from_pandas(data2, chunk_size=(6, 10))
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testSameIndex(self):
data = pd.DataFrame(np.random.rand(10, 10),
index=np.random.randint(0, 2, size=(10, )),
columns=['c' + str(i) for i in range(10)])
df = from_pandas(data, chunk_size=3)
df2 = self.func(df, df)
expected = self.func(data, data)
result = self.executor.execute_dataframe(df2, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
series = from_pandas_series(data.iloc[0], chunk_size=3)
df3 = self.func(df, series)
expected = self.func(data, data.iloc[0])
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
series = from_pandas_series(data.iloc[:, 0], chunk_size=3)
df4 = getattr(df, self.func_name)(series, axis=0)
expected = getattr(data, self.func_name)(data.iloc[:, 0], axis=0)
result = self.executor.execute_dataframe(df4, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testChained(self):
data1 = pd.DataFrame(np.random.rand(10, 10))
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(np.random.rand(10, 10))
df2 = from_pandas(data2, chunk_size=6)
df3 = self.func(df1, df2)
data4 = pd.DataFrame(np.random.rand(10, 10))
df4 = from_pandas(data4, chunk_size=6)
df5 = self.func(df3, df4)
result = self.executor.execute_dataframe(df5, concat=True)[0]
expected = self.func(self.func(data1, data2), data4)
pd.testing.assert_frame_equal(expected, result)
def testRfunc(self):
data1 = pd.DataFrame(np.random.rand(10, 10))
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(np.random.rand(10, 10))
df2 = from_pandas(data2, chunk_size=6)
df3 = getattr(df1, self.rfunc_name)(df2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
expected = self.func(data2, data1)
pd.testing.assert_frame_equal(expected, result)
data3 = pd.DataFrame(np.random.rand(10, 10))
df4 = from_pandas(data3, chunk_size=5)
df5 = getattr(df4, self.rfunc_name)(1)
result = self.executor.execute_dataframe(df5, concat=True)[0]
expected2 = self.func(1, data3)
pd.testing.assert_frame_equal(expected2, result)
def testWithMultiForms(self):
# test multiple forms
# such as self+other, self.add(other), add(self,other)
data1 = pd.DataFrame(np.random.rand(10, 10))
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(np.random.rand(10, 10))
df2 = from_pandas(data2, chunk_size=6)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(self.func(df1, df2),
concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
result = self.executor.execute_dataframe(self.func(df1, df2),
concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
result = self.executor.execute_dataframe(getattr(df1,
self.func_name)(df2),
concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
result = self.executor.execute_dataframe(getattr(df1,
self.rfunc_name)(df2),
concat=True)[0]
pd.testing.assert_frame_equal(self.func(data2, data1), result)
def testDataframeAndScalar(self):
# test dataframe and scalar
pdf = pd.DataFrame(np.random.rand(10, 10))
df = from_pandas(pdf, chunk_size=2)
expected = self.func(pdf, 1)
result = self.executor.execute_dataframe(self.func(df, 1),
concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
result2 = self.executor.execute_dataframe(self.func(df, 1),
concat=True)[0]
pd.testing.assert_frame_equal(expected, result2)
result3 = self.executor.execute_dataframe(getattr(df,
self.func_name)(1),
concat=True)[0]
pd.testing.assert_frame_equal(expected, result3)
# test scalar and dataframe
result4 = self.executor.execute_dataframe(self.func(df, 1),
concat=True)[0]
pd.testing.assert_frame_equal(expected, result4)
expected2 = self.func(1, pdf)
result5 = self.executor.execute_dataframe(self.func(1, df),
concat=True)[0]
pd.testing.assert_frame_equal(expected2, result5)
result6 = self.executor.execute_dataframe(getattr(df,
self.rfunc_name)(1),
concat=True)[0]
pd.testing.assert_frame_equal(expected2, result6)
def testWithShuffleOnStringIndex(self):
# no axis is monotonic, and the index values are strings.
data1 = pd.DataFrame(
np.random.rand(10, 10),
index=[str(x) for x in [0, 10, 2, 3, 4, 5, 6, 7, 8, 9]],
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(
np.random.rand(10, 10),
index=[str(x) for x in [11, 1, 2, 5, 7, 6, 8, 9, 10, 3]],
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=6)
df3 = self.func(df1, df2)
expected = self.func(data1, data2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testDataframeAndSeries(self):
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
s1 = from_pandas_series(data2[1], chunk_size=(6, ))
# operate on single-column dataframe and series
df1 = from_pandas(data1[[1]], chunk_size=(5, 5))
r1 = getattr(df1, self.func_name)(s1, axis='index')
expected = getattr(data1[[1]], self.func_name)(data2[1], axis='index')
result = self.executor.execute_dataframe(r1, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# operate on dataframe and series without shuffle
df2 = from_pandas(data1, chunk_size=(5, 5))
r2 = getattr(df2, self.func_name)(s1, axis='index')
expected = getattr(data1, self.func_name)(data2[1], axis='index')
result = self.executor.execute_dataframe(r2, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# operate on dataframe and series with shuffle
df3 = from_pandas(data1, chunk_size=(5, 5))
r3 = getattr(df3, self.func_name)(s1, axis='columns')
expected = getattr(data1, self.func_name)(data2[1], axis='columns')
result = self.executor.execute_dataframe(r3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# test both one chunk, axis=0
pdf = pd.DataFrame({
'ca': [1, 3, 2],
'cb': [360, 180, 2]
},
index=[1, 2, 3])
df = from_pandas(pdf)
series = pd.Series([0, 1, 2], index=[1, 2, 3])
mars_series = from_pandas_series(series)
result = self.executor.execute_dataframe(getattr(df, self.func_name)(
mars_series, axis=0),
concat=True)[0]
expected = getattr(pdf, self.func_name)(series, axis=0)
pd.testing.assert_frame_equal(expected, result)
# test different number of chunks, axis=0
pdf = pd.DataFrame({
'ca': [1, 3, 2],
'cb': [360, 180, 2]
},
index=[1, 2, 3])
df = from_pandas(pdf, chunk_size=1)
series = pd.Series([0, 1, 2], index=[1, 2, 3])
mars_series = from_pandas_series(series)
result = self.executor.execute_dataframe(getattr(df, self.func_name)(
mars_series, axis=0),
concat=True)[0]
expected = getattr(pdf, self.func_name)(series, axis=0)
pd.testing.assert_frame_equal(expected, result)
# test with row shuffle, axis=0
pdf = pd.DataFrame({
'ca': [1, 3, 2],
'cb': [360, 180, 2]
},
index=[2, 1, 3])
df = from_pandas(pdf, chunk_size=1)
series = pd.Series([0, 1, 2], index=[3, 1, 2])
mars_series = from_pandas_series(series)
result = self.executor.execute_dataframe(getattr(df, self.func_name)(
mars_series, axis=0),
concat=True)[0]
expected = getattr(pdf, self.func_name)(series,
axis=0).reindex([3, 1, 2])
# modify the order of rows
result = result.reindex(index=[3, 1, 2])
pd.testing.assert_frame_equal(expected, result)
# test both one chunk, axis=1
pdf = pd.DataFrame({
1: [1, 3, 2],
2: [360, 180, 2],
3: [1, 2, 3]
},
index=['ra', 'rb', 'rc'])
df = from_pandas(pdf)
series = pd.Series([0, 1, 2], index=[1, 2, 3])
mars_series = from_pandas_series(series)
result = self.executor.execute_dataframe(getattr(df, self.func_name)(
mars_series, axis=1),
concat=True)[0]
expected = getattr(pdf, self.func_name)(series, axis=1)
pd.testing.assert_frame_equal(expected, result)
# test different number of chunks, axis=1
pdf = pd.DataFrame({
1: [1, 3, 2],
2: [360, 180, 2],
3: [1, 2, 3]
},
index=['ra', 'rb', 'rc'])
df = from_pandas(pdf, chunk_size=1)
series = pd.Series([0, 1, 2], index=[1, 2, 3])
mars_series = from_pandas_series(series)
result = self.executor.execute_dataframe(getattr(df, self.func_name)(
mars_series, axis=1),
concat=True)[0]
expected = getattr(pdf, self.func_name)(series, axis=1)
pd.testing.assert_frame_equal(expected, result)
# test with row shuffle, axis=1
pdf = pd.DataFrame({
1: [1, 3, 2],
3: [1, 2, 3],
2: [360, 180, 2]
},
index=['ra', 'rb', 'rc'])
df = from_pandas(pdf, chunk_size=1)
series = pd.Series([0, 1, 2], index=[3, 1, 2])
mars_series = from_pandas_series(series)
result = self.executor.execute_dataframe(getattr(df, self.func_name)(
mars_series, axis=1),
concat=True)[0]
expected = getattr(pdf, self.func_name)(series, axis=1)
# modify the order of columns
result = result[[1, 2, 3]]
pd.testing.assert_frame_equal(expected, result)
def testSeries(self):
# only one chunk
s1 = pd.Series(np.arange(10) + 1)
s2 = pd.Series(np.arange(10) + 1)
r = self.func(from_pandas_series(s1, chunk_size=10),
from_pandas_series(s2, chunk_size=10))
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = self.func(s1, s2)
pd.testing.assert_series_equal(expected, result)
# same index
s1 = pd.Series(np.arange(10) + 1)
s2 = pd.Series(np.arange(10) + 1)
r = self.func(from_pandas_series(s1, chunk_size=4),
from_pandas_series(s2, chunk_size=6))
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = self.func(s1, s2)
pd.testing.assert_series_equal(expected, result)
# no shuffle
s1 = pd.Series(np.arange(10) + 1, index=range(10))
s2 = pd.Series(np.arange(10) + 1, index=range(10, 0, -1))
r = self.func(from_pandas_series(s1, chunk_size=4),
from_pandas_series(s2, chunk_size=6))
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = self.func(s1, s2)
pd.testing.assert_series_equal(expected, result)
# shuffle
s1 = pd.Series(np.arange(10) + 1,
index=np.random.permutation(range(10)))
s2 = pd.Series(np.arange(10) + 1,
index=np.random.permutation(range(10, 0, -1)))
r = self.func(from_pandas_series(s1, chunk_size=4),
from_pandas_series(s2, chunk_size=6))
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = self.func(s1, s2)
pd.testing.assert_series_equal(expected, result)
# operate with scalar
s1 = pd.Series(np.arange(10) + 1,
index=np.random.permutation(range(10)))
r = self.func(from_pandas_series(s1, chunk_size=4), 4)
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = self.func(s1, 4)
pd.testing.assert_series_equal(expected, result)
# reverse with scalar
s1 = pd.Series(np.arange(10) + 1,
index=np.random.permutation(range(10)))
r = self.func(4, from_pandas_series(s1, chunk_size=4))
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = self.func(4, s1)
pd.testing.assert_series_equal(expected, result)
def testWithPlainValue(self):
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=6)
s1 = df1[2]
r = getattr(df1, self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
axis=0)
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = getattr(data1,
self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
axis=0)
pd.testing.assert_frame_equal(expected, result)
r = getattr(df1, self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10),
axis=0)
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = getattr(data1,
self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10),
axis=0)
pd.testing.assert_frame_equal(expected, result)
r = getattr(s1, self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = getattr(data1[2],
self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
pd.testing.assert_series_equal(expected, result)
r = getattr(s1, self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10))
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = getattr(data1[2],
self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10))
pd.testing.assert_series_equal(expected, result)
# specify index, not the default range index
data1 = pd.DataFrame(np.random.rand(10, 7),
index=np.arange(5, 15),
columns=[4, 1, 3, 2, 5, 6, 7])
df1 = from_pandas(data1, chunk_size=6)
s1 = df1[2]
r = getattr(df1,
self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
axis=0)
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = getattr(data1, self.func_name)(np.array(
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
axis=0)
pd.testing.assert_frame_equal(expected, result)
r = getattr(df1, self.func_name)(from_array(
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])),
axis=0)
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = getattr(data1, self.func_name)(np.array(
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
axis=0)
pd.testing.assert_frame_equal(expected, result)
r = getattr(s1,
self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = getattr(data1[2], self.func_name)(np.array(
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
pd.testing.assert_series_equal(expected, result)
r = getattr(s1, self.func_name)(from_array(
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])))
result = self.executor.execute_dataframe(r, concat=True)[0]
expected = getattr(data1[2], self.func_name)(np.array(
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))
pd.testing.assert_series_equal(expected, result)
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testRechunkExecution(self):
raw = np.random.random((11, 8))
arr = tensor(raw, chunk_size=3)
arr2 = arr.rechunk(4)
res = self.executor.execute_tensor(arr2)
self.assertTrue(np.array_equal(res[0], raw[:4, :4]))
self.assertTrue(np.array_equal(res[1], raw[:4, 4:]))
self.assertTrue(np.array_equal(res[2], raw[4:8, :4]))
self.assertTrue(np.array_equal(res[3], raw[4:8, 4:]))
self.assertTrue(np.array_equal(res[4], raw[8:, :4]))
self.assertTrue(np.array_equal(res[5], raw[8:, 4:]))
def testCopytoExecution(self):
a = ones((2, 3), chunk_size=1)
b = tensor([3, -1, 3], chunk_size=2)
copyto(a, b, where=b > 1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.array([[3, 1, 3], [3, 1, 3]])
np.testing.assert_equal(res, expected)
a = ones((2, 3), chunk_size=1)
b = tensor(np.asfortranarray(np.random.rand(2, 3)), chunk_size=2)
copyto(b, a)
res = self.executor.execute_tensor(b, concat=True)[0]
expected = np.asfortranarray(np.ones((2, 3)))
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['F_CONTIGUOUS'])
self.assertFalse(res.flags['C_CONTIGUOUS'])
def testAstypeExecution(self):
raw = np.random.random((10, 5))
arr = tensor(raw, chunk_size=3)
arr2 = arr.astype('i8')
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw.astype('i8'))
raw = sps.random(10, 5, density=.2)
arr = tensor(raw, chunk_size=3)
arr2 = arr.astype('i8')
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(
np.array_equal(res[0].toarray(),
raw.astype('i8').toarray()))
raw = np.asfortranarray(np.random.random((10, 5)))
arr = tensor(raw, chunk_size=3)
arr2 = arr.astype('i8', order='C')
res = self.executor.execute_tensor(arr2, concat=True)[0]
np.testing.assert_array_equal(res, raw.astype('i8'))
self.assertTrue(res.flags['C_CONTIGUOUS'])
self.assertFalse(res.flags['F_CONTIGUOUS'])
def testTransposeExecution(self):
raw = np.random.random((11, 8, 5))
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr)
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw.T)
arr3 = transpose(arr, axes=(-2, -1, -3))
res = self.executor.execute_tensor(arr3, concat=True)
np.testing.assert_array_equal(res[0], raw.transpose(1, 2, 0))
raw = sps.random(11, 8)
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr)
self.assertTrue(arr2.issparse())
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0].toarray(), raw.T.toarray())
# test order
raw = np.asfortranarray(np.random.random((11, 8, 5)))
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = np.transpose(raw).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr, (1, 2, 0))
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = np.transpose(raw, (1, 2, 0)).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testSwapaxesExecution(self):
raw = np.random.random((11, 8, 5))
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(2, 0)
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw.swapaxes(2, 0))
raw = sps.random(11, 8, density=.2)
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(1, 0)
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0].toarray(),
raw.toarray().swapaxes(1, 0))
# test order
raw = np.asfortranarray(np.random.rand(11, 8, 5))
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(2, 0)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.swapaxes(2, 0).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(0, 2)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.swapaxes(0, 2).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(1, 0)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.swapaxes(1, 0).copy(order='A')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testMoveaxisExecution(self):
x = zeros((3, 4, 5), chunk_size=2)
t = moveaxis(x, 0, -1)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (4, 5, 3))
t = moveaxis(x, -1, 0)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 3, 4))
t = moveaxis(x, [0, 1], [-1, -2])
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 4, 3))
t = moveaxis(x, [0, 1, 2], [-1, -2, -3])
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 4, 3))
def testBroadcastToExecution(self):
raw = np.random.random((10, 5, 1))
arr = tensor(raw, chunk_size=2)
arr2 = broadcast_to(arr, (5, 10, 5, 6))
res = self.executor.execute_tensor(arr2, concat=True)[0]
np.testing.assert_array_equal(res, np.broadcast_to(raw, (5, 10, 5, 6)))
# test chunk with unknown shape
arr1 = mt.random.rand(3, 4, chunk_size=2)
arr2 = mt.random.permutation(arr1)
arr3 = broadcast_to(arr2, (2, 3, 4))
res = self.executor.execute_tensor(arr3, concat=True)[0]
self.assertEqual(res.shape, (2, 3, 4))
def testBroadcastArraysExecutions(self):
x_data = [[1, 2, 3]]
x = tensor(x_data, chunk_size=1)
y_data = [[1], [2], [3]]
y = tensor(y_data, chunk_size=2)
a = broadcast_arrays(x, y)
res = [self.executor.execute_tensor(arr, concat=True)[0] for arr in a]
expected = np.broadcast_arrays(x_data, y_data)
for r, e in zip(res, expected):
np.testing.assert_equal(r, e)
def testWhereExecution(self):
raw_cond = np.random.randint(0, 2, size=(4, 4), dtype='?')
raw_x = np.random.rand(4, 1)
raw_y = np.random.rand(4, 4)
cond, x, y = tensor(raw_cond, chunk_size=2), tensor(
raw_x, chunk_size=2), tensor(raw_y, chunk_size=2)
arr = where(cond, x, y)
res = self.executor.execute_tensor(arr, concat=True)
self.assertTrue(
np.array_equal(res[0], np.where(raw_cond, raw_x, raw_y)))
raw_cond = sps.csr_matrix(
np.random.randint(0, 2, size=(4, 4), dtype='?'))
raw_x = sps.random(4, 1, density=.1)
raw_y = sps.random(4, 4, density=.1)
cond, x, y = tensor(raw_cond, chunk_size=2), tensor(
raw_x, chunk_size=2), tensor(raw_y, chunk_size=2)
arr = where(cond, x, y)
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertTrue(
np.array_equal(
res.toarray(),
np.where(raw_cond.toarray(), raw_x.toarray(),
raw_y.toarray())))
def testReshapeExecution(self):
raw_data = np.random.rand(10, 20, 30)
x = tensor(raw_data, chunk_size=6)
y = x.reshape(-1, 30)
res = self.executor.execute_tensor(y, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(-1, 30))
y2 = x.reshape(10, -1)
res = self.executor.execute_tensor(y2, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(10, -1))
y3 = x.reshape(-1)
res = self.executor.execute_tensor(y3, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(-1))
y4 = x.ravel()
res = self.executor.execute_tensor(y4, concat=True)
np.testing.assert_array_equal(res[0], raw_data.ravel())
raw_data = np.random.rand(30, 100, 20)
x = tensor(raw_data, chunk_size=6)
y = x.reshape(-1, 20, 5, 5, 4)
res = self.executor.execute_tensor(y, concat=True)
np.testing.assert_array_equal(res[0],
raw_data.reshape(-1, 20, 5, 5, 4))
y2 = x.reshape(3000, 10, 2)
res = self.executor.execute_tensor(y2, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(3000, 10, 2))
y3 = x.reshape(60, 25, 40)
res = self.executor.execute_tensor(y3, concat=True)
np.testing.assert_array_equal(res[0], raw_data.reshape(60, 25, 40))
y4 = x.reshape(60, 25, 40)
y4.op.extra_params['_reshape_with_shuffle'] = True
size_res = self.executor.execute_tensor(y4, mock=True)
res = self.executor.execute_tensor(y4, concat=True)
self.assertEqual(res[0].nbytes, sum(v[0] for v in size_res))
self.assertTrue(np.array_equal(res[0], raw_data.reshape(60, 25, 40)))
y5 = x.ravel(order='F')
res = self.executor.execute_tensor(y5, concat=True)[0]
expected = raw_data.ravel(order='F')
np.testing.assert_array_equal(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testExpandDimsExecution(self):
raw_data = np.random.rand(10, 20, 30)
x = tensor(raw_data, chunk_size=6)
y = expand_dims(x, 1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 1)))
y = expand_dims(x, 0)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 0)))
y = expand_dims(x, 3)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 3)))
y = expand_dims(x, -1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -1)))
y = expand_dims(x, -4)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -4)))
with self.assertRaises(np.AxisError):
expand_dims(x, -5)
with self.assertRaises(np.AxisError):
expand_dims(x, 4)
def testRollAxisExecution(self):
x = ones((3, 4, 5, 6), chunk_size=1)
y = rollaxis(x, 3, 1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(
np.array_equal(res[0], np.rollaxis(np.ones((3, 4, 5, 6)), 3, 1)))
def testAtleast1dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_1d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.array([1])))
self.assertTrue(np.array_equal(res[1], np.ones(3)))
self.assertTrue(np.array_equal(res[2], np.ones((3, 4))))
def testAtleast2dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_2d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.array([[1]])))
self.assertTrue(np.array_equal(res[1], np.atleast_2d(np.ones(3))))
self.assertTrue(np.array_equal(res[2], np.ones((3, 4))))
def testAtleast3dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_3d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.atleast_3d(x)))
self.assertTrue(np.array_equal(res[1], np.atleast_3d(np.ones(3))))
self.assertTrue(np.array_equal(res[2], np.atleast_3d(np.ones((3, 4)))))
def testArgwhereExecution(self):
x = arange(6, chunk_size=2).reshape(2, 3)
t = argwhere(x > 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.argwhere(np.arange(6).reshape(2, 3) > 1)
np.testing.assert_array_equal(res, expected)
data = np.asfortranarray(np.random.rand(10, 20))
x = tensor(data, chunk_size=10)
t = argwhere(x > 0.5)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.argwhere(data > 0.5)
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['F_CONTIGUOUS'])
self.assertFalse(res.flags['C_CONTIGUOUS'])
def testArraySplitExecution(self):
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = array_split(x, 3, axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.array_split(np.arange(48).reshape(2, 3, 8), 3, axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
ss = array_split(x, [3, 5, 6, 10], axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.array_split(np.arange(48).reshape(2, 3, 8),
[3, 5, 6, 10],
axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
def testSplitExecution(self):
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = split(x, 4, axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(np.arange(48).reshape(2, 3, 8), 4, axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
ss = split(x, [3, 5, 6, 10], axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10],
axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# hsplit
x = arange(120, chunk_size=3).reshape(2, 12, 5)
ss = hsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.hsplit(np.arange(120).reshape(2, 12, 5), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# vsplit
x = arange(48, chunk_size=3).reshape(8, 3, 2)
ss = vsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.vsplit(np.arange(48).reshape(8, 3, 2), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# dsplit
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = dsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.dsplit(np.arange(48).reshape(2, 3, 8), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
x_data = sps.random(12, 8, density=.1)
x = tensor(x_data, chunk_size=3)
ss = split(x, 4, axis=0)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(x_data.toarray(), 4, axis=0)
self.assertEqual(len(res), len(expected))
[
np.testing.assert_equal(r.toarray(), e)
for r, e in zip(res, expected)
]
def testRollExecution(self):
x = arange(10, chunk_size=2)
t = roll(x, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10), 2)
np.testing.assert_equal(res, expected)
x2 = x.reshape(2, 5)
t = roll(x2, 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1)
np.testing.assert_equal(res, expected)
t = roll(x2, 1, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=0)
np.testing.assert_equal(res, expected)
t = roll(x2, 1, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=1)
np.testing.assert_equal(res, expected)
def testSqueezeExecution(self):
data = np.array([[[0], [1], [2]]])
x = tensor(data, chunk_size=1)
t = squeeze(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.squeeze(data)
np.testing.assert_equal(res, expected)
t = squeeze(x, axis=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.squeeze(data, axis=2)
np.testing.assert_equal(res, expected)
def testPtpExecution(self):
x = arange(4, chunk_size=1).reshape(2, 2)
t = ptp(x, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ptp(np.arange(4).reshape(2, 2), axis=0)
np.testing.assert_equal(res, expected)
t = ptp(x, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ptp(np.arange(4).reshape(2, 2), axis=1)
np.testing.assert_equal(res, expected)
t = ptp(x)
res = self.executor.execute_tensor(t)[0]
expected = np.ptp(np.arange(4).reshape(2, 2))
np.testing.assert_equal(res, expected)
def testDiffExecution(self):
data = np.array([1, 2, 4, 7, 0])
x = tensor(data, chunk_size=2)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data)
np.testing.assert_equal(res, expected)
t = diff(x, n=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data, n=2)
np.testing.assert_equal(res, expected)
data = np.array([[1, 3, 6, 10], [0, 5, 6, 8]])
x = tensor(data, chunk_size=2)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data)
np.testing.assert_equal(res, expected)
t = diff(x, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data, axis=0)
np.testing.assert_equal(res, expected)
x = mt.arange('1066-10-13', '1066-10-16', dtype=mt.datetime64)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(
np.arange('1066-10-13', '1066-10-16', dtype=np.datetime64))
np.testing.assert_equal(res, expected)
def testEdiff1d(self):
data = np.array([1, 2, 4, 7, 0])
x = tensor(data, chunk_size=2)
t = ediff1d(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data)
np.testing.assert_equal(res, expected)
to_begin = tensor(-99, chunk_size=2)
to_end = tensor([88, 99], chunk_size=2)
t = ediff1d(x, to_begin=to_begin, to_end=to_end)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data, to_begin=-99, to_end=np.array([88, 99]))
np.testing.assert_equal(res, expected)
data = [[1, 2, 4], [1, 6, 24]]
t = ediff1d(tensor(data, chunk_size=2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data)
np.testing.assert_equal(res, expected)
def testDigitizeExecution(self):
data = np.array([0.2, 6.4, 3.0, 1.6])
x = tensor(data, chunk_size=2)
bins = np.array([0.0, 1.0, 2.5, 4.0, 10.0])
inds = digitize(x, bins)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data, bins)
np.testing.assert_equal(res, expected)
b = tensor(bins, chunk_size=2)
inds = digitize(x, b)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data, bins)
np.testing.assert_equal(res, expected)
data = np.array([1.2, 10.0, 12.4, 15.5, 20.])
x = tensor(data, chunk_size=2)
bins = np.array([0, 5, 10, 15, 20])
inds = digitize(x, bins, right=True)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data, bins, right=True)
np.testing.assert_equal(res, expected)
inds = digitize(x, bins, right=False)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data, bins, right=False)
np.testing.assert_equal(res, expected)
data = sps.random(10, 1, density=.1) * 12
x = tensor(data, chunk_size=2)
bins = np.array([1.0, 2.0, 2.5, 4.0, 10.0])
inds = digitize(x, bins)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data.toarray(), bins, right=False)
np.testing.assert_equal(res.toarray(), expected)
def testAverageExecution(self):
data = arange(1, 5, chunk_size=1)
t = average(data)
res = self.executor.execute_tensor(t)[0]
expected = np.average(np.arange(1, 5))
self.assertEqual(res, expected)
t = average(arange(1, 11, chunk_size=2),
weights=arange(10, 0, -1, chunk_size=2))
res = self.executor.execute_tensor(t)[0]
expected = np.average(range(1, 11), weights=range(10, 0, -1))
self.assertEqual(res, expected)
data = arange(6, chunk_size=2).reshape((3, 2))
t = average(data,
axis=1,
weights=tensor([1. / 4, 3. / 4], chunk_size=2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.average(np.arange(6).reshape(3, 2),
axis=1,
weights=(1. / 4, 3. / 4))
np.testing.assert_equal(res, expected)
with self.assertRaises(TypeError):
average(data, weights=tensor([1. / 4, 3. / 4], chunk_size=2))
def testCovExecution(self):
data = np.array([[0, 2], [1, 1], [2, 0]]).T
x = tensor(data, chunk_size=1)
t = cov(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.cov(data)
np.testing.assert_equal(res, expected)
data_x = [-2.1, -1, 4.3]
data_y = [3, 1.1, 0.12]
x = tensor(data_x, chunk_size=1)
y = tensor(data_y, chunk_size=1)
X = stack((x, y), axis=0)
t = cov(x, y)
r = tall(t == cov(X))
self.assertTrue(self.executor.execute_tensor(r)[0])
def testCorrcoefExecution(self):
data_x = [-2.1, -1, 4.3]
data_y = [3, 1.1, 0.12]
x = tensor(data_x, chunk_size=1)
y = tensor(data_y, chunk_size=1)
t = corrcoef(x, y)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.corrcoef(data_x, data_y)
np.testing.assert_equal(res, expected)
def testFlipExecution(self):
a = arange(8, chunk_size=2).reshape((2, 2, 2))
t = flip(a, 0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flip(np.arange(8).reshape(2, 2, 2), 0)
np.testing.assert_equal(res, expected)
t = flip(a, 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flip(np.arange(8).reshape(2, 2, 2), 1)
np.testing.assert_equal(res, expected)
t = flipud(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flipud(np.arange(8).reshape(2, 2, 2))
np.testing.assert_equal(res, expected)
t = fliplr(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.fliplr(np.arange(8).reshape(2, 2, 2))
np.testing.assert_equal(res, expected)
def testRepeatExecution(self):
a = repeat(3, 4)
res = self.executor.execute_tensor(a)[0]
expected = np.repeat(3, 4)
np.testing.assert_equal(res, expected)
x_data = np.random.randn(20, 30)
x = tensor(x_data, chunk_size=(3, 4))
t = repeat(x, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, 2)
np.testing.assert_equal(res, expected)
t = repeat(x, 3, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, 3, axis=1)
np.testing.assert_equal(res, expected)
t = repeat(x, np.arange(20), axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, np.arange(20), axis=0)
np.testing.assert_equal(res, expected)
t = repeat(x, arange(20, chunk_size=5), axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, np.arange(20), axis=0)
np.testing.assert_equal(res, expected)
x_data = sps.random(20, 30, density=.1)
x = tensor(x_data, chunk_size=(3, 4))
t = repeat(x, 2, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data.toarray(), 2, axis=1)
np.testing.assert_equal(res.toarray(), expected)
def testTileExecution(self):
a_data = np.array([0, 1, 2])
a = tensor(a_data, chunk_size=2)
t = tile(a, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, 2)
np.testing.assert_equal(res, expected)
t = tile(a, (2, 2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, (2, 2))
np.testing.assert_equal(res, expected)
t = tile(a, (2, 1, 2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, (2, 1, 2))
np.testing.assert_equal(res, expected)
b_data = np.array([[1, 2], [3, 4]])
b = tensor(b_data, chunk_size=1)
t = tile(b, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(b_data, 2)
np.testing.assert_equal(res, expected)
t = tile(b, (2, 1))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(b_data, (2, 1))
np.testing.assert_equal(res, expected)
c_data = np.array([1, 2, 3, 4])
c = tensor(c_data, chunk_size=3)
t = tile(c, (4, 1))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(c_data, (4, 1))
np.testing.assert_equal(res, expected)
def testIsInExecution(self):
element = 2 * arange(4, chunk_size=1).reshape((2, 2))
test_elements = [1, 2, 4, 8]
mask = isin(element, test_elements)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements)
np.testing.assert_equal(res, expected)
res = self.executor.execute_tensor(element[mask], concat=True)[0]
expected = np.array([2, 4])
np.testing.assert_equal(res, expected)
mask = isin(element, test_elements, invert=True)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)),
test_elements,
invert=True)
np.testing.assert_equal(res, expected)
res = self.executor.execute_tensor(element[mask], concat=True)[0]
expected = np.array([0, 6])
np.testing.assert_equal(res, expected)
test_set = {1, 2, 4, 8}
mask = isin(element, test_set)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_set)
np.testing.assert_equal(res, expected)
def testRavelExecution(self):
arr = ones((10, 5), chunk_size=2)
flat_arr = mt.ravel(arr)
res = self.executor.execute_tensor(flat_arr, concat=True)[0]
self.assertEqual(len(res), 50)
np.testing.assert_equal(res, np.ones(50))
def testSearchsortedExecution(self):
raw = np.sort(np.random.randint(100, size=(16, )))
# test different chunk_size, 3 will have combine, 6 will skip combine
for chunk_size in (3, 6):
arr = tensor(raw, chunk_size=chunk_size)
# test scalar, with value in the middle
t1 = searchsorted(arr, 20)
res = self.executor.execute_tensor(t1, concat=True)[0]
expected = np.searchsorted(raw, 20)
np.testing.assert_array_equal(res, expected)
# test scalar, with value larger than 100
t2 = searchsorted(arr, 200)
res = self.executor.execute_tensor(t2, concat=True)[0]
expected = np.searchsorted(raw, 200)
np.testing.assert_array_equal(res, expected)
# test scalar, side left, with value exact in the middle of the array
t3 = searchsorted(arr, raw[10], side='left')
res = self.executor.execute_tensor(t3, concat=True)[0]
expected = np.searchsorted(raw, raw[10], side='left')
np.testing.assert_array_equal(res, expected)
# test scalar, side right, with value exact in the middle of the array
t4 = searchsorted(arr, raw[10], side='right')
res = self.executor.execute_tensor(t4, concat=True)[0]
expected = np.searchsorted(raw, raw[10], side='right')
np.testing.assert_array_equal(res, expected)
# test scalar, side left, with value exact in the end of the array
t5 = searchsorted(arr, raw[15], side='left')
res = self.executor.execute_tensor(t5, concat=True)[0]
expected = np.searchsorted(raw, raw[15], side='left')
np.testing.assert_array_equal(res, expected)
# test scalar, side right, with value exact in the end of the array
t6 = searchsorted(arr, raw[15], side='right')
res = self.executor.execute_tensor(t6, concat=True)[0]
expected = np.searchsorted(raw, raw[15], side='right')
np.testing.assert_array_equal(res, expected)
# test scalar, side left, with value exact in the start of the array
t7 = searchsorted(arr, raw[0], side='left')
res = self.executor.execute_tensor(t7, concat=True)[0]
expected = np.searchsorted(raw, raw[0], side='left')
np.testing.assert_array_equal(res, expected)
# test scalar, side right, with value exact in the start of the array
t8 = searchsorted(arr, raw[0], side='right')
res = self.executor.execute_tensor(t8, concat=True)[0]
expected = np.searchsorted(raw, raw[0], side='right')
np.testing.assert_array_equal(res, expected)
raw2 = np.random.randint(100, size=(3, 4))
# test tensor, side left
t9 = searchsorted(arr, tensor(raw2, chunk_size=2), side='left')
res = self.executor.execute_tensor(t9, concat=True)[0]
expected = np.searchsorted(raw, raw2, side='left')
np.testing.assert_array_equal(res, expected)
# test tensor, side right
t10 = searchsorted(arr, tensor(raw2, chunk_size=2), side='right')
res = self.executor.execute_tensor(t10, concat=True)[0]
expected = np.searchsorted(raw, raw2, side='right')
np.testing.assert_array_equal(res, expected)
# test one chunk
arr = tensor(raw, chunk_size=16)
# test scalar, tensor to search has 1 chunk
t11 = searchsorted(arr, 20)
res = self.executor.execute_tensor(t11, concat=True)[0]
expected = np.searchsorted(raw, 20)
np.testing.assert_array_equal(res, expected)
# test tensor with 1 chunk, tensor to search has 1 chunk
t12 = searchsorted(arr, tensor(raw2, chunk_size=4))
res = self.executor.execute_tensor(t12, concat=True)[0]
expected = np.searchsorted(raw, raw2)
np.testing.assert_array_equal(res, expected)
# test tensor with more than 1 chunk, tensor to search has 1 chunk
t13 = searchsorted(arr, tensor(raw2, chunk_size=2))
res = self.executor.execute_tensor(t13, concat=True)[0]
expected = np.searchsorted(raw, raw2)
np.testing.assert_array_equal(res, expected)
# test sorter
raw3 = np.random.randint(100, size=(16, ))
arr = tensor(raw3, chunk_size=3)
order = np.argsort(raw3)
order_arr = tensor(order, chunk_size=4)
t14 = searchsorted(arr, 20, sorter=order_arr)
res = self.executor.execute_tensor(t14, concat=True)[0]
expected = np.searchsorted(raw3, 20, sorter=order)
np.testing.assert_array_equal(res, expected)
def testUniqueExecution(self):
rs = np.random.RandomState(0)
raw = rs.randint(10, size=(10, ))
for chunk_size in (10, 3):
x = tensor(raw, chunk_size=chunk_size)
y = unique(x)
res = self.executor.execute_tensor(y, concat=True)[0]
expected = np.unique(raw)
np.testing.assert_array_equal(res, expected)
y, indices = unique(x, return_index=True)
res = self.executor.execute_tensors([y, indices])
expected = np.unique(raw, return_index=True)
self.assertEqual(len(res), 2)
self.assertEqual(len(expected), 2)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
y, inverse = unique(x, return_inverse=True)
res = self.executor.execute_tensors([y, inverse])
expected = np.unique(raw, return_inverse=True)
self.assertEqual(len(res), 2)
self.assertEqual(len(expected), 2)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
y, counts = unique(x, return_counts=True)
res = self.executor.execute_tensors([y, counts])
expected = np.unique(raw, return_counts=True)
self.assertEqual(len(res), 2)
self.assertEqual(len(expected), 2)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
y, indices, inverse, counts = unique(x,
return_index=True,
return_inverse=True,
return_counts=True)
res = self.executor.execute_tensors([y, indices, inverse, counts])
expected = np.unique(raw,
return_index=True,
return_inverse=True,
return_counts=True)
self.assertEqual(len(res), 4)
self.assertEqual(len(expected), 4)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
np.testing.assert_array_equal(res[2], expected[2])
np.testing.assert_array_equal(res[3], expected[3])
y, indices, counts = unique(x,
return_index=True,
return_counts=True)
res = self.executor.execute_tensors([y, indices, counts])
expected = np.unique(raw, return_index=True, return_counts=True)
self.assertEqual(len(res), 3)
self.assertEqual(len(expected), 3)
np.testing.assert_array_equal(res[0], expected[0])
np.testing.assert_array_equal(res[1], expected[1])
np.testing.assert_array_equal(res[2], expected[2])
raw2 = rs.randint(10, size=(4, 5, 6))
x2 = tensor(raw2, chunk_size=chunk_size)
y2 = unique(x2)
res = self.executor.execute_tensor(y2, concat=True)[0]
expected = np.unique(raw2)
np.testing.assert_array_equal(res, expected)
y2 = unique(x2, axis=1)
res = self.executor.execute_tensor(y2, concat=True)[0]
expected = np.unique(raw2, axis=1)
np.testing.assert_array_equal(res, expected)
y2 = unique(x2, axis=2)
res = self.executor.execute_tensor(y2, concat=True)[0]
expected = np.unique(raw2, axis=2)
np.testing.assert_array_equal(res, expected)
raw = rs.randint(10, size=(10, 20))
raw[:, 0] = raw[:, 11] = rs.randint(10, size=(10, ))
x = tensor(raw, chunk_size=2)
y, ind, inv, counts = unique(x,
aggregate_size=3,
axis=1,
return_index=True,
return_inverse=True,
return_counts=True)
res_unique, res_ind, res_inv, res_counts = self.executor.execute_tensors(
(y, ind, inv, counts))
exp_unique, exp_ind, exp_counts = np.unique(raw,
axis=1,
return_index=True,
return_counts=True)
raw_res_unique = res_unique
res_unique_df = pd.DataFrame(res_unique)
res_unique_ind = np.asarray(
res_unique_df.sort_values(list(range(res_unique.shape[0])),
axis=1).columns)
res_unique = res_unique[:, res_unique_ind]
res_ind = res_ind[res_unique_ind]
res_counts = res_counts[res_unique_ind]
np.testing.assert_array_equal(res_unique, exp_unique)
np.testing.assert_array_equal(res_ind, exp_ind)
np.testing.assert_array_equal(raw_res_unique[:, res_inv], raw)
np.testing.assert_array_equal(res_counts, exp_counts)
x = (mt.random.RandomState(0).rand(1000, chunk_size=20) > 0.5).astype(
np.int32)
y = unique(x)
res = np.sort(self.executor.execute_tensor(y, concat=True)[0])
np.testing.assert_array_equal(res, np.array([0, 1]))
@require_cupy
def testToGPUExecution(self):
raw = np.random.rand(10, 10)
x = tensor(raw, chunk_size=3)
gx = to_gpu(x)
res = self.executor.execute_tensor(gx, concat=True)[0]
np.testing.assert_array_equal(res.get(), raw)
@require_cupy
def testToCPUExecution(self):
raw = np.random.rand(10, 10)
x = tensor(raw, chunk_size=3, gpu=True)
cx = to_cpu(x)
res = self.executor.execute_tensor(cx, concat=True)[0]
np.testing.assert_array_equal(res, raw)
def testSortExecution(self):
# only 1 chunk when axis = -1
raw = np.random.rand(100, 10)
x = tensor(raw, chunk_size=10)
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
# 1-d chunk
raw = np.random.rand(100)
x = tensor(raw, chunk_size=10)
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
# structured dtype
raw = np.empty(100, dtype=[('id', np.int32), ('size', np.int64)])
raw['id'] = np.random.randint(1000, size=100, dtype=np.int32)
raw['size'] = np.random.randint(1000, size=100, dtype=np.int64)
x = tensor(raw, chunk_size=10)
sx = sort(x, order=['size', 'id'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, order=['size', 'id']))
# test flatten case
raw = np.random.rand(10, 10)
x = tensor(raw, chunk_size=5)
sx = sort(x, axis=None)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=None))
# test multi-dimension
raw = np.random.rand(10, 100)
x = tensor(raw, chunk_size=(2, 10))
sx = sort(x, psrs_kinds=['quicksort'] * 3)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
raw = np.random.rand(10, 99)
x = tensor(raw, chunk_size=(2, 10))
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
# test 3-d
raw = np.random.rand(20, 25, 28)
x = tensor(raw, chunk_size=(10, 5, 7))
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
sx = sort(x, axis=0)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=0))
sx = sort(x, axis=1)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=1))
# test multi-dimension with structured type
raw = np.empty((10, 100), dtype=[('id', np.int32), ('size', np.int64)])
raw['id'] = np.random.randint(1000, size=(10, 100), dtype=np.int32)
raw['size'] = np.random.randint(1000, size=(10, 100), dtype=np.int64)
x = tensor(raw, chunk_size=(3, 10))
sx = sort(x)
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw))
sx = sort(x, order=['size', 'id'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, order=['size', 'id']))
sx = sort(x, order=['size'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, order=['size']))
sx = sort(x, axis=0, order=['size', 'id'])
res = self.executor.execute_tensor(sx, concat=True)[0]
np.testing.assert_array_equal(
res, np.sort(raw, axis=0, order=['size', 'id']))
raw = np.random.rand(10, 12)
a = tensor(raw, chunk_size=(5, 4))
a.sort(axis=1)
res = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(raw, axis=1))
a.sort(axis=0)
res = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res, np.sort(np.sort(raw, axis=1),
axis=0))
def setUp(self):
self.executor = Executor('numpy')
self.old_chunk = options.tensor.chunk_size
options.tensor.chunk_size = 10
def testExecutorWithGeventProvider(self):
executor = Executor(sync_provider_type=Executor.SyncProviderType.GEVENT)
a = mt.ones((10, 10), chunk_size=2)
res = executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((10, 10)))
def setUp(self):
self.executor = Executor('numpy')
local_session = LocalSession()
local_session._executor = self.executor
self.session = Session()
self.session._sess = local_session
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
self.old_chunk = options.tensor.chunk_size
options.tensor.chunk_size = 10
def tearDown(self):
options.tensor.chunk_size = self.old_chunk
def testBoolIndexingExecution(self):
raw = np.random.random((11, 8, 12, 14))
arr = tensor(raw, chunk_size=3)
index = arr < .5
arr2 = arr[index]
size_res = self.executor.execute_tensor(arr2, mock=True)
res = self.executor.execute_tensor(arr2)
self.assertEqual(sum(s[0] for s in size_res), arr.nbytes)
np.testing.assert_array_equal(np.sort(np.concatenate(res)),
np.sort(raw[raw < .5]))
index2 = tensor(raw[:, :, 0, 0], chunk_size=3) < .5
arr3 = arr[index2]
res = self.executor.execute_tensor(arr3)
self.assertEqual(sum(it.size for it in res),
raw[raw[:, :, 0, 0] < .5].size)
def testFancyIndexingNumpyExecution(self):
# test fancy index of type numpy ndarray
raw = np.random.random((11, 8, 12, 14))
arr = tensor(raw, chunk_size=(2, 3, 2, 3))
index = [8, 10, 3, 1, 9, 10]
arr2 = arr[index]
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw[index])
index = np.random.permutation(8)
arr3 = arr[:2, ..., index]
res = self.executor.execute_tensor(arr3, concat=True)
np.testing.assert_array_equal(res[0], raw[:2, ..., index])
index = [1, 3, 9, 10]
arr4 = arr[..., index, :5]
res = self.executor.execute_tensor(arr4, concat=True)
np.testing.assert_array_equal(res[0], raw[..., index, :5])
index1 = [8, 10, 3, 1, 9, 10]
index2 = [1, 3, 9, 10, 2, 7]
arr5 = arr[index1, :, index2]
res = self.executor.execute_tensor(arr5, concat=True)
np.testing.assert_array_equal(res[0], raw[index1, :, index2])
index1 = [1, 3, 5, 7, 9, 10]
index2 = [1, 9, 9, 10, 2, 7]
arr6 = arr[index1, :, index2]
res = self.executor.execute_tensor(arr6, concat=True)
np.testing.assert_array_equal(res[0], raw[index1, :, index2])
# fancy index is ordered, no concat required
self.assertGreater(len(arr6.nsplits[0]), 1)
index1 = [[8, 10, 3], [1, 9, 10]]
index2 = [[1, 3, 9], [10, 2, 7]]
arr7 = arr[index1, :, index2]
res = self.executor.execute_tensor(arr7, concat=True)
np.testing.assert_array_equal(res[0], raw[index1, :, index2])
index1 = [[1, 3], [3, 7], [7, 7]]
index2 = [1, 9]
arr8 = arr[0, index1, :, index2]
res = self.executor.execute_tensor(arr8, concat=True)
np.testing.assert_array_equal(res[0], raw[0, index1, :, index2])
def testFancyIndexingTensorExecution(self):
# test fancy index of type tensor
raw = np.random.random((11, 8, 12, 14))
arr = tensor(raw, chunk_size=(2, 3, 2, 3))
raw_index = [8, 10, 3, 1, 9, 10]
index = tensor(raw_index, chunk_size=4)
arr2 = arr[index]
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw[raw_index])
raw_index = np.random.permutation(8)
index = tensor(raw_index, chunk_size=3)
arr3 = arr[:2, ..., index]
res = self.executor.execute_tensor(arr3, concat=True)
np.testing.assert_array_equal(res[0], raw[:2, ..., raw_index])
raw_index = [1, 3, 9, 10]
index = tensor(raw_index)
arr4 = arr[..., index, :5]
res = self.executor.execute_tensor(arr4, concat=True)
np.testing.assert_array_equal(res[0], raw[..., raw_index, :5])
raw_index1 = [8, 10, 3, 1, 9, 10]
raw_index2 = [1, 3, 9, 10, 2, 7]
index1 = tensor(raw_index1, chunk_size=4)
index2 = tensor(raw_index2, chunk_size=3)
arr5 = arr[index1, :, index2]
res = self.executor.execute_tensor(arr5, concat=True)
np.testing.assert_array_equal(res[0], raw[raw_index1, :, raw_index2])
raw_index1 = [1, 3, 5, 7, 9, 10]
raw_index2 = [1, 9, 9, 10, 2, 7]
index1 = tensor(raw_index1, chunk_size=3)
index2 = tensor(raw_index2, chunk_size=4)
arr6 = arr[index1, :, index2]
res = self.executor.execute_tensor(arr6, concat=True)
np.testing.assert_array_equal(res[0], raw[raw_index1, :, raw_index2])
raw_index1 = [[8, 10, 3], [1, 9, 10]]
raw_index2 = [[1, 3, 9], [10, 2, 7]]
index1 = tensor(raw_index1)
index2 = tensor(raw_index2, chunk_size=2)
arr7 = arr[index1, :, index2]
res = self.executor.execute_tensor(arr7, concat=True)
np.testing.assert_array_equal(res[0], raw[raw_index1, :, raw_index2])
raw_index1 = [[1, 3], [3, 7], [7, 7]]
raw_index2 = [1, 9]
index1 = tensor(raw_index1, chunk_size=(2, 1))
index2 = tensor(raw_index2)
arr8 = arr[0, index1, :, index2]
res = self.executor.execute_tensor(arr8, concat=True)
np.testing.assert_array_equal(res[0], raw[0, raw_index1, :,
raw_index2])
raw_a = np.random.rand(30, 30)
a = tensor(raw_a, chunk_size=(13, 17))
b = a.argmax(axis=0)
c = a[b, arange(30)]
res = self.executor.execute_tensor(c, concat=True)
np.testing.assert_array_equal(
res[0], raw_a[raw_a.argmax(axis=0),
np.arange(30)])
def testSliceExecution(self):
raw = np.random.random((11, 8, 12, 14))
arr = tensor(raw, chunk_size=3)
arr2 = arr[2:9:2, 3:7, -1:-9:-2, 12:-11:-4]
res = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_array_equal(res[0], raw[2:9:2, 3:7, -1:-9:-2,
12:-11:-4])
arr3 = arr[-4, 2:]
res = self.executor.execute_tensor(arr3, concat=True)
np.testing.assert_equal(res[0], raw[-4, 2:])
raw = sps.random(12, 14, density=.1)
arr = tensor(raw, chunk_size=3)
arr2 = arr[-1:-9:-2, 12:-11:-4]
res = self.executor.execute_tensor(arr2, concat=True)[0]
np.testing.assert_equal(res.toarray(),
raw.toarray()[-1:-9:-2, 12:-11:-4])
def testMixedIndexingExecution(self):
raw = np.random.random((11, 8, 12, 13))
arr = tensor(raw, chunk_size=3)
raw_cond = raw[0, :, 0, 0] < .5
cond = tensor(raw[0, :, 0, 0], chunk_size=3) < .5
arr2 = arr[10::-2, cond, None, ..., :5]
size_res = self.executor.execute_tensor(arr2, mock=True)
res = self.executor.execute_tensor(arr2, concat=True)
new_shape = list(arr2.shape)
new_shape[1] = cond.shape[0]
self.assertEqual(sum(s[0] for s in size_res),
int(np.prod(new_shape) * arr2.dtype.itemsize))
np.testing.assert_array_equal(res[0], raw[10::-2, raw_cond, None,
..., :5])
b_raw = np.random.random(8)
cond = tensor(b_raw, chunk_size=2) < .5
arr3 = arr[-2::-3, cond, ...]
res = self.executor.execute_tensor(arr3, concat=True)
np.testing.assert_array_equal(res[0], raw[-2::-3, b_raw < .5, ...])
def testSetItemExecution(self):
raw = data = np.random.randint(0, 10, size=(11, 8, 12, 13))
arr = tensor(raw.copy(), chunk_size=3)
raw = raw.copy()
idx = slice(2, 9, 2), slice(3, 7), slice(-1, -9, -2), 2
arr[idx] = 20
res = self.executor.execute_tensor(arr, concat=True)
raw[idx] = 20
np.testing.assert_array_equal(res[0], raw)
raw = data
shape = raw[idx].shape
arr2 = tensor(raw.copy(), chunk_size=3)
raw = raw.copy()
replace = np.random.randint(10, 20,
size=shape[:-1] + (1, )).astype('f4')
arr2[idx] = tensor(replace, chunk_size=4)
res = self.executor.execute_tensor(arr2, concat=True)
raw[idx] = replace
np.testing.assert_array_equal(res[0], raw)
def testSetItemStructuredExecution(self):
rec_type = np.dtype([('a', np.int32), ('b', np.double),
('c', np.dtype([('a', np.int16),
('b', np.int64)]))])
raw = np.zeros((4, 5), dtype=rec_type)
arr = tensor(raw.copy(), chunk_size=3)
arr[1:4, 1] = (3, 4., (5, 6))
arr[1:4, 2] = 8
arr[1:3] = np.arange(5)
arr[2:4] = np.arange(10).reshape(2, 5)
arr[0] = np.arange(5)
raw[1:4, 1] = (3, 4., (5, 6))
raw[1:4, 2] = 8
raw[1:3] = np.arange(5)
raw[2:4] = np.arange(10).reshape(2, 5)
raw[0] = np.arange(5)
res = self.executor.execute_tensor(arr, concat=True)
self.assertEqual(arr.dtype, raw.dtype)
self.assertEqual(arr.shape, raw.shape)
np.testing.assert_array_equal(res[0], raw)
def testTakeExecution(self):
data = np.random.rand(10, 20, 30)
t = tensor(data, chunk_size=10)
a = t.take([4, 1, 2, 6, 200])
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.take(data, [4, 1, 2, 6, 200])
np.testing.assert_array_equal(res, expected)
a = take(t, [5, 19, 2, 13], axis=1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.take(data, [5, 19, 2, 13], axis=1)
np.testing.assert_array_equal(res, expected)
def testCompressExecution(self):
data = np.array([[1, 2], [3, 4], [5, 6]])
a = tensor(data, chunk_size=1)
t = compress([0, 1], a, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([0, 1], data, axis=0)
np.testing.assert_array_equal(res, expected)
t = compress([0, 1], a, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([0, 1], data, axis=1)
np.testing.assert_array_equal(res, expected)
t = a.compress([0, 1, 1])
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([0, 1, 1], data)
np.testing.assert_array_equal(res, expected)
t = compress([False, True, True], a, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([False, True, True], data, axis=0)
np.testing.assert_array_equal(res, expected)
t = compress([False, True], a, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([False, True], data, axis=1)
np.testing.assert_array_equal(res, expected)
with self.assertRaises(np.AxisError):
compress([0, 1, 1], a, axis=1)
def testExtractExecution(self):
data = np.arange(12).reshape((3, 4))
a = tensor(data, chunk_size=2)
condition = mod(a, 3) == 0
t = extract(condition, a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.extract(np.mod(data, 3) == 0, data)
np.testing.assert_array_equal(res, expected)
def testChooseExecution(self):
options.tensor.chunk_size = 2
choices = [[0, 1, 2, 3], [10, 11, 12, 13], [20, 21, 22, 23],
[30, 31, 32, 33]]
a = choose([2, 3, 1, 0], choices)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.choose([2, 3, 1, 0], choices)
np.testing.assert_array_equal(res, expected)
a = choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1)
expected = np.choose([2, 4, 1, 0], choices, mode='clip')
res = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res, expected)
a = choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
expected = np.choose([2, 4, 1, 0], choices,
mode='wrap') # 4 goes to (4 mod 4)
res = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_array_equal(res, expected)
a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]]
choices = [-10, 10]
b = choose(a, choices)
expected = np.choose(a, choices)
res = self.executor.execute_tensor(b, concat=True)[0]
np.testing.assert_array_equal(res, expected)
a = np.array([0, 1]).reshape((2, 1, 1))
c1 = np.array([1, 2, 3]).reshape((1, 3, 1))
c2 = np.array([-1, -2, -3, -4, -5]).reshape((1, 1, 5))
b = choose(a, (c1, c2))
expected = np.choose(a, (c1, c2))
res = self.executor.execute_tensor(b, concat=True)[0]
np.testing.assert_array_equal(res, expected)
def testUnravelExecution(self):
a = tensor([22, 41, 37], chunk_size=1)
t = stack(unravel_index(a, (7, 6)))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.stack(np.unravel_index([22, 41, 37], (7, 6)))
np.testing.assert_array_equal(res, expected)
def testNonzeroExecution(self):
data = np.array([[1, 0, 0], [0, 2, 0], [1, 1, 0]])
x = tensor(data, chunk_size=2)
t = hstack(nonzero(x))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.hstack(np.nonzero(data))
np.testing.assert_array_equal(res, expected)
def testFlatnonzeroExecution(self):
x = arange(-2, 3, chunk_size=2)
t = flatnonzero(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flatnonzero(np.arange(-2, 3))
np.testing.assert_equal(res, expected)
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
self.old_chunk = options.tensor.chunk_size
options.tensor.chunk_size = 10
def tearDown(self):
options.tensor.chunk_size = self.old_chunk
def testBoolIndexingExecution(self):
raw = np.random.random((11, 8, 12, 14))
arr = tensor(raw, chunk_size=3)
index = arr < .5
arr2 = arr[index]
size_res = self.executor.execute_tensor(arr2, mock=True)
res = self.executor.execute_tensor(arr2)
self.assertEqual(sum(s[0] for s in size_res), arr.nbytes)
self.assertTrue(np.array_equal(np.sort(np.concatenate(res)), np.sort(raw[raw < .5])))
index2 = tensor(raw[:, :, 0, 0], chunk_size=3) < .5
arr3 = arr[index2]
res = self.executor.execute_tensor(arr3)
self.assertEqual(sum(it.size for it in res), raw[raw[:, :, 0, 0] < .5].size)
def testFancyIndexingExecution(self):
raw = np.random.random((11, 8, 12, 14))
arr = tensor(raw, chunk_size=(2, 3, 2, 3))
index = [8, 10, 3, 1, 9, 10]
arr2 = arr[index]
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.array_equal(res[0], raw[index]))
index = np.random.permutation(8)
arr3 = arr[:2, ..., index]
res = self.executor.execute_tensor(arr3, concat=True)
self.assertTrue(np.array_equal(res[0], raw[:2, ..., index]))
index = [1, 3, 9, 10]
arr4 = arr[..., index, :5]
res = self.executor.execute_tensor(arr4, concat=True)
self.assertTrue(np.array_equal(res[0], raw[..., index, :5]))
def testSliceExecution(self):
raw = np.random.random((11, 8, 12, 14))
arr = tensor(raw, chunk_size=3)
arr2 = arr[2:9:2, 3:7, -1:-9:-2, 12:-11:-4]
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.array_equal(res[0], raw[2:9:2, 3:7, -1:-9:-2, 12:-11:-4]))
raw = sps.random(12, 14, density=.1)
arr = tensor(raw, chunk_size=3)
arr2 = arr[-1:-9:-2, 12:-11:-4]
res = self.executor.execute_tensor(arr2, concat=True)[0]
np.testing.assert_equal(res.toarray(), raw.toarray()[-1:-9:-2, 12:-11:-4])
def testMixedIndexingExecution(self):
raw = np.random.random((11, 8, 12, 13))
arr = tensor(raw, chunk_size=3)
raw_cond = raw[0, :, 0, 0] < .5
cond = tensor(raw[0, :, 0, 0], chunk_size=3) < .5
arr2 = arr[10::-2, cond, None, ..., :5]
size_res = self.executor.execute_tensor(arr2, mock=True)
res = self.executor.execute_tensor(arr2, concat=True)
new_shape = list(arr2.shape)
new_shape[1] = cond.shape[0]
self.assertEqual(sum(s[0] for s in size_res), int(np.prod(new_shape) * arr2.dtype.itemsize))
self.assertTrue(np.array_equal(res[0], raw[10::-2, raw_cond, None, ..., :5]))
b_raw = np.random.random(8)
cond = tensor(b_raw, chunk_size=2) < .5
arr3 = arr[-2::-3, cond, ...]
res = self.executor.execute_tensor(arr3, concat=True)
self.assertTrue(np.array_equal(res[0], raw[-2::-3, b_raw < .5, ...]))
def testSetItemExecution(self):
raw = data = np.random.randint(0, 10, size=(11, 8, 12, 13))
arr = tensor(raw.copy(), chunk_size=3)
raw = raw.copy()
idx = slice(2, 9, 2), slice(3, 7), slice(-1, -9, -2), 2
arr[idx] = 20
res = self.executor.execute_tensor(arr, concat=True)
raw[idx] = 20
self.assertTrue(np.array_equal(res[0], raw))
raw = data
shape = raw[idx].shape
arr2 = tensor(raw.copy(), chunk_size=3)
raw = raw.copy()
replace = np.random.randint(10, 20, size=shape[:-1] + (1,)).astype('f4')
arr2[idx] = tensor(replace, chunk_size=4)
res = self.executor.execute_tensor(arr2, concat=True)
raw[idx] = replace
self.assertTrue(np.array_equal(res[0], raw))
def testTakeExecution(self):
data = np.random.rand(10, 20, 30)
t = tensor(data, chunk_size=10)
a = t.take([4, 1, 2, 6, 200])
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.take(data, [4, 1, 2, 6, 200])
self.assertTrue(np.array_equal(res, expected))
a = take(t, [5, 19, 2, 13], axis=1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.take(data, [5, 19, 2, 13], axis=1)
self.assertTrue(np.array_equal(res, expected))
def testCompressExecution(self):
data = np.array([[1, 2], [3, 4], [5, 6]])
a = tensor(data, chunk_size=1)
t = compress([0, 1], a, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([0, 1], data, axis=0)
self.assertTrue(np.array_equal(res, expected))
t = compress([0, 1], a, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([0, 1], data, axis=1)
self.assertTrue(np.array_equal(res, expected))
t = a.compress([0, 1, 1])
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([0, 1, 1], data)
self.assertTrue(np.array_equal(res, expected))
t = compress([False, True, True], a, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([False, True, True], data, axis=0)
self.assertTrue(np.array_equal(res, expected))
t = compress([False, True], a, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.compress([False, True], data, axis=1)
self.assertTrue(np.array_equal(res, expected))
with self.assertRaises(np.AxisError):
compress([0, 1, 1], a, axis=1)
def testExtractExecution(self):
data = np.arange(12).reshape((3, 4))
a = tensor(data, chunk_size=2)
condition = mod(a, 3) == 0
t = extract(condition, a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.extract(np.mod(data, 3) == 0, data)
self.assertTrue(np.array_equal(res, expected))
def testChooseExecution(self):
options.tensor.chunk_size = 2
choices = [[0, 1, 2, 3], [10, 11, 12, 13],
[20, 21, 22, 23], [30, 31, 32, 33]]
a = choose([2, 3, 1, 0], choices)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.choose([2, 3, 1, 0], choices)
self.assertTrue(np.array_equal(res, expected))
a = choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1)
expected = np.choose([2, 4, 1, 0], choices, mode='clip')
res = self.executor.execute_tensor(a, concat=True)[0]
self.assertTrue(np.array_equal(res, expected))
a = choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
expected = np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4)
res = self.executor.execute_tensor(a, concat=True)[0]
self.assertTrue(np.array_equal(res, expected))
a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]]
choices = [-10, 10]
b = choose(a, choices)
expected = np.choose(a, choices)
res = self.executor.execute_tensor(b, concat=True)[0]
self.assertTrue(np.array_equal(res, expected))
a = np.array([0, 1]).reshape((2, 1, 1))
c1 = np.array([1, 2, 3]).reshape((1, 3, 1))
c2 = np.array([-1, -2, -3, -4, -5]).reshape((1, 1, 5))
b = choose(a, (c1, c2))
expected = np.choose(a, (c1, c2))
res = self.executor.execute_tensor(b, concat=True)[0]
self.assertTrue(np.array_equal(res, expected))
def testUnravelExecution(self):
a = tensor([22, 41, 37], chunk_size=1)
t = stack(unravel_index(a, (7, 6)))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.stack(np.unravel_index([22, 41, 37], (7, 6)))
self.assertTrue(np.array_equal(res, expected))
def testNonzeroExecution(self):
data = np.array([[1, 0, 0], [0, 2, 0], [1, 1, 0]])
x = tensor(data, chunk_size=2)
t = hstack(nonzero(x))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.hstack(np.nonzero(data))
self.assertTrue(np.array_equal(res, expected))
def testFlatnonzeroExecution(self):
x = arange(-2, 3, chunk_size=2)
t = flatnonzero(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flatnonzero(np.arange(-2, 3))
np.testing.assert_equal(res, expected)
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testRandExecution(self):
arr = tensor.random.rand(10, 20, chunk_size=3, dtype='f4')
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertEqual(res.shape, (10, 20))
self.assertTrue(np.all(res < 1))
self.assertTrue(np.all(res > 0))
self.assertEqual(res.dtype, np.float32)
def testRandnExecution(self):
arr = tensor.random.randn(10, 20, chunk_size=3)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20))
arr = tensor.random.randn(10, 20, chunk_size=5).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).randn(5, 5)))
def testRandintExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
arr = tensor.random.randint(0, 2, size=(10, 30), chunk_size=3)
size_res = size_executor.execute_tensor(arr, mock=True)
self.assertEqual(arr.nbytes, sum(tp[0] for tp in size_res))
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertEqual(res.shape, (10, 30))
self.assertTrue(np.all(res >= 0))
self.assertTrue(np.all(res < 2))
def testRandomIntegersExecution(self):
arr = tensor.random.random_integers(0, 10, size=(10, 20), chunk_size=3)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20))
arr = tensor.random.random_integers(0, 10, size=(10, 20),
chunk_size=5).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
np.testing.assert_equal(
res,
np.random.RandomState(0).random_integers(0, 10, size=(5, 5)))
def testRandomSampleExecution(self):
arr = tensor.random.random_sample(size=(10, 20), chunk_size=3)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20))
arr = tensor.random.random_sample(size=(10, 20), chunk_size=5).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).random_sample(size=(5, 5))))
def testRandomExecution(self):
arr = tensor.random.random(size=(10, 20), chunk_size=3)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20))
arr = tensor.random.random(size=(10, 20), chunk_size=5).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).random_sample(size=(5, 5))))
def testRandfExecution(self):
arr = tensor.random.ranf(size=(10, 20), chunk_size=3)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20))
arr = tensor.random.ranf(size=(10, 20), chunk_size=5).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).random_sample(size=(5, 5))))
def testSampleExecution(self):
arr = tensor.random.sample(size=(10, 20), chunk_size=3)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20))
arr = tensor.random.sample(size=(10, 20), chunk_size=5).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).random_sample(size=(5, 5))))
def testChoiceExecution(self):
arr = tensor.random.choice(5, size=3, chunk_size=1)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (3, ))
arr = tensor.random.choice(5, size=(15, ), chunk_size=5).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).choice(5, size=(5, ))))
arr = tensor.random.choice([1, 4, 9], size=3, chunk_size=1)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (3, ))
arr = tensor.random.choice([1, 4, 9], size=(15, ),
chunk_size=5).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).choice([1, 4, 9], size=(5, ))))
with self.assertRaises(ValueError):
tensor.random.choice([1, 3, 4],
size=5,
replace=False,
chunk_size=2)
arr = tensor.random.choice([1, 4, 9],
size=3,
replace=False,
chunk_size=1)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (3, ))
arr = tensor.random.choice([1, 4, 9],
size=(3, ),
replace=False,
chunk_size=1).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).choice([1, 4, 9],
size=(1, ),
replace=False)))
arr = tensor.random.choice([1, 4, 9],
size=3,
p=[.2, .5, .3],
chunk_size=1)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (3, ))
arr = tensor.random.choice([1, 4, 9],
size=(15, ),
chunk_size=5,
p=[.2, .5, .3]).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).choice([1, 4, 9],
size=(5, ),
p=[.2, .5, .3])))
def testSparseRandintExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
arr = tensor.random.randint(1,
2,
size=(30, 50),
density=.1,
chunk_size=10,
dtype='f4')
size_res = size_executor.execute_tensor(arr, mock=True)
self.assertAlmostEqual(arr.nbytes * 0.1, sum(tp[0] for tp in size_res))
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertTrue(issparse(res))
self.assertEqual(res.shape, (30, 50))
self.assertTrue(np.all(res.data >= 1))
self.assertTrue(np.all(res.data < 2))
self.assertAlmostEqual((res >= 1).toarray().sum(),
30 * 50 * .1,
delta=20)
def testBetaExecute(self):
arr = tensor.random.beta(1, 2, chunk_size=2).tiles()
arr.chunks[0].op._seed = 0
self.assertEqual(
self.executor.execute_tensor(arr)[0],
np.random.RandomState(0).beta(1, 2))
arr = tensor.random.beta([1, 2], [3, 4], chunk_size=2).tiles()
arr.chunks[0].op._seed = 0
self.assertTrue(
np.array_equal(
self.executor.execute_tensor(arr)[0],
np.random.RandomState(0).beta([1, 2], [3, 4])))
arr = tensor.random.beta([[2, 3]],
from_ndarray([[4, 6], [5, 2]], chunk_size=2),
chunk_size=1,
size=(3, 2, 2)).tiles()
for c in arr.chunks:
c.op._seed = 0
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertEqual(res[0, 0, 0], np.random.RandomState(0).beta(2, 4))
self.assertEqual(res[0, 0, 1], np.random.RandomState(0).beta(3, 6))
self.assertEqual(res[0, 1, 0], np.random.RandomState(0).beta(2, 5))
self.assertEqual(res[0, 1, 1], np.random.RandomState(0).beta(3, 2))
arr = tensor.random.RandomState(0).beta([[3, 4]], [[1], [2]],
chunk_size=1)
tensor.random.seed(0)
arr2 = tensor.random.beta([[3, 4]], [[1], [2]], chunk_size=1)
self.assertTrue(
np.array_equal(
self.executor.execute_tensor(arr, concat=True)[0],
self.executor.execute_tensor(arr2, concat=True)[0]))
def testBinomialExecute(self):
arr = tensor.random.binomial(10, .5, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.binomial(10, .5, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).binomial(10, .5, 10)))
def testChisquareExecute(self):
arr = tensor.random.chisquare(2, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.chisquare(2, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).chisquare(2, 10)))
def testDirichletExecute(self):
arr = tensor.random.dirichlet((10, 5, 3), 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 3))
arr = tensor.random.dirichlet((10, 5, 3), 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).dirichlet((10, 5, 3), 10)))
def testExponentialExecute(self):
arr = tensor.random.exponential(1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.exponential(1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).exponential(1.0, 10)))
def testFExecute(self):
arr = tensor.random.f(1.0, 2.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.f(1.0, 2.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).f(1.0, 2.0, 10)))
def testGammaExecute(self):
arr = tensor.random.gamma(1.0, 2.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.gamma(1.0, 2.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).gamma(1.0, 2.0, 10)))
def testGeometricExecution(self):
arr = tensor.random.geometric(1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.geometric(1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).geometric(1.0, 10)))
def testGumbelExecution(self):
arr = tensor.random.gumbel(.5, 1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.gumbel(.5, 1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).gumbel(.5, 1.0, 10)))
def testHypergeometricExecution(self):
arr = tensor.random.hypergeometric(10, 20, 15, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.hypergeometric(10, 20, 15, 100,
chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).hypergeometric(10, 20, 15, 10)))
def testLaplaceExecution(self):
arr = tensor.random.laplace(.5, 1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.laplace(.5, 1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).laplace(.5, 1.0, 10)))
def testLogisticExecution(self):
arr = tensor.random.logistic(.5, 1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.logistic(.5, 1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
np.testing.assert_equal(
res,
np.random.RandomState(0).logistic(.5, 1.0, 10))
def testLognormalExecution(self):
arr = tensor.random.lognormal(.5, 1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.lognormal(.5, 1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).lognormal(.5, 1.0,
10)))
def testLogseriesExecution(self):
arr = tensor.random.logseries(.5, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.logseries(.5, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).logseries(.5, 10)))
def testMultinomialExecution(self):
arr = tensor.random.multinomial(10, [.2, .5, .3], 100, chunk_size=10)
self.assertEqual(arr.shape, (100, 3))
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 3))
arr = tensor.random.multinomial(10, [.2, .5, .3], 100,
chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).multinomial(10, [.2, .5, .3],
10)))
def testMultivariateNormalExecution(self):
arr = tensor.random.multivariate_normal([1, 2], [[1, 0], [0, 1]],
100,
chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 2))
arr = tensor.random.multivariate_normal([1, 2], [[1, 0], [0, 1]],
100,
chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).multivariate_normal(
[1, 2], [[1, 0], [0, 1]], 10)))
def testNegativeBinomialExecution(self):
arr = tensor.random.negative_binomial(5, 1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.negative_binomial(5, 1.0, 100,
chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).negative_binomial(5, 1.0, 10)))
def testNoncentralChisquareExecution(self):
arr = tensor.random.noncentral_chisquare(.5, 1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.noncentral_chisquare(.5, 1.0, 100,
chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).noncentral_chisquare(.5, 1.0,
10)))
def testNoncentralFExecution(self):
arr = tensor.random.noncentral_f(1.5, 1.0, 1.1, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.noncentral_f(1.5, 1.0, 1.1, 100,
chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).noncentral_f(1.5, 1.0, 1.1, 10)))
def testNormalExecute(self):
arr = tensor.random.normal(10, 1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.normal(10, 1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).normal(10, 1.0, 10)))
def testParetoExecute(self):
arr = tensor.random.pareto(1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.pareto(1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).pareto(1.0, 10)))
def testPoissonExecute(self):
arr = tensor.random.poisson(1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.poisson(1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).poisson(1.0, 10)))
def testPowerExecute(self):
arr = tensor.random.power(1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.power(1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).power(1.0, 10)))
def testRayleighExecute(self):
arr = tensor.random.rayleigh(1.0, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.rayleigh(1.0, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).rayleigh(1.0, 10)))
def testStandardCauchyExecute(self):
arr = tensor.random.standard_cauchy(100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.standard_cauchy(100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).standard_cauchy(10)))
def testStandardExponentialExecute(self):
arr = tensor.random.standard_exponential(100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.standard_exponential(100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).standard_exponential(10)))
def testStandardGammaExecute(self):
arr = tensor.random.standard_gamma(.1, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.standard_gamma(.1, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).standard_gamma(.1,
10)))
def testStandardNormalExecute(self):
arr = tensor.random.standard_normal(100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.standard_normal(100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).standard_normal(10)))
def testStandardTExecute(self):
arr = tensor.random.standard_t(.1, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.standard_t(.1, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).standard_t(.1, 10)))
def testTriangularExecute(self):
arr = tensor.random.triangular(.1, .2, .3, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.triangular(.1, .2, .3, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(
res,
np.random.RandomState(0).triangular(.1, .2, .3, 10)))
def testUniformExecute(self):
arr = tensor.random.uniform(.1, .2, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.uniform(.1, .2, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).uniform(.1, .2, 10)))
def testVonmisesExecute(self):
arr = tensor.random.vonmises(.1, .2, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.vonmises(.1, .2, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).vonmises(.1, .2, 10)))
def testWaldExecute(self):
arr = tensor.random.wald(.1, .2, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.wald(.1, .2, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).wald(.1, .2, 10)))
def testWeibullExecute(self):
arr = tensor.random.weibull(.1, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.weibull(.1, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).weibull(.1, 10)))
def testZipfExecute(self):
arr = tensor.random.zipf(1.1, 100, chunk_size=10)
self.assertEqual(
self.executor.execute_tensor(arr, concat=True)[0].shape, (100, ))
arr = tensor.random.zipf(1.1, 100, chunk_size=10).tiles()
for chunk in arr.chunks:
chunk.op._seed = 0
for res in self.executor.execute_tensor(arr):
self.assertTrue(
np.array_equal(res,
np.random.RandomState(0).zipf(1.1, 10)))
class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
def testMerge(self):
df1 = pd.DataFrame(np.arange(20).reshape((4, 5)) + 1,
columns=['a', 'b', 'c', 'd', 'e'])
df2 = pd.DataFrame(np.arange(20).reshape((5, 4)) + 1,
columns=['a', 'b', 'x', 'y'])
mdf1 = from_pandas(df1, chunk_size=2)
mdf2 = from_pandas(df2, chunk_size=2)
# Note [Index of Merge]
#
# When `left_index` and `right_index` of `merge` is both false, pandas will generate an RangeIndex to
# the final result dataframe.
#
# We chunked the `left` and `right` dataframe, thus every result chunk will have its own RangeIndex.
# When they are contenated we don't generate a new RangeIndex for the result, thus we cannot obtain the
# same index value with pandas. But we guarantee that the content of dataframe is correct.
# merge on index
expected0 = df1.merge(df2)
jdf0 = mdf1.merge(mdf2)
result0 = self.executor.execute_dataframe(jdf0, concat=True)[0]
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected0, 0),
sort_dataframe_inplace(result0, 0))
# merge on left index and `right_on`
expected1 = df1.merge(df2, how='left', right_on='x', left_index=True)
jdf1 = mdf1.merge(mdf2, how='left', right_on='x', left_index=True)
result1 = self.executor.execute_dataframe(jdf1, concat=True)[0]
expected1.set_index('a_x', inplace=True)
result1.set_index('a_x', inplace=True)
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected1, 0),
sort_dataframe_inplace(result1, 0))
# merge on `left_on` and right index
expected2 = df1.merge(df2, how='right', left_on='a', right_index=True)
jdf2 = mdf1.merge(mdf2, how='right', left_on='a', right_index=True)
result2 = self.executor.execute_dataframe(jdf2, concat=True)[0]
expected2.set_index('a', inplace=True)
result2.set_index('a', inplace=True)
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected2, 0),
sort_dataframe_inplace(result2, 0))
# merge on `left_on` and `right_on`
expected3 = df1.merge(df2, how='left', left_on='a', right_on='x')
jdf3 = mdf1.merge(mdf2, how='left', left_on='a', right_on='x')
result3 = self.executor.execute_dataframe(jdf3, concat=True)[0]
expected3.set_index('a_x', inplace=True)
result3.set_index('a_x', inplace=True)
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected3, 0),
sort_dataframe_inplace(result3, 0))
# merge on `on`
expected4 = df1.merge(df2, how='right', on='a')
jdf4 = mdf1.merge(mdf2, how='right', on='a')
result4 = self.executor.execute_dataframe(jdf4, concat=True)[0]
expected4.set_index('a', inplace=True)
result4.set_index('a', inplace=True)
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected4, 0),
sort_dataframe_inplace(result4, 0))
# merge on multiple columns
expected5 = df1.merge(df2, how='inner', on=['a', 'b'])
jdf5 = mdf1.merge(mdf2, how='inner', on=['a', 'b'])
result5 = self.executor.execute_dataframe(jdf5, concat=True)[0]
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected5, 0),
sort_dataframe_inplace(result5, 0))
def testJoin(self):
df1 = pd.DataFrame([[1, 3, 3], [4, 2, 6], [7, 8, 9]],
index=['a1', 'a2', 'a3'])
df2 = pd.DataFrame([[1, 2, 3], [1, 5, 6], [7, 8, 9]],
index=['a1', 'b2', 'b3']) + 1
df2 = pd.concat([df2, df2 + 1])
mdf1 = from_pandas(df1, chunk_size=2)
mdf2 = from_pandas(df2, chunk_size=2)
# default `how`
expected0 = df1.join(df2, lsuffix='l_', rsuffix='r_')
jdf0 = mdf1.join(mdf2, lsuffix='l_', rsuffix='r_')
result0 = self.executor.execute_dataframe(jdf0, concat=True)[0]
pd.testing.assert_frame_equal(expected0.sort_index(),
result0.sort_index())
# how = 'left'
expected1 = df1.join(df2, how='left', lsuffix='l_', rsuffix='r_')
jdf1 = mdf1.join(mdf2, how='left', lsuffix='l_', rsuffix='r_')
result1 = self.executor.execute_dataframe(jdf1, concat=True)[0]
pd.testing.assert_frame_equal(expected1.sort_index(),
result1.sort_index())
# how = 'right'
expected2 = df1.join(df2, how='right', lsuffix='l_', rsuffix='r_')
jdf2 = mdf1.join(mdf2, how='right', lsuffix='l_', rsuffix='r_')
result2 = self.executor.execute_dataframe(jdf2, concat=True)[0]
pd.testing.assert_frame_equal(expected2.sort_index(),
result2.sort_index())
# how = 'inner'
expected3 = df1.join(df2, how='inner', lsuffix='l_', rsuffix='r_')
jdf3 = mdf1.join(mdf2, how='inner', lsuffix='l_', rsuffix='r_')
result3 = self.executor.execute_dataframe(jdf3, concat=True)[0]
pd.testing.assert_frame_equal(expected3.sort_index(),
result3.sort_index())
# how = 'outer'
expected4 = df1.join(df2, how='outer', lsuffix='l_', rsuffix='r_')
jdf4 = mdf1.join(mdf2, how='outer', lsuffix='l_', rsuffix='r_')
result4 = self.executor.execute_dataframe(jdf4, concat=True)[0]
pd.testing.assert_frame_equal(expected4.sort_index(),
result4.sort_index())
def testJoinOn(self):
df1 = pd.DataFrame([[1, 3, 3], [4, 2, 6], [7, 8, 9]],
columns=['a1', 'a2', 'a3'])
df2 = pd.DataFrame([[1, 2, 3], [1, 5, 6], [7, 8, 9]],
columns=['a1', 'b2', 'b3']) + 1
df2 = pd.concat([df2, df2 + 1])
mdf1 = from_pandas(df1, chunk_size=2)
mdf2 = from_pandas(df2, chunk_size=2)
expected0 = df1.join(df2, on=None, lsuffix='_l', rsuffix='_r')
jdf0 = mdf1.join(mdf2, on=None, lsuffix='_l', rsuffix='_r')
result0 = self.executor.execute_dataframe(jdf0, concat=True)[0]
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected0, 0),
sort_dataframe_inplace(result0, 0))
expected1 = df1.join(df2,
how='left',
on='a1',
lsuffix='_l',
rsuffix='_r')
jdf1 = mdf1.join(mdf2, how='left', on='a1', lsuffix='_l', rsuffix='_r')
result1 = self.executor.execute_dataframe(jdf1, concat=True)[0]
# Note [Columns of Left Join]
#
# I believe we have no chance to obtain the entirely same result with pandas here:
#
# Look at the following example:
#
# >>> df1
# a1 a2 a3
# 0 1 3 3
# >>> df2
# a1 b2 b3
# 1 2 6 7
# >>> df3
# a1 b2 b3
# 1 2 6 7
# 1 2 6 7
#
# >>> df1.merge(df2, how='left', left_on='a1', left_index=False, right_index=True)
# a1_x a2 a3 a1_y b2 b3
# 0 1 3 3 2 6 7
# >>> df1.merge(df3, how='left', left_on='a1', left_index=False, right_index=True)
# a1 a1_x a2 a3 a1_y b2 b3
# 0 1 1 3 3 2 6 7
# 0 1 1 3 3 2 6 7
#
# Note that the result of `df1.merge(df3)` has an extra column `a` compared to `df1.merge(df2)`.
# The value of column `a` is the same of `a1_x`, just because `1` occurs twice in index of `df3`.
# I haven't invistagated why pandas has such behaviour...
#
# We cannot yeild the same result with pandas, because, the `df3` is chunked, then some of the
# result chunk has 6 columns, others may have 7 columns, when concatenated into one DataFrame
# some cells of column `a` will have value `NaN`, which is different from the result of pandas.
#
# But we can guarantee that other effective columns have absolutely same value with pandas.
columns_to_compare = jdf1.columns_value.to_pandas()
pd.testing.assert_frame_equal(
sort_dataframe_inplace(expected1[columns_to_compare], 0, 1),
sort_dataframe_inplace(result1[columns_to_compare], 0, 1))
# Note [Index of Join on EmptyDataFrame]
#
# It is tricky that it is non-trivial to get the same `index` result with pandas.
#
# Look at the following example:
#
# >>> df1
# a1 a2 a3
# 1 4 2 6
# >>> df2
# a1 b2 b3
# 1 2 6 7
# 2 8 9 10
# >>> df3
# Empty DataFrame
# Columns: [a1, a2, a3]
# Index: []
# >>> df1.join(df2, how='right', on='a2', lsuffix='_l', rsuffix='_r')
# a1_l a2 a3 a1_r b2 b3
# 1.0 4.0 2 6.0 8 9 10
# NaN NaN 1 NaN 2 6 7
# >>> df3.join(df2, how='right', on='a2', lsuffix='_l', rsuffix='_r')
# a1_l a2 a3 a1_r b2 b3
# 1 NaN 1 NaN 2 6 7
# 2 NaN 2 NaN 8 9 10
#
# When the `left` dataframe is not empty, the mismatched rows in `right` will have index value `NaN`,
# and the matched rows have index value from `right`. When the `left` dataframe is empty, the mismatched
# rows have index value from `right`.
#
# Since we chunked the `left` dataframe, it is uneasy to obtain the same index value with pandas in the
# final result dataframe, but we guaranteed that the dataframe content is correctly.
expected2 = df1.join(df2,
how='right',
on='a2',
lsuffix='_l',
rsuffix='_r')
jdf2 = mdf1.join(mdf2,
how='right',
on='a2',
lsuffix='_l',
rsuffix='_r')
result2 = self.executor.execute_dataframe(jdf2, concat=True)[0]
expected2.set_index('a2', inplace=True)
result2.set_index('a2', inplace=True)
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected2, 0),
sort_dataframe_inplace(result2, 0))
expected3 = df1.join(df2,
how='inner',
on='a2',
lsuffix='_l',
rsuffix='_r')
jdf3 = mdf1.join(mdf2,
how='inner',
on='a2',
lsuffix='_l',
rsuffix='_r')
result3 = self.executor.execute_dataframe(jdf3, concat=True)[0]
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected3, 0),
sort_dataframe_inplace(result3, 0))
expected4 = df1.join(df2,
how='outer',
on='a2',
lsuffix='_l',
rsuffix='_r')
jdf4 = mdf1.join(mdf2,
how='outer',
on='a2',
lsuffix='_l',
rsuffix='_r')
result4 = self.executor.execute_dataframe(jdf4, concat=True)[0]
expected4.set_index('a2', inplace=True)
result4.set_index('a2', inplace=True)
pd.testing.assert_frame_equal(sort_dataframe_inplace(expected4, 0),
sort_dataframe_inplace(result4, 0))
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testSumProdExecution(self):
arr = ones((10, 8), chunk_size=3)
self.assertEqual([80], self.executor.execute_tensor(arr.sum()))
self.assertEqual(
(10, ) * 8,
tuple(np.concatenate(self.executor.execute_tensor(
arr.sum(axis=0)))))
arr = ones((3, 3), chunk_size=2)
self.assertEqual([512], self.executor.execute_tensor((arr * 2).prod()))
self.assertEqual((8, ) * 3,
tuple(
np.concatenate(
self.executor.execute_tensor(
(arr * 2).prod(axis=0)))))
raw = sps.random(10, 20, density=.1)
arr = tensor(raw, chunk_size=3)
res = self.executor.execute_tensor(arr.sum())[0]
self.assertAlmostEqual(res, raw.sum())
# test order
raw = np.asfortranarray(np.random.rand(10, 20, 30))
arr = tensor(raw, chunk_size=13)
arr2 = arr.sum(axis=-1)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.sum(axis=-1)
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testMaxMinExecution(self):
raw = np.random.randint(10000, size=(10, 10, 10))
arr = tensor(raw, chunk_size=3)
self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max()))
self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min()))
np.testing.assert_array_equal(
raw.max(axis=0),
self.executor.execute_tensor(arr.max(axis=0), concat=True)[0])
np.testing.assert_array_equal(
raw.min(axis=0),
self.executor.execute_tensor(arr.min(axis=0), concat=True)[0])
np.testing.assert_array_equal(
raw.max(axis=(1, 2)),
self.executor.execute_tensor(arr.max(axis=(1, 2)), concat=True)[0])
np.testing.assert_array_equal(
raw.min(axis=(1, 2)),
self.executor.execute_tensor(arr.min(axis=(1, 2)), concat=True)[0])
raw = sps.random(10, 10, density=.5)
arr = tensor(raw, chunk_size=3)
self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max()))
self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min()))
def testAllAnyExecution(self):
raw1 = np.zeros((10, 15))
raw2 = np.ones((10, 15))
raw3 = np.array([[True, False, True, False], [True, True, True, True],
[False, False, False, False],
[False, True, False, True]])
arr1 = tensor(raw1, chunk_size=3)
arr2 = tensor(raw2, chunk_size=3)
arr3 = tensor(raw3, chunk_size=4)
self.assertFalse(self.executor.execute_tensor(arr1.all())[0])
self.assertTrue(self.executor.execute_tensor(arr2.all())[0])
self.assertFalse(self.executor.execute_tensor(arr1.any())[0])
self.assertTrue(self.executor.execute_tensor(arr1.any()))
np.testing.assert_array_equal(
raw3.all(axis=1),
self.executor.execute_tensor(arr3.all(axis=1))[0])
np.testing.assert_array_equal(
raw3.any(axis=0),
self.executor.execute_tensor(arr3.any(axis=0))[0])
raw = sps.random(10, 10, density=.5) > .5
arr = tensor(raw, chunk_size=3)
self.assertEqual(raw.A.all(),
self.executor.execute_tensor(arr.all())[0])
self.assertEqual(raw.A.any(),
self.executor.execute_tensor(arr.any())[0])
def testMeanExecution(self):
raw1 = np.random.random((20, 25))
raw2 = np.random.randint(10, size=(20, 25))
arr1 = tensor(raw1, chunk_size=3)
res1 = self.executor.execute_tensor(arr1.mean())
expected1 = raw1.mean()
self.assertTrue(np.allclose(res1[0], expected1))
res2 = self.executor.execute_tensor(arr1.mean(axis=0))
expected2 = raw1.mean(axis=0)
self.assertTrue(np.allclose(np.concatenate(res2), expected2))
res3 = self.executor.execute_tensor(arr1.mean(axis=1, keepdims=True))
expected3 = raw1.mean(axis=1, keepdims=True)
self.assertTrue(np.allclose(np.concatenate(res3), expected3))
arr2 = tensor(raw2, chunk_size=3)
res1 = self.executor.execute_tensor(arr2.mean())
expected1 = raw2.mean()
self.assertEqual(res1[0], expected1)
res2 = self.executor.execute_tensor(arr2.mean(axis=0))
expected2 = raw2.mean(axis=0)
self.assertTrue(np.allclose(np.concatenate(res2), expected2))
res3 = self.executor.execute_tensor(arr2.mean(axis=1, keepdims=True))
expected3 = raw2.mean(axis=1, keepdims=True)
self.assertTrue(np.allclose(np.concatenate(res3), expected3))
raw1 = sps.random(20, 25, density=.1)
arr1 = tensor(raw1, chunk_size=3)
res1 = self.executor.execute_tensor(arr1.mean())
expected1 = raw1.mean()
self.assertTrue(np.allclose(res1[0], expected1))
arr2 = tensor(raw1, chunk_size=30)
res1 = self.executor.execute_tensor(arr2.mean())
expected1 = raw1.mean()
self.assertTrue(np.allclose(res1[0], expected1))
arr = mean(1)
self.assertEqual(self.executor.execute_tensor(arr)[0], 1)
def testVarExecution(self):
raw1 = np.random.random((20, 25))
raw2 = np.random.randint(10, size=(20, 25))
arr1 = tensor(raw1, chunk_size=3)
res1 = self.executor.execute_tensor(arr1.var())
expected1 = raw1.var()
self.assertTrue(np.allclose(res1[0], expected1))
res2 = self.executor.execute_tensor(arr1.var(axis=0))
expected2 = raw1.var(axis=0)
self.assertTrue(np.allclose(np.concatenate(res2), expected2))
res3 = self.executor.execute_tensor(arr1.var(axis=1, keepdims=True))
expected3 = raw1.var(axis=1, keepdims=True)
self.assertTrue(np.allclose(np.concatenate(res3), expected3))
arr2 = tensor(raw2, chunk_size=3)
res1 = self.executor.execute_tensor(arr2.var())
expected1 = raw2.var()
self.assertAlmostEqual(res1[0], expected1)
res2 = self.executor.execute_tensor(arr2.var(axis=0))
expected2 = raw2.var(axis=0)
self.assertTrue(np.allclose(np.concatenate(res2), expected2))
res3 = self.executor.execute_tensor(arr2.var(axis=1, keepdims=True))
expected3 = raw2.var(axis=1, keepdims=True)
self.assertTrue(np.allclose(np.concatenate(res3), expected3))
res4 = self.executor.execute_tensor(arr2.var(ddof=1))
expected4 = raw2.var(ddof=1)
self.assertAlmostEqual(res4[0], expected4)
raw1 = sps.random(20, 25, density=.1)
arr1 = tensor(raw1, chunk_size=3)
res1 = self.executor.execute_tensor(arr1.var())
expected1 = raw1.toarray().var()
self.assertTrue(np.allclose(res1[0], expected1))
arr2 = tensor(raw1, chunk_size=30)
res1 = self.executor.execute_tensor(arr2.var())
expected1 = raw1.toarray().var()
self.assertTrue(np.allclose(res1[0], expected1))
arr = var(1)
self.assertEqual(self.executor.execute_tensor(arr)[0], 0)
def testStdExecution(self):
raw1 = np.random.random((20, 25))
raw2 = np.random.randint(10, size=(20, 25))
arr1 = tensor(raw1, chunk_size=3)
res1 = self.executor.execute_tensor(arr1.std())
expected1 = raw1.std()
self.assertTrue(np.allclose(res1[0], expected1))
res2 = self.executor.execute_tensor(arr1.std(axis=0))
expected2 = raw1.std(axis=0)
self.assertTrue(np.allclose(np.concatenate(res2), expected2))
res3 = self.executor.execute_tensor(arr1.std(axis=1, keepdims=True))
expected3 = raw1.std(axis=1, keepdims=True)
self.assertTrue(np.allclose(np.concatenate(res3), expected3))
arr2 = tensor(raw2, chunk_size=3)
res1 = self.executor.execute_tensor(arr2.std())
expected1 = raw2.std()
self.assertAlmostEqual(res1[0], expected1)
res2 = self.executor.execute_tensor(arr2.std(axis=0))
expected2 = raw2.std(axis=0)
self.assertTrue(np.allclose(np.concatenate(res2), expected2))
res3 = self.executor.execute_tensor(arr2.std(axis=1, keepdims=True))
expected3 = raw2.std(axis=1, keepdims=True)
self.assertTrue(np.allclose(np.concatenate(res3), expected3))
res4 = self.executor.execute_tensor(arr2.std(ddof=1))
expected4 = raw2.std(ddof=1)
self.assertAlmostEqual(res4[0], expected4)
raw1 = sps.random(20, 25, density=.1)
arr1 = tensor(raw1, chunk_size=3)
res1 = self.executor.execute_tensor(arr1.std())
expected1 = raw1.toarray().std()
self.assertTrue(np.allclose(res1[0], expected1))
arr2 = tensor(raw1, chunk_size=30)
res1 = self.executor.execute_tensor(arr2.std())
expected1 = raw1.toarray().std()
self.assertTrue(np.allclose(res1[0], expected1))
arr = std(1)
self.assertEqual(self.executor.execute_tensor(arr)[0], 0)
def testArgReduction(self):
raw = np.random.random((20, 20, 20))
arr = tensor(raw, chunk_size=3)
self.assertEqual(raw.argmax(),
self.executor.execute_tensor(arr.argmax())[0])
self.assertEqual(raw.argmin(),
self.executor.execute_tensor(arr.argmin())[0])
np.testing.assert_array_equal(
raw.argmax(axis=0),
self.executor.execute_tensor(arr.argmax(axis=0), concat=True)[0])
np.testing.assert_array_equal(
raw.argmin(axis=0),
self.executor.execute_tensor(arr.argmin(axis=0), concat=True)[0])
raw_format = sps.random(20, 20, density=.1, format='lil')
random_min = np.random.randint(0, 200)
random_max = np.random.randint(200, 400)
raw_format[np.unravel_index(random_min, raw_format.shape)] = -1
raw_format[np.unravel_index(random_max, raw_format.shape)] = 2
raw = raw_format.tocoo()
arr = tensor(raw, chunk_size=3)
self.assertEqual(raw.argmax(),
self.executor.execute_tensor(arr.argmax())[0])
self.assertEqual(raw.argmin(),
self.executor.execute_tensor(arr.argmin())[0])
# test order
raw = np.asfortranarray(np.random.rand(10, 20, 30))
arr = tensor(raw, chunk_size=13)
arr2 = arr.argmax(axis=-1)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.argmax(axis=-1)
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testNanReduction(self):
raw = np.random.choice(a=[0, 1, np.nan],
size=(10, 10),
p=[0.3, 0.4, 0.3])
arr = tensor(raw, chunk_size=3)
self.assertEqual(np.nansum(raw),
self.executor.execute_tensor(nansum(arr))[0])
self.assertEqual(np.nanprod(raw),
self.executor.execute_tensor(nanprod(arr))[0])
self.assertEqual(np.nanmax(raw),
self.executor.execute_tensor(nanmax(arr))[0])
self.assertEqual(np.nanmin(raw),
self.executor.execute_tensor(nanmin(arr))[0])
self.assertEqual(np.nanmean(raw),
self.executor.execute_tensor(nanmean(arr))[0])
self.assertAlmostEqual(np.nanvar(raw),
self.executor.execute_tensor(nanvar(arr))[0])
self.assertAlmostEqual(
np.nanvar(raw, ddof=1),
self.executor.execute_tensor(nanvar(arr, ddof=1))[0])
self.assertAlmostEqual(np.nanstd(raw),
self.executor.execute_tensor(nanstd(arr))[0])
self.assertAlmostEqual(
np.nanstd(raw, ddof=1),
self.executor.execute_tensor(nanstd(arr, ddof=1))[0])
arr = tensor(raw, chunk_size=10)
self.assertEqual(np.nansum(raw),
self.executor.execute_tensor(nansum(arr))[0])
self.assertEqual(np.nanprod(raw),
self.executor.execute_tensor(nanprod(arr))[0])
self.assertEqual(np.nanmax(raw),
self.executor.execute_tensor(nanmax(arr))[0])
self.assertEqual(np.nanmin(raw),
self.executor.execute_tensor(nanmin(arr))[0])
self.assertEqual(np.nanmean(raw),
self.executor.execute_tensor(nanmean(arr))[0])
self.assertAlmostEqual(np.nanvar(raw),
self.executor.execute_tensor(nanvar(arr))[0])
self.assertAlmostEqual(
np.nanvar(raw, ddof=1),
self.executor.execute_tensor(nanvar(arr, ddof=1))[0])
self.assertAlmostEqual(np.nanstd(raw),
self.executor.execute_tensor(nanstd(arr))[0])
self.assertAlmostEqual(
np.nanstd(raw, ddof=1),
self.executor.execute_tensor(nanstd(arr, ddof=1))[0])
raw = np.random.random((10, 10))
raw[:3, :3] = np.nan
arr = tensor(raw, chunk_size=3)
self.assertEqual(np.nanargmin(raw),
self.executor.execute_tensor(nanargmin(arr))[0])
self.assertEqual(np.nanargmax(raw),
self.executor.execute_tensor(nanargmax(arr))[0])
raw = np.full((10, 10), np.nan)
arr = tensor(raw, chunk_size=3)
self.assertEqual(0, self.executor.execute_tensor(nansum(arr))[0])
self.assertEqual(1, self.executor.execute_tensor(nanprod(arr))[0])
self.assertTrue(np.isnan(self.executor.execute_tensor(nanmax(arr))[0]))
self.assertTrue(np.isnan(self.executor.execute_tensor(nanmin(arr))[0]))
self.assertTrue(np.isnan(
self.executor.execute_tensor(nanmean(arr))[0]))
with self.assertRaises(ValueError):
_ = self.executor.execute_tensor(nanargmin(arr))[0] # noqa: F841
with self.assertRaises(ValueError):
_ = self.executor.execute_tensor(nanargmax(arr))[0] # noqa: F841
raw = sps.random(10, 10, density=.1, format='csr')
raw[:3, :3] = np.nan
arr = tensor(raw, chunk_size=3)
self.assertAlmostEqual(np.nansum(raw.A),
self.executor.execute_tensor(nansum(arr))[0])
self.assertAlmostEqual(np.nanprod(raw.A),
self.executor.execute_tensor(nanprod(arr))[0])
self.assertAlmostEqual(np.nanmax(raw.A),
self.executor.execute_tensor(nanmax(arr))[0])
self.assertAlmostEqual(np.nanmin(raw.A),
self.executor.execute_tensor(nanmin(arr))[0])
self.assertAlmostEqual(np.nanmean(raw.A),
self.executor.execute_tensor(nanmean(arr))[0])
self.assertAlmostEqual(np.nanvar(raw.A),
self.executor.execute_tensor(nanvar(arr))[0])
self.assertAlmostEqual(
np.nanvar(raw.A, ddof=1),
self.executor.execute_tensor(nanvar(arr, ddof=1))[0])
self.assertAlmostEqual(np.nanstd(raw.A),
self.executor.execute_tensor(nanstd(arr))[0])
self.assertAlmostEqual(
np.nanstd(raw.A, ddof=1),
self.executor.execute_tensor(nanstd(arr, ddof=1))[0])
arr = nansum(1)
self.assertEqual(self.executor.execute_tensor(arr)[0], 1)
def testCumReduction(self):
raw = np.random.randint(5, size=(8, 8, 8))
arr = tensor(raw, chunk_size=3)
res1 = self.executor.execute_tensor(arr.cumsum(axis=1), concat=True)
res2 = self.executor.execute_tensor(arr.cumprod(axis=1), concat=True)
expected1 = raw.cumsum(axis=1)
expected2 = raw.cumprod(axis=1)
np.testing.assert_array_equal(res1[0], expected1)
np.testing.assert_array_equal(res2[0], expected2)
raw = sps.random(8, 8, density=.1)
arr = tensor(raw, chunk_size=3)
res1 = self.executor.execute_tensor(arr.cumsum(axis=1), concat=True)
res2 = self.executor.execute_tensor(arr.cumprod(axis=1), concat=True)
expected1 = raw.A.cumsum(axis=1)
expected2 = raw.A.cumprod(axis=1)
self.assertTrue(np.allclose(res1[0], expected1))
self.assertTrue(np.allclose(res2[0], expected2))
# test order
raw = np.asfortranarray(np.random.rand(10, 20, 30))
arr = tensor(raw, chunk_size=13)
arr2 = arr.cumsum(axis=-1)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.cumsum(axis=-1)
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
def testNanCumReduction(self):
raw = np.random.randint(5, size=(8, 8, 8))
raw[:2, 2:4, 4:6] = np.nan
arr = tensor(raw, chunk_size=3)
res1 = self.executor.execute_tensor(nancumsum(arr, axis=1),
concat=True)
res2 = self.executor.execute_tensor(nancumprod(arr, axis=1),
concat=True)
expected1 = np.nancumsum(raw, axis=1)
expected2 = np.nancumprod(raw, axis=1)
np.testing.assert_array_equal(res1[0], expected1)
np.testing.assert_array_equal(res2[0], expected2)
raw = sps.random(8, 8, density=.1, format='lil')
raw[:2, 2:4] = np.nan
arr = tensor(raw, chunk_size=3)
res1 = self.executor.execute_tensor(nancumsum(arr, axis=1),
concat=True)[0]
res2 = self.executor.execute_tensor(nancumprod(arr, axis=1),
concat=True)[0]
expected1 = np.nancumsum(raw.A, axis=1)
expected2 = np.nancumprod(raw.A, axis=1)
self.assertTrue(np.allclose(res1, expected1))
self.assertTrue(np.allclose(res2, expected2))
def testOutReductionExecution(self):
raw = np.random.randint(5, size=(8, 8, 8))
arr = tensor(raw, chunk_size=3)
arr2 = ones((8, 8), dtype='i8', chunk_size=3)
arr.sum(axis=1, out=arr2)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.sum(axis=1)
np.testing.assert_array_equal(res, expected)
def testOutCumReductionExecution(self):
raw = np.random.randint(5, size=(8, 8, 8))
arr = tensor(raw, chunk_size=3)
arr.cumsum(axis=0, out=arr)
res = self.executor.execute_tensor(arr, concat=True)[0]
expected = raw.cumsum(axis=0)
np.testing.assert_array_equal(res, expected)
def testCountNonzeroExecution(self):
raw = [[0, 1, 7, 0, 0], [3, 0, 0, 2, 19]]
arr = tensor(raw, chunk_size=2)
t = count_nonzero(arr)
res = self.executor.execute_tensor(t)[0]
expected = np.count_nonzero(raw)
np.testing.assert_equal(res, expected)
t = count_nonzero(arr, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.count_nonzero(raw, axis=0)
np.testing.assert_equal(res, expected)
t = count_nonzero(arr, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.count_nonzero(raw, axis=1)
np.testing.assert_equal(res, expected)
raw = sps.csr_matrix(raw)
arr = tensor(raw, chunk_size=2)
t = count_nonzero(arr)
res = self.executor.execute_tensor(t)[0]
expected = np.count_nonzero(raw.A)
np.testing.assert_equal(res, expected)
t = count_nonzero(arr, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.count_nonzero(raw.A, axis=0)
np.testing.assert_equal(res, expected)
t = count_nonzero(arr, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.count_nonzero(raw.A, axis=1)
np.testing.assert_equal(res, expected)
def testAllcloseExecution(self):
a = tensor([1e10, 1e-7], chunk_size=1)
b = tensor([1.00001e10, 1e-8], chunk_size=1)
t = allclose(a, b)
res = self.executor.execute_tensor(t)[0]
self.assertFalse(res)
a = tensor([1e10, 1e-8], chunk_size=1)
b = tensor([1.00001e10, 1e-9], chunk_size=1)
t = allclose(a, b)
res = self.executor.execute_tensor(t)[0]
self.assertTrue(res)
a = tensor([1.0, np.nan], chunk_size=1)
b = tensor([1.0, np.nan], chunk_size=1)
t = allclose(a, b, equal_nan=True)
res = self.executor.execute_tensor(t)[0]
self.assertTrue(res)
a = tensor(sps.csr_matrix([[1e10, 1e-7], [0, 0]]), chunk_size=1)
b = tensor(sps.csr_matrix([[1.00001e10, 1e-8], [0, 0]]), chunk_size=1)
t = allclose(a, b)
res = self.executor.execute_tensor(t)[0]
self.assertFalse(res)
def testArrayEqual(self):
a = ones((10, 5), chunk_size=1)
b = ones((10, 5), chunk_size=2)
c = array_equal(a, b)
res = bool(self.executor.execute_tensor(c)[0])
self.assertTrue(res)
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testQRExecution(self):
data = np.random.randn(18, 6)
a = tensor(data, chunk_size=(3, 6))
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(9, 6))
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=3)
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
# test for Short-and-Fat QR
data = np.random.randn(6, 18)
a = tensor(data, chunk_size=(6, 9))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(3, 3))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(6, 3))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
def testSVDExecution(self):
data = np.random.randn(18, 6) + 1j * np.random.randn(18, 6)
a = tensor(data, chunk_size=(9, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(18, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(2, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
data = np.random.randn(6, 18) + 1j * np.random.randn(6, 18)
a = tensor(data)
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
# test for matrix of ones
data = np.ones((20, 10))
a = tensor(data, chunk_size=10)
s = svd(a)[1]
res = self.executor.execute_tensor(s, concat=True)[0]
expected = np.linalg.svd(a)[1]
np.testing.assert_array_almost_equal(res, expected)
def testCholeskyExecution(self):
data = np.array([[1, -2j], [2j, 5]])
a = tensor(data, chunk_size=1)
L = cholesky(a, lower=True)
t = L.dot(L.T.conj())
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=2)
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(1, 2))
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
def testLUExecution(self):
np.random.seed(1)
data = np.random.randint(1, 10, (6, 6))
a = tensor(data)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
# test for sparse
data = sps.csr_matrix([[2, 0, 0, 0, 5, 2],
[0, 6, 1, 0, 0, 6],
[8, 0, 9, 0, 0, 2],
[0, 6, 0, 8, 7, 3],
[7, 0, 6, 1, 7, 0],
[0, 0, 0, 7, 0, 8]])
a = tensor(data)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
def testSolveTriangular(self):
from mars.tensor import tril, triu
np.random.seed(1)
data1 = np.random.randint(1, 10, (20, 20))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=20)
b = tensor(data2, chunk_size=20)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=10)
b = tensor(data2, chunk_size=10)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
data1 = np.random.randint(1, 10, (10, 10))
data2 = np.random.randint(1, 10, (10, 5))
A = tensor(data1, chunk_size=10)
b = tensor(data2, chunk_size=10)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=3)
b = tensor(data2, chunk_size=3)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
# test sparse
data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10))))
data2 = np.random.random((10,))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve_triangular(A, b)
result_x = self.executor.execute_tensor(x, concat=True)[0]
result_b = data1.dot(result_x)
self.assertIsInstance(result_x, SparseNDArray)
np.testing.assert_allclose(result_b, data2)
data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10))))
data2 = np.random.random((10, 2))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve_triangular(A, b)
result_x = self.executor.execute_tensor(x, concat=True)[0]
result_b = data1.dot(result_x)
self.assertIsInstance(result_x, SparseNDArray)
np.testing.assert_allclose(result_b, data2)
def testSolve(self):
import scipy.linalg
np.random.seed(1)
data1 = np.random.randint(1, 10, (20, 20))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 5))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 20))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
# test for not all chunks are square in matrix A
data2 = np.random.randint(1, 10, (20,))
A = tensor(data1, chunk_size=6)
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=(7, 6))
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
# test sparse
data1 = sps.csr_matrix(np.random.randint(1, 10, (20, 20)))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(data1.dot(res), data2)
data2 = np.random.randint(1, 10, (20, 5))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 20))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(res, data2)
# test for not all chunks are square in matrix A
data2 = np.random.randint(1, 10, (20,))
A = tensor(data1, chunk_size=6)
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
def testSolveSymPos(self):
import scipy.linalg
np.random.seed(1)
data = np.random.randint(1, 10, (20, 20))
data_l = np.tril(data)
data1 = data_l.dot(data_l.T)
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b, sym_pos=True)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
def testInv(self):
import scipy.linalg
np.random.seed(1)
data = np.random.randint(1, 10, (20, 20))
A = tensor(data)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
A = tensor(data, chunk_size=5)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
B = A.T.dot(A)
inv_B = inv(B)
res = self.executor.execute_tensor(inv_B, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data.T.dot(data))))
res = self.executor.execute_tensor(B.dot(inv_B), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test for not all chunks are square in matrix A
A = tensor(data, chunk_size=8)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test sparse
data = np.random.randint(1, 10, (20, 20))
sp_data = sps.csr_matrix(data)
A = tensor(sp_data, chunk_size=5)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test for not all chunks are square in matrix A
A = tensor(sp_data, chunk_size=8)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
def testNormExecution(self):
d = np.arange(9) - 4
d2 = d.reshape(3, 3)
ma = [tensor(d, chunk_size=2), tensor(d, chunk_size=9),
tensor(d2, chunk_size=(2, 3)), tensor(d2, chunk_size=3)]
for i, a in enumerate(ma):
data = d if i < 2 else d2
for ord in (None, 'nuc', np.inf, -np.inf, 0, 1, -1, 2, -2):
for axis in (0, 1, (0, 1)):
for keepdims in (True, False):
try:
expected = np.linalg.norm(data, ord=ord, axis=axis, keepdims=keepdims)
t = norm(a, ord=ord, axis=axis, keepdims=keepdims)
concat = t.ndim > 0
res = self.executor.execute_tensor(t, concat=concat)[0]
np.testing.assert_allclose(res, expected, atol=.0001)
except ValueError:
continue
m = norm(tensor(d))
expected = self.executor.execute_tensor(m)[0]
res = np.linalg.norm(d)
self.assertEqual(expected, res)
d = uniform(-0.5, 0.5, size=(500, 2), chunk_size=50)
inside = (norm(d, axis=1) < 0.5).sum().astype(float)
t = inside / 500 * 4
res = self.executor.execute_tensor(t)[0]
self.assertAlmostEqual(res, 3.14, delta=1)
def testTensordotExecution(self):
size_executor = Executor(sync_provider_type=Executor.SyncProviderType.MOCK)
a_data = np.arange(60).reshape(3, 4, 5)
a = tensor(a_data, chunk_size=2)
b_data = np.arange(24).reshape(4, 3, 2)
b = tensor(b_data, chunk_size=2)
axes = ([1, 0], [0, 1])
c = tensordot(a, b, axes=axes)
size_res = size_executor.execute_tensor(c, mock=True)
res = self.executor.execute_tensor(c)
expected = np.tensordot(a_data, b_data, axes=axes)
self.assertEqual(sum(s[0] for s in size_res), c.nbytes)
self.assertTrue(np.array_equal(res[0], expected[:2, :]))
self.assertTrue(np.array_equal(res[1], expected[2:4, :]))
self.assertTrue(np.array_equal(res[2], expected[4:, :]))
a = ones((1000, 2000), chunk_size=500)
b = ones((2000, 100), chunk_size=500)
c = dot(a, b)
res = self.executor.execute_tensor(c)
expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((10, 8), chunk_size=2)
b = ones((8, 10), chunk_size=2)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertEqual(len(res), 25)
for r in res:
self.assertTrue(np.array_equal(r, np.tile([8], [2, 2])))
a = ones((500, 500), chunk_size=500)
b = ones((500, 100), chunk_size=500)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100])))
raw_a = np.random.random((100, 200, 50))
raw_b = np.random.random((200, 10, 100))
a = tensor(raw_a, chunk_size=50)
b = tensor(raw_b, chunk_size=33)
c = tensordot(a, b, axes=((0, 1), (2, 0)))
res = self.executor.execute_tensor(c, concat=True)
expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes))
self.assertTrue(np.allclose(res[0], expected))
a = ones((1000, 2000), chunk_size=500)
b = ones((100, 2000), chunk_size=500)
c = inner(a, b)
res = self.executor.execute_tensor(c)
expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((100, 100), chunk_size=30)
b = ones((100, 100), chunk_size=30)
c = a.dot(b)
res = self.executor.execute_tensor(c, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def testSparseDotExecution(self):
size_executor = Executor(sync_provider_type=Executor.SyncProviderType.MOCK)
a_data = sps.random(5, 9, density=.1)
b_data = sps.random(9, 10, density=.2)
a = tensor(a_data, chunk_size=2)
b = tensor(b_data, chunk_size=3)
c = dot(a, b)
size_res = size_executor.execute_tensor(c, mock=True)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertEqual(sum(s[0] for s in size_res), 0)
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c2 = dot(a, b, sparse=False)
res = self.executor.execute_tensor(c2, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c3 = tensordot(a, b.T, (-1, -1), sparse=False)
res = self.executor.execute_tensor(c3, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c = inner(a, b.T)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c = inner(a, b.T, sparse=False)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
# test vector inner
a_data = np.random.rand(5)
b_data = np.random.rand(5)
a = tensor(a_data, chunk_size=2).tosparse()
b = tensor(b_data, chunk_size=2).tosparse()
c = inner(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(np.isscalar(res))
np.testing.assert_allclose(res, np.inner(a_data, b_data))
def testVdotExecution(self):
a_data = np.array([1 + 2j, 3 + 4j])
b_data = np.array([5 + 6j, 7 + 8j])
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=1)
t = vdot(a, b)
res = self.executor.execute_tensor(t)[0]
expected = np.vdot(a_data, b_data)
np.testing.assert_equal(res, expected)
a_data = np.array([[1, 4], [5, 6]])
b_data = np.array([[4, 1], [2, 2]])
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=1)
t = vdot(a, b)
res = self.executor.execute_tensor(t)[0]
expected = np.vdot(a_data, b_data)
np.testing.assert_equal(res, expected)
def testMatmulExecution(self):
data_a = np.random.randn(10, 20)
data_b = np.random.randn(20)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b)
np.testing.assert_allclose(res, expected)
data_a = np.random.randn(10, 20)
data_b = np.random.randn(10)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(b, a)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_b, data_a)
np.testing.assert_allclose(res, expected)
data_a = np.random.randn(15, 1, 20, 30)
data_b = np.random.randn(1, 11, 30, 20)
a = tensor(data_a, chunk_size=12)
b = tensor(data_b, chunk_size=13)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b)
np.testing.assert_allclose(res, expected, atol=.0001)
a = arange(2 * 2 * 4, chunk_size=1).reshape((2, 2, 4))
b = arange(2 * 2 * 4, chunk_size=1).reshape((2, 4, 2))
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(np.arange(2 * 2 * 4).reshape(2, 2, 4),
np.arange(2 * 2 * 4).reshape(2, 4, 2))
np.testing.assert_allclose(res, expected, atol=.0001)
data_a = sps.random(10, 20)
data_b = sps.random(20, 5)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a.toarray(), data_b.toarray())
np.testing.assert_allclose(res.toarray(), expected)
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor()
def testBaseExecution(self):
executor_numpy = Executor('numpy')
raw1 = np.random.randint(10, size=(10, 10, 10))
raw2 = np.random.randint(10, size=(10, 10, 10))
arr1 = tensor(raw1, chunk_size=3)
arr2 = tensor(raw2, chunk_size=3)
arr3 = arr1 + arr2 + 10
arr4 = 10 + arr1 + arr2
res3 = executor_numpy.execute_tensor(arr3, concat=True)
res3_cmp = self.executor.execute_tensor(arr4, concat=True)
self.assertTrue(np.array_equal(res3[0], res3_cmp[0]))
res5 = executor_numpy.execute_tensor((arr1 + arr1), concat=True)
res5_cmp = self.executor.execute_tensor((arr1 + arr1), concat=True)
self.assertTrue(np.array_equal(res5[0], res5_cmp[0]))
def testFuseSizeExecution(self):
executor_size = Executor()
executor_numpy = Executor()
raw1 = np.random.randint(10, size=(10, 10, 10))
arr1 = tensor(raw1, chunk_size=3)
arr2 = arr1 + 10
arr3 = arr2 * 3
arr4 = arr3 + 5
res4_size = executor_size.execute_tensor(arr4, mock=True)
res4 = executor_numpy.execute_tensor(arr4, concat=True)
res4_cmp = self.executor.execute_tensor(arr4, concat=True)
self.assertEqual(sum(s[0] for s in res4_size), arr4.nbytes)
self.assertTrue(np.array_equal(res4[0], res4_cmp[0]))
def testUnaryExecution(self):
from mars.tensor.expressions.arithmetic import UNARY_UFUNC, arccosh, invert, sin, conj
_sp_unary_ufunc = {arccosh, invert, conj}
_new_unary_ufunc = list(UNARY_UFUNC - _sp_unary_ufunc)
executor_numexpr = Executor()
def _normalize_by_sin(func1, func2, arr):
return func1(abs(sin((func2(arr)))))
for i, j in itertools.permutations(range(len(_new_unary_ufunc)), 2):
raw = np.random.random((8, 8, 8))
arr1 = tensor(raw, chunk_size=4)
func1 = _new_unary_ufunc[i]
func2 = _new_unary_ufunc[j]
arr2 = _normalize_by_sin(func1, func2, arr1)
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
np.testing.assert_allclose(res[0], res_cmp[0])
raw = np.random.randint(100, size=(8, 8, 8))
arr1 = tensor(raw, chunk_size=4)
arr2 = arccosh(1 + abs(invert(arr1)))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testBinExecution(self):
from mars.tensor.expressions.arithmetic import BIN_UFUNC, mod, fmod, \
bitand, bitor, bitxor, lshift, rshift, ldexp
_sp_bin_ufunc = [mod, fmod, bitand, bitor, bitxor, lshift, rshift]
_new_bin_ufunc = list(BIN_UFUNC - set(_sp_bin_ufunc) - {ldexp})
executor_numexpr = Executor()
for i, j in itertools.permutations(range(len(_new_bin_ufunc)), 2):
raw = np.random.random((9, 9, 9))
arr1 = tensor(raw, chunk_size=5)
func1 = _new_bin_ufunc[i]
func2 = _new_bin_ufunc[j]
arr2 = func1(1, func2(2, arr1))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
for i, j in itertools.permutations(range(len(_sp_bin_ufunc)), 2):
raw = np.random.randint(1, 100, size=(10, 10, 10))
arr1 = tensor(raw, chunk_size=3)
func1 = _sp_bin_ufunc[i]
func2 = _sp_bin_ufunc[j]
arr2 = func1(10, func2(arr1, 5))
res = executor_numexpr.execute_tensor(arr2, concat=True)
res_cmp = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.allclose(res[0], res_cmp[0]))
def testReductionExecution(self):
raw1 = np.random.randint(5, size=(8, 8, 8))
raw2 = np.random.randint(5, size=(8, 8, 8))
arr1 = tensor(raw1, chunk_size=3)
arr2 = tensor(raw2, chunk_size=3)
res1 = self.executor.execute_tensor((arr1 + 1).sum(keepdims=True))
res2 = self.executor.execute_tensor((arr1 + 1).prod(keepdims=True))
self.assertTrue(np.array_equal((raw1 + 1).sum(keepdims=True), res1[0]))
self.assertTrue(np.array_equal((raw1 + 1).prod(keepdims=True),
res2[0]))
res1 = self.executor.execute_tensor((arr1 + 1).sum(axis=1),
concat=True)
res2 = self.executor.execute_tensor((arr1 + 1).prod(axis=1),
concat=True)
res3 = self.executor.execute_tensor((arr1 + 1).max(axis=1),
concat=True)
res4 = self.executor.execute_tensor((arr1 + 1).min(axis=1),
concat=True)
self.assertTrue(np.array_equal((raw1 + 1).sum(axis=1), res1[0]))
self.assertTrue(np.array_equal((raw1 + 1).prod(axis=1), res2[0]))
self.assertTrue(np.array_equal((raw1 + 1).max(axis=1), res3[0]))
self.assertTrue(np.array_equal((raw1 + 1).min(axis=1), res4[0]))
raw3 = raw2 - raw1 + 10
arr3 = -arr1 + arr2 + 10
res1 = self.executor.execute_tensor(arr3.sum(axis=(0, 1)), concat=True)
res2 = self.executor.execute_tensor(arr3.prod(axis=(0, 1)),
concat=True)
res3 = self.executor.execute_tensor(arr3.max(axis=(0, 1)), concat=True)
res4 = self.executor.execute_tensor(arr3.min(axis=(0, 1)), concat=True)
self.assertTrue(np.array_equal(raw3.sum(axis=(0, 1)), res1[0]))
self.assertTrue(np.array_equal(raw3.prod(axis=(0, 1)), res2[0]))
self.assertTrue(np.array_equal(raw3.max(axis=(0, 1)), res3[0]))
self.assertTrue(np.array_equal(raw3.min(axis=(0, 1)), res4[0]))
def testBoolReductionExecution(self):
raw = np.random.randint(5, size=(8, 8, 8))
arr = tensor(raw, chunk_size=2)
res = self.executor.execute_tensor((arr > 3).sum(axis=1), concat=True)
np.testing.assert_array_equal(res[0], (raw > 3).sum(axis=1))
res = self.executor.execute_tensor((arr > 3).sum())
np.testing.assert_array_equal(res, (raw > 3).sum())
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testQRExecution(self):
data = np.random.randn(18, 6)
a = tensor(data, chunk_size=(3, 6))
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(9, 6))
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=3)
q, r = qr(a)
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
# test for Short-and-Fat QR
data = np.random.randn(6, 18)
a = tensor(data, chunk_size=(6, 9))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(3, 3))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(6, 3))
q, r = qr(a, method='sfqr')
t = q.dot(r)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
def testSVDExecution(self):
data = np.random.randn(18, 6) + 1j * np.random.randn(18, 6)
a = tensor(data, chunk_size=(9, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(18, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
a = tensor(data, chunk_size=(2, 6))
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
data = np.random.randn(6, 18) + 1j * np.random.randn(6, 18)
a = tensor(data)
U, s, V = svd(a)
t = U.dot(diag(s).dot(V))
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, data))
# test for matrix of ones
data = np.ones((20, 10))
a = tensor(data, chunk_size=10)
s = svd(a)[1]
res = self.executor.execute_tensor(s, concat=True)[0]
expected = np.linalg.svd(a)[1]
np.testing.assert_array_almost_equal(res, expected)
def testRandomizedSVDExecution(self):
n_samples = 100
n_features = 500
rank = 5
k = 10
for dtype in (np.int32, np.int64, np.float32, np.float64):
# generate a matrix X of approximate effective rank `rank` and no noise
# component (very structured signal):
X = make_low_rank_matrix(n_samples=n_samples,
n_features=n_features,
effective_rank=rank,
tail_strength=0.0,
random_state=0).astype(dtype, copy=False)
self.assertEqual(X.shape, (n_samples, n_features))
dtype = np.dtype(dtype)
decimal = 5 if dtype == np.float32 else 7
# compute the singular values of X using the slow exact method
X_res = self.executor.execute_tensor(X, concat=True)[0]
U, s, V = np.linalg.svd(X_res, full_matrices=False)
# Convert the singular values to the specific dtype
U = U.astype(dtype, copy=False)
s = s.astype(dtype, copy=False)
V = V.astype(dtype, copy=False)
for normalizer in ['auto', 'LU',
'QR']: # 'none' would not be stable
# compute the singular values of X using the fast approximate method
Ua, sa, Va = randomized_svd(
X,
k,
power_iteration_normalizer=normalizer,
random_state=0)
# If the input dtype is float, then the output dtype is float of the
# same bit size (f32 is not upcast to f64)
# But if the input dtype is int, the output dtype is float64
if dtype.kind == 'f':
self.assertEqual(Ua.dtype, dtype)
self.assertEqual(sa.dtype, dtype)
self.assertEqual(Va.dtype, dtype)
else:
self.assertEqual(Ua.dtype, np.float64)
self.assertEqual(sa.dtype, np.float64)
self.assertEqual(Va.dtype, np.float64)
self.assertEqual(Ua.shape, (n_samples, k))
self.assertEqual(sa.shape, (k, ))
self.assertEqual(Va.shape, (k, n_features))
# ensure that the singular values of both methods are equal up to the
# real rank of the matrix
sa_res = self.executor.execute_tensor(sa, concat=True)[0]
np.testing.assert_almost_equal(s[:k], sa_res, decimal=decimal)
# check the singular vectors too (while not checking the sign)
dot_res = self.executor.execute_tensor(dot(Ua, Va),
concat=True)[0]
np.testing.assert_almost_equal(np.dot(U[:, :k], V[:k, :]),
dot_res,
decimal=decimal)
def testCholeskyExecution(self):
data = np.random.randint(1, 10, (10, 10))
symmetric_data = data.dot(data.T)
a = tensor(symmetric_data, chunk_size=5)
U = cholesky(a)
t = U.T.dot(U)
res_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.triu(res_u), res_u)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, symmetric_data))
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.allclose(res, symmetric_data))
a = tensor(symmetric_data, chunk_size=2)
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, symmetric_data)
a = tensor(symmetric_data, chunk_size=(1, 2))
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, symmetric_data)
a = tensor(symmetric_data, chunk_size=4)
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.triu(res_u), res_u)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, symmetric_data)
a = tensor(symmetric_data, chunk_size=3)
L = cholesky(a, lower=True)
U = cholesky(a)
t = L.dot(U)
res_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.triu(res_u), res_u)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, symmetric_data)
def testLUExecution(self):
np.random.seed(1)
# square matrix
data = np.random.randint(1, 10, (6, 6))
a = tensor(data)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
# shape[0] > shape[1]
data = np.random.randint(1, 10, (10, 6))
a = tensor(data)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l, result_u = self.executor.execute_tensors([L, U])
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
# shape[0] < shape[1]
data = np.random.randint(1, 10, (6, 10))
a = tensor(data)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
# check lower and upper triangular matrix
result_l, result_u = self.executor.execute_tensors([L, U])
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data)
# test for sparse
data = sps.csr_matrix([[2, 0, 0, 0, 5, 2], [0, 6, 1, 0, 0, 6],
[8, 0, 9, 0, 0, 2], [0, 6, 0, 8, 7, 3],
[7, 0, 6, 1, 7, 0], [0, 0, 0, 7, 0, 8]])
a = tensor(data)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=2)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=(2, 3))
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
a = tensor(data, chunk_size=4)
P, L, U = lu(a)
result_l = self.executor.execute_tensor(L, concat=True)[0]
result_u = self.executor.execute_tensor(U, concat=True)[0]
# check lower and upper triangular matrix
np.testing.assert_allclose(np.tril(result_l), result_l)
np.testing.assert_allclose(np.triu(result_u), result_u)
self.assertIsInstance(result_l, SparseNDArray)
self.assertIsInstance(result_u, SparseNDArray)
t = P.dot(L).dot(U)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_array_almost_equal(data.A, res)
def testSolveTriangular(self):
from mars.tensor import tril, triu
np.random.seed(1)
data1 = np.random.randint(1, 10, (20, 20))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=20)
b = tensor(data2, chunk_size=20)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=10)
b = tensor(data2, chunk_size=10)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
data1 = np.random.randint(1, 10, (10, 10))
data2 = np.random.randint(1, 10, (10, 5))
A = tensor(data1, chunk_size=10)
b = tensor(data2, chunk_size=10)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=3)
b = tensor(data2, chunk_size=3)
x = solve_triangular(A, b)
t = triu(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
x = solve_triangular(A, b, lower=True)
t = tril(A).dot(x)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, data2)
# test sparse
data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10))))
data2 = np.random.random((10, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve_triangular(A, b)
result_x = self.executor.execute_tensor(x, concat=True)[0]
result_b = data1.dot(result_x)
self.assertIsInstance(result_x, SparseNDArray)
np.testing.assert_allclose(result_b, data2)
data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10))))
data2 = np.random.random((10, 2))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve_triangular(A, b)
result_x = self.executor.execute_tensor(x, concat=True)[0]
result_b = data1.dot(result_x)
self.assertIsInstance(result_x, SparseNDArray)
np.testing.assert_allclose(result_b, data2)
def testSolve(self):
import scipy.linalg
np.random.seed(1)
data1 = np.random.randint(1, 10, (20, 20))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 5))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 20))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
# test for not all chunks are square in matrix A
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=6)
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
A = tensor(data1, chunk_size=(7, 6))
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
# test sparse
data1 = sps.csr_matrix(np.random.randint(1, 10, (20, 20)))
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(x, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(data1.dot(res), data2)
data2 = np.random.randint(1, 10, (20, 5))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(res, data2)
data2 = np.random.randint(1, 10, (20, 20))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_allclose(res, data2)
# test for not all chunks are square in matrix A
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=6)
b = tensor(data2, chunk_size=6)
x = solve(A, b)
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
def testSolveSymPos(self):
import scipy.linalg
np.random.seed(1)
data = np.random.randint(1, 10, (20, 20))
data_l = np.tril(data)
data1 = data_l.dot(data_l.T)
data2 = np.random.randint(1, 10, (20, ))
A = tensor(data1, chunk_size=5)
b = tensor(data2, chunk_size=5)
x = solve(A, b, sym_pos=True)
res = self.executor.execute_tensor(x, concat=True)[0]
np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2))
res = self.executor.execute_tensor(A.dot(x), concat=True)[0]
np.testing.assert_allclose(res, data2)
def testInv(self):
import scipy.linalg
np.random.seed(1)
data = np.random.randint(1, 10, (20, 20))
A = tensor(data)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
A = tensor(data, chunk_size=5)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
B = A.T.dot(A)
inv_B = inv(B)
res = self.executor.execute_tensor(inv_B, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data.T.dot(data))))
res = self.executor.execute_tensor(B.dot(inv_B), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test for not all chunks are square in matrix A
A = tensor(data, chunk_size=8)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test sparse
data = np.random.randint(1, 10, (20, 20))
sp_data = sps.csr_matrix(data)
A = tensor(sp_data, chunk_size=5)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
# test for not all chunks are square in matrix A
A = tensor(sp_data, chunk_size=8)
inv_A = inv(A)
res = self.executor.execute_tensor(inv_A, concat=True)[0]
self.assertIsInstance(res, SparseNDArray)
self.assertTrue(np.allclose(res, scipy.linalg.inv(data)))
res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0]
self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float)))
def testNormExecution(self):
d = np.arange(9) - 4
d2 = d.reshape(3, 3)
ma = [
tensor(d, chunk_size=2),
tensor(d, chunk_size=9),
tensor(d2, chunk_size=(2, 3)),
tensor(d2, chunk_size=3)
]
for i, a in enumerate(ma):
data = d if i < 2 else d2
for ord in (None, 'nuc', np.inf, -np.inf, 0, 1, -1, 2, -2):
for axis in (0, 1, (0, 1)):
for keepdims in (True, False):
try:
expected = np.linalg.norm(data,
ord=ord,
axis=axis,
keepdims=keepdims)
t = norm(a, ord=ord, axis=axis, keepdims=keepdims)
concat = t.ndim > 0
res = self.executor.execute_tensor(
t, concat=concat)[0]
np.testing.assert_allclose(res,
expected,
atol=.0001)
except ValueError:
continue
m = norm(tensor(d))
expected = self.executor.execute_tensor(m)[0]
res = np.linalg.norm(d)
self.assertEqual(expected, res)
d = uniform(-0.5, 0.5, size=(500, 2), chunk_size=50)
inside = (norm(d, axis=1) < 0.5).sum().astype(float)
t = inside / 500 * 4
res = self.executor.execute_tensor(t)[0]
self.assertAlmostEqual(res, 3.14, delta=1)
def testTensordotExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
a_data = np.arange(60).reshape(3, 4, 5)
a = tensor(a_data, chunk_size=2)
b_data = np.arange(24).reshape(4, 3, 2)
b = tensor(b_data, chunk_size=2)
axes = ([1, 0], [0, 1])
c = tensordot(a, b, axes=axes)
size_res = size_executor.execute_tensor(c, mock=True)
self.assertEqual(sum(s[0] for s in size_res), c.nbytes)
self.assertEqual(sum(s[1] for s in size_res), c.nbytes)
res = self.executor.execute_tensor(c)
expected = np.tensordot(a_data, b_data, axes=axes)
self.assertTrue(np.array_equal(res[0], expected[:2, :]))
self.assertTrue(np.array_equal(res[1], expected[2:4, :]))
self.assertTrue(np.array_equal(res[2], expected[4:, :]))
a = ones((1000, 2000), chunk_size=500)
b = ones((2000, 100), chunk_size=500)
c = dot(a, b)
res = self.executor.execute_tensor(c)
expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((10, 8), chunk_size=2)
b = ones((8, 10), chunk_size=2)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertEqual(len(res), 25)
for r in res:
self.assertTrue(np.array_equal(r, np.tile([8], [2, 2])))
a = ones((500, 500), chunk_size=500)
b = ones((500, 100), chunk_size=500)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100])))
raw_a = np.random.random((100, 200, 50))
raw_b = np.random.random((200, 10, 100))
a = tensor(raw_a, chunk_size=50)
b = tensor(raw_b, chunk_size=33)
c = tensordot(a, b, axes=((0, 1), (2, 0)))
res = self.executor.execute_tensor(c, concat=True)
expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes))
self.assertTrue(np.allclose(res[0], expected))
a = ones((1000, 2000), chunk_size=500)
b = ones((100, 2000), chunk_size=500)
c = inner(a, b)
res = self.executor.execute_tensor(c)
expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((100, 100), chunk_size=30)
b = ones((100, 100), chunk_size=30)
c = a.dot(b)
res = self.executor.execute_tensor(c, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def testSparseDotSizeExecution(self):
from mars.tensor.linalg.tensordot import TensorTensorDot
from mars.executor import register, register_default
chunk_sizes = dict()
chunk_nbytes = dict()
chunk_input_sizes = dict()
chunk_input_nbytes = dict()
def execute_size(t):
def _tensordot_size_recorder(ctx, op):
TensorTensorDot.estimate_size(ctx, op)
chunk_key = op.outputs[0].key
chunk_sizes[chunk_key] = ctx[chunk_key]
chunk_nbytes[chunk_key] = op.outputs[0].nbytes
input_sizes = dict(
(inp.op.key, ctx[inp.key][0]) for inp in op.inputs)
chunk_input_sizes[chunk_key] = sum(input_sizes.values())
input_nbytes = dict(
(inp.op.key, inp.nbytes) for inp in op.inputs)
chunk_input_nbytes[chunk_key] = sum(input_nbytes.values())
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
try:
chunk_sizes.clear()
chunk_nbytes.clear()
chunk_input_sizes.clear()
chunk_input_nbytes.clear()
register(TensorTensorDot,
size_estimator=_tensordot_size_recorder)
size_executor.execute_tensor(t, mock=True)
finally:
register_default(TensorTensorDot)
a_data = sps.random(5, 9, density=.1)
b_data = sps.random(9, 10, density=.2)
a = tensor(a_data, chunk_size=2)
b = tensor(b_data, chunk_size=3)
c = dot(a, b)
execute_size(c)
for key in chunk_input_sizes.keys():
self.assertGreaterEqual(chunk_sizes[key][1],
chunk_input_sizes[key])
c2 = dot(a, b, sparse=False)
execute_size(c2)
for key in chunk_input_sizes.keys():
self.assertEqual(chunk_sizes[key][0], chunk_nbytes[key])
self.assertEqual(chunk_sizes[key][1],
chunk_input_nbytes[key] + chunk_nbytes[key])
def testSparseDotExecution(self):
a_data = sps.random(5, 9, density=.1)
b_data = sps.random(9, 10, density=.2)
a = tensor(a_data, chunk_size=2)
b = tensor(b_data, chunk_size=3)
c = dot(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c2 = dot(a, b, sparse=False)
res = self.executor.execute_tensor(c2, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c3 = tensordot(a, b.T, (-1, -1), sparse=False)
res = self.executor.execute_tensor(c3, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
c = inner(a, b.T)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(issparse(res))
np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray())
c = inner(a, b.T, sparse=False)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertFalse(issparse(res))
np.testing.assert_allclose(res, a_data.dot(b_data).toarray())
# test vector inner
a_data = np.random.rand(5)
b_data = np.random.rand(5)
a = tensor(a_data, chunk_size=2).tosparse()
b = tensor(b_data, chunk_size=2).tosparse()
c = inner(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(np.isscalar(res))
np.testing.assert_allclose(res, np.inner(a_data, b_data))
def testVdotExecution(self):
a_data = np.array([1 + 2j, 3 + 4j])
b_data = np.array([5 + 6j, 7 + 8j])
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=1)
t = vdot(a, b)
res = self.executor.execute_tensor(t)[0]
expected = np.vdot(a_data, b_data)
np.testing.assert_equal(res, expected)
a_data = np.array([[1, 4], [5, 6]])
b_data = np.array([[4, 1], [2, 2]])
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=1)
t = vdot(a, b)
res = self.executor.execute_tensor(t)[0]
expected = np.vdot(a_data, b_data)
np.testing.assert_equal(res, expected)
def testMatmulExecution(self):
data_a = np.random.randn(10, 20)
data_b = np.random.randn(20)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b)
np.testing.assert_allclose(res, expected)
data_a = np.random.randn(10, 20)
data_b = np.random.randn(10)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(b, a)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_b, data_a)
np.testing.assert_allclose(res, expected)
data_a = np.random.randn(15, 1, 20, 30)
data_b = np.random.randn(1, 11, 30, 20)
a = tensor(data_a, chunk_size=12)
b = tensor(data_b, chunk_size=13)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b)
np.testing.assert_allclose(res, expected, atol=.0001)
a = arange(2 * 2 * 4, chunk_size=1).reshape((2, 2, 4))
b = arange(2 * 2 * 4, chunk_size=1).reshape((2, 4, 2))
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(
np.arange(2 * 2 * 4).reshape(2, 2, 4),
np.arange(2 * 2 * 4).reshape(2, 4, 2))
np.testing.assert_allclose(res, expected, atol=.0001)
data_a = sps.random(10, 20)
data_b = sps.random(20, 5)
a = tensor(data_a, chunk_size=2)
b = tensor(data_b, chunk_size=3)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a.toarray(), data_b.toarray())
np.testing.assert_allclose(res.toarray(), expected)
# test order
data_a = np.asfortranarray(np.random.randn(10, 20))
data_b = np.asfortranarray(np.random.randn(20, 30))
a = tensor(data_a, chunk_size=12)
b = tensor(data_b, chunk_size=13)
c = matmul(a, b)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b)
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
c = matmul(a, b, order='A')
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b, order='A')
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
c = matmul(a, b, order='C')
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.matmul(data_a, data_b, order='C')
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'],
expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'],
expected.flags['F_CONTIGUOUS'])
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def _nan_equal(self, a, b):
try:
np.testing.assert_equal(a, b)
except AssertionError:
return False
return True
def testBaseExecution(self):
arr = ones((10, 8), chunk_size=2)
arr2 = arr + 1
res = self.executor.execute_tensor(arr2)
self.assertTrue((res[0] == np.ones((2, 2)) + 1).all())
data = np.random.random((10, 8, 3))
arr = tensor(data, chunk_size=2)
arr2 = arr + 1
res = self.executor.execute_tensor(arr2)
self.assertTrue((res[0] == data[:2, :2, :2] + 1).all())
@staticmethod
def _get_func(op):
if isinstance(op, six.string_types):
return getattr(np, op)
return op
def testUfuncExecution(self):
from mars.tensor.expressions.arithmetic import UNARY_UFUNC, BIN_UFUNC, arccosh, \
invert, mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp
from mars.tensor.execution.arithmetic import OP_TO_HANDLER
_sp_unary_ufunc = {arccosh, invert}
_sp_bin_ufunc = {
mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp
}
data1 = np.random.random((5, 9, 4))
data2 = np.random.random((5, 9, 4))
rand = np.random.random()
arr1 = tensor(data1, chunk_size=3)
arr2 = tensor(data2, chunk_size=3)
_new_unary_ufunc = UNARY_UFUNC - _sp_unary_ufunc
for func in _new_unary_ufunc:
res_tensor = func(arr1)
res = self.executor.execute_tensor(res_tensor, concat=True)
expected = self._get_func(OP_TO_HANDLER[type(
res_tensor.op)])(data1)
self.assertTrue(np.allclose(res[0], expected))
_new_bin_ufunc = BIN_UFUNC - _sp_bin_ufunc
for func in _new_bin_ufunc:
res_tensor1 = func(arr1, arr2)
res_tensor2 = func(arr1, rand)
res_tensor3 = func(rand, arr1)
res1 = self.executor.execute_tensor(res_tensor1, concat=True)
res2 = self.executor.execute_tensor(res_tensor2, concat=True)
res3 = self.executor.execute_tensor(res_tensor3, concat=True)
expected1 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])(
data1, data2)
expected2 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])(
data1, rand)
expected3 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])(
rand, data1)
self.assertTrue(np.allclose(res1[0], expected1))
self.assertTrue(np.allclose(res2[0], expected2))
self.assertTrue(np.allclose(res3[0], expected3))
data1 = np.random.randint(2, 10, size=(10, 10, 10))
data2 = np.random.randint(2, 10, size=(10, 10, 10))
rand = np.random.randint(1, 10)
arr1 = tensor(data1, chunk_size=3)
arr2 = tensor(data2, chunk_size=3)
for func in _sp_unary_ufunc:
res_tensor = func(arr1)
res = self.executor.execute_tensor(res_tensor, concat=True)
expected = self._get_func(OP_TO_HANDLER[type(
res_tensor.op)])(data1)
self.assertTrue(np.allclose(res[0], expected))
for func in _sp_bin_ufunc:
res_tensor1 = func(arr1, arr2)
res_tensor2 = func(arr1, rand)
res_tensor3 = func(rand, arr1)
res1 = self.executor.execute_tensor(res_tensor1, concat=True)
res2 = self.executor.execute_tensor(res_tensor2, concat=True)
res3 = self.executor.execute_tensor(res_tensor3, concat=True)
expected1 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])(
data1, data2)
expected2 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])(
data1, rand)
expected3 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])(
rand, data1)
self.assertTrue(np.allclose(res1[0], expected1))
self.assertTrue(np.allclose(res2[0], expected2))
self.assertTrue(np.allclose(res3[0], expected3))
@staticmethod
def _get_sparse_func(op):
from mars.lib.sparse.core import issparse
if isinstance(op, six.string_types):
op = getattr(np, op)
def func(*args):
new_args = []
for arg in args:
if issparse(arg):
new_args.append(arg.toarray())
else:
new_args.append(arg)
return op(*new_args)
return func
@staticmethod
def toarray(x):
if hasattr(x, 'toarray'):
return x.toarray()
return x
def testSparseUfuncExexution(self):
from mars.tensor.expressions.arithmetic import UNARY_UFUNC, BIN_UFUNC, arccosh, \
invert, mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp
from mars.tensor.execution.arithmetic import OP_TO_HANDLER
_sp_unary_ufunc = {arccosh, invert}
_sp_bin_ufunc = {
mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp
}
data1 = sps.random(5, 9, density=.1)
data2 = sps.random(5, 9, density=.2)
rand = np.random.random()
arr1 = tensor(data1, chunk_size=3)
arr2 = tensor(data2, chunk_size=3)
_new_unary_ufunc = UNARY_UFUNC - _sp_unary_ufunc
for func in _new_unary_ufunc:
res_tensor = func(arr1)
res = self.executor.execute_tensor(res_tensor, concat=True)
expected = self._get_sparse_func(OP_TO_HANDLER[type(
res_tensor.op)])(data1)
self._nan_equal(self.toarray(res[0]), expected)
_new_bin_ufunc = BIN_UFUNC - _sp_bin_ufunc
for func in _new_bin_ufunc:
res_tensor1 = func(arr1, arr2)
res_tensor2 = func(arr1, rand)
res_tensor3 = func(rand, arr1)
res1 = self.executor.execute_tensor(res_tensor1, concat=True)
res2 = self.executor.execute_tensor(res_tensor2, concat=True)
res3 = self.executor.execute_tensor(res_tensor3, concat=True)
expected1 = self._get_sparse_func(OP_TO_HANDLER[type(
res_tensor1.op)])(data1, data2)
expected2 = self._get_sparse_func(OP_TO_HANDLER[type(
res_tensor1.op)])(data1, rand)
expected3 = self._get_sparse_func(OP_TO_HANDLER[type(
res_tensor1.op)])(rand, data1)
self._nan_equal(self.toarray(res1[0]), expected1)
self._nan_equal(self.toarray(res2[0]), expected2)
self._nan_equal(self.toarray(res3[0]), expected3)
data1 = np.random.randint(2, 10, size=(10, 10))
data2 = np.random.randint(2, 10, size=(10, 10))
rand = np.random.randint(1, 10)
arr1 = tensor(data1, chunk_size=3).tosparse()
arr2 = tensor(data2, chunk_size=3).tosparse()
for func in _sp_unary_ufunc:
res_tensor = func(arr1)
res = self.executor.execute_tensor(res_tensor, concat=True)
expected = self._get_sparse_func(OP_TO_HANDLER[type(
res_tensor.op)])(data1)
self._nan_equal(self.toarray(res[0]), expected)
for func in _sp_bin_ufunc:
res_tensor1 = func(arr1, arr2)
res_tensor2 = func(arr1, rand)
res_tensor3 = func(rand, arr1)
res1 = self.executor.execute_tensor(res_tensor1, concat=True)
res2 = self.executor.execute_tensor(res_tensor2, concat=True)
res3 = self.executor.execute_tensor(res_tensor3, concat=True)
expected1 = self._get_sparse_func(OP_TO_HANDLER[type(
res_tensor1.op)])(data1, data2)
expected2 = self._get_sparse_func(OP_TO_HANDLER[type(
res_tensor1.op)])(data1, rand)
expected3 = self._get_sparse_func(OP_TO_HANDLER[type(
res_tensor1.op)])(rand, data1)
self._nan_equal(self.toarray(res1[0]), expected1)
self._nan_equal(self.toarray(res2[0]), expected2)
self._nan_equal(self.toarray(res3[0]), expected3)
def testAddWithOutExecution(self):
data1 = np.random.random((5, 9, 4))
data2 = np.random.random((9, 4))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
add(arr1, arr2, out=arr1)
res = self.executor.execute_tensor(arr1, concat=True)[0]
self.assertTrue(np.array_equal(res, data1 + data2))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
arr3 = add(arr1, arr2, out=arr1.astype('i4'), casting='unsafe')
res = self.executor.execute_tensor(arr3, concat=True)[0]
np.testing.assert_array_equal(res, (data1 + data2).astype('i4'))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
arr3 = truediv(arr1, arr2, out=arr1, where=arr2 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
self.assertTrue(
np.array_equal(
res,
np.true_divide(data1,
data2,
out=data1.copy(),
where=data2 > .5)))
arr1 = tensor(data1.copy(), chunk_size=4)
arr2 = tensor(data2.copy(), chunk_size=4)
arr3 = add(arr1, arr2, where=arr1 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
expected = np.add(data1, data2, where=data1 > .5)
self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5]))
arr1 = tensor(data1.copy(), chunk_size=4)
arr3 = add(arr1, 1, where=arr1 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
expected = np.add(data1, 1, where=data1 > .5)
self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5]))
def testFrexpExecution(self):
data1 = np.random.random((5, 9, 4))
arr1 = tensor(data1.copy(), chunk_size=3)
o1, o2 = frexp(arr1)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.frexp(data1))
self.assertTrue(np.allclose(res, expected))
arr1 = tensor(data1.copy(), chunk_size=3)
o1 = zeros(data1.shape, chunk_size=3)
o2 = zeros(data1.shape, dtype='i8', chunk_size=3)
frexp(arr1, o1, o2)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.frexp(data1))
self.assertTrue(np.allclose(res, expected))
data1 = sps.random(5, 9, density=.1)
arr1 = tensor(data1.copy(), chunk_size=3)
o1, o2 = frexp(arr1)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.frexp(data1.toarray()))
np.testing.assert_equal(res.toarray(), expected)
def testModfExecution(self):
data1 = np.random.random((5, 9))
arr1 = tensor(data1.copy(), chunk_size=3)
o1, o2 = modf(arr1)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.modf(data1))
self.assertTrue(np.allclose(res, expected))
arr1 = tensor(data1.copy(), chunk_size=3)
o1 = zeros(data1.shape, chunk_size=3)
o2 = zeros(data1.shape, chunk_size=3)
modf(arr1, o1, o2)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.modf(data1))
self.assertTrue(np.allclose(res, expected))
data1 = sps.random(5, 9, density=.1)
arr1 = tensor(data1.copy(), chunk_size=3)
o1, o2 = modf(arr1)
o = o1 + o2
res = self.executor.execute_tensor(o, concat=True)[0]
expected = sum(np.modf(data1.toarray()))
np.testing.assert_equal(res.toarray(), expected)
def testClipExecution(self):
a_data = np.arange(10)
a = tensor(a_data.copy(), chunk_size=3)
b = clip(a, 1, 8)
res = self.executor.execute_tensor(b, concat=True)[0]
expected = np.clip(a_data, 1, 8)
self.assertTrue(np.array_equal(res, expected))
a = tensor(a_data.copy(), chunk_size=3)
clip(a, 3, 6, out=a)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.clip(a_data, 3, 6)
self.assertTrue(np.array_equal(res, expected))
with option_context() as options:
options.tensor.chunk_size = 3
a = tensor(a_data.copy(), chunk_size=3)
b = clip(a, [3, 4, 1, 1, 1, 4, 4, 4, 4, 4], 8)
res = self.executor.execute_tensor(b, concat=True)[0]
expected = np.clip(a_data, [3, 4, 1, 1, 1, 4, 4, 4, 4, 4], 8)
self.assertTrue(np.array_equal(res, expected))
# test sparse clip
a_data = sps.csr_matrix([[0, 2, 8], [0, 0, -1]])
a = tensor(a_data, chunk_size=3)
b_data = sps.csr_matrix([[0, 3, 0], [1, 0, -2]])
c = clip(a, b_data, 4)
res = self.executor.execute_tensor(c, concat=True)[0]
expected = np.clip(a_data.toarray(), b_data.toarray(), 4)
self.assertTrue(np.array_equal(res.toarray(), expected))
def testAroundExecution(self):
data = np.random.randn(10, 20)
x = tensor(data, chunk_size=3)
t = x.round(2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.around(data, decimals=2)
np.testing.assert_allclose(res, expected)
data = sps.random(10, 20, density=.2)
x = tensor(data, chunk_size=3)
t = x.round(2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.around(data.toarray(), decimals=2)
np.testing.assert_allclose(res.toarray(), expected)
def testIsCloseExecution(self):
data = np.array([1.05, 1.0, 1.01, np.nan])
data2 = np.array([1.04, 1.0, 1.03, np.nan])
x = tensor(data, chunk_size=2)
y = tensor(data2, chunk_size=3)
z = isclose(x, y, atol=.01)
res = self.executor.execute_tensor(z, concat=True)[0]
expected = np.isclose(data, data2, atol=.01)
np.testing.assert_equal(res, expected)
z = isclose(x, y, atol=.01, equal_nan=True)
res = self.executor.execute_tensor(z, concat=True)[0]
expected = np.isclose(data, data2, atol=.01, equal_nan=True)
np.testing.assert_equal(res, expected)
# test sparse
data = sps.csr_matrix(np.array([0, 1.0, 1.01, np.nan]))
data2 = sps.csr_matrix(np.array([0, 1.0, 1.03, np.nan]))
x = tensor(data, chunk_size=2)
y = tensor(data2, chunk_size=3)
z = isclose(x, y, atol=.01)
res = self.executor.execute_tensor(z, concat=True)[0]
expected = np.isclose(data.toarray(), data2.toarray(), atol=.01)
np.testing.assert_equal(res, expected)
z = isclose(x, y, atol=.01, equal_nan=True)
res = self.executor.execute_tensor(z, concat=True)[0]
expected = np.isclose(data.toarray(),
data2.toarray(),
atol=.01,
equal_nan=True)
np.testing.assert_equal(res, expected)
def testDtypeExecution(self):
a = ones((10, 20), dtype='f4', chunk_size=5)
c = truediv(a, 2, dtype='f8')
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertEqual(res.dtype, np.float64)
c = truediv(a, 0, dtype='f8')
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(np.isinf(res[0, 0]))
with self.assertRaises(FloatingPointError):
with np.errstate(divide='raise'):
c = truediv(a, 0, dtype='f8')
_ = self.executor.execute_tensor(c,
concat=True)[0] # noqa: F841
def testSetGetRealExecution(self):
a_data = np.array([1 + 2j, 3 + 4j, 5 + 6j])
a = tensor(a_data, chunk_size=2)
res = self.executor.execute_tensor(a.real, concat=True)[0]
expected = a_data.real
np.testing.assert_equal(res, expected)
a.real = 9
res = self.executor.execute_tensor(a, concat=True)[0]
expected = a_data.copy()
expected.real = 9
np.testing.assert_equal(res, expected)
a.real = np.array([9, 8, 7])
res = self.executor.execute_tensor(a, concat=True)[0]
expected = a_data.copy()
expected.real = np.array([9, 8, 7])
np.testing.assert_equal(res, expected)
# test sparse
a_data = np.array([[1 + 2j, 3 + 4j, 0], [0, 0, 0]])
a = tensor(sps.csr_matrix(a_data))
res = self.executor.execute_tensor(a.real, concat=True)[0].toarray()
expected = a_data.real
np.testing.assert_equal(res, expected)
a.real = 9
res = self.executor.execute_tensor(a, concat=True)[0].toarray()
expected = a_data.copy()
expected.real = 9
np.testing.assert_equal(res, expected)
a.real = np.array([9, 8, 7])
res = self.executor.execute_tensor(a, concat=True)[0].toarray()
expected = a_data.copy()
expected.real = np.array([9, 8, 7])
np.testing.assert_equal(res, expected)
def testSetGetImagExecution(self):
a_data = np.array([1 + 2j, 3 + 4j, 5 + 6j])
a = tensor(a_data, chunk_size=2)
res = self.executor.execute_tensor(a.imag, concat=True)[0]
expected = a_data.imag
np.testing.assert_equal(res, expected)
a.imag = 9
res = self.executor.execute_tensor(a, concat=True)[0]
expected = a_data.copy()
expected.imag = 9
np.testing.assert_equal(res, expected)
a.imag = np.array([9, 8, 7])
res = self.executor.execute_tensor(a, concat=True)[0]
expected = a_data.copy()
expected.imag = np.array([9, 8, 7])
np.testing.assert_equal(res, expected)
# test sparse
a_data = np.array([[1 + 2j, 3 + 4j, 0], [0, 0, 0]])
a = tensor(sps.csr_matrix(a_data))
res = self.executor.execute_tensor(a.imag, concat=True)[0].toarray()
expected = a_data.imag
np.testing.assert_equal(res, expected)
a.imag = 9
res = self.executor.execute_tensor(a, concat=True)[0].toarray()
expected = a_data.copy()
expected.imag = 9
np.testing.assert_equal(res, expected)
a.imag = np.array([9, 8, 7])
res = self.executor.execute_tensor(a, concat=True)[0].toarray()
expected = a_data.copy()
expected.imag = np.array([9, 8, 7])
np.testing.assert_equal(res, expected)
def testTensordotExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
a_data = np.arange(60).reshape(3, 4, 5)
a = tensor(a_data, chunk_size=2)
b_data = np.arange(24).reshape(4, 3, 2)
b = tensor(b_data, chunk_size=2)
axes = ([1, 0], [0, 1])
c = tensordot(a, b, axes=axes)
size_res = size_executor.execute_tensor(c, mock=True)
self.assertEqual(sum(s[0] for s in size_res), c.nbytes)
self.assertEqual(sum(s[1] for s in size_res), c.nbytes)
res = self.executor.execute_tensor(c)
expected = np.tensordot(a_data, b_data, axes=axes)
self.assertTrue(np.array_equal(res[0], expected[:2, :]))
self.assertTrue(np.array_equal(res[1], expected[2:4, :]))
self.assertTrue(np.array_equal(res[2], expected[4:, :]))
a = ones((1000, 2000), chunk_size=500)
b = ones((2000, 100), chunk_size=500)
c = dot(a, b)
res = self.executor.execute_tensor(c)
expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((10, 8), chunk_size=2)
b = ones((8, 10), chunk_size=2)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertEqual(len(res), 25)
for r in res:
self.assertTrue(np.array_equal(r, np.tile([8], [2, 2])))
a = ones((500, 500), chunk_size=500)
b = ones((500, 100), chunk_size=500)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100])))
raw_a = np.random.random((100, 200, 50))
raw_b = np.random.random((200, 10, 100))
a = tensor(raw_a, chunk_size=50)
b = tensor(raw_b, chunk_size=33)
c = tensordot(a, b, axes=((0, 1), (2, 0)))
res = self.executor.execute_tensor(c, concat=True)
expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes))
self.assertTrue(np.allclose(res[0], expected))
a = ones((1000, 2000), chunk_size=500)
b = ones((100, 2000), chunk_size=500)
c = inner(a, b)
res = self.executor.execute_tensor(c)
expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((100, 100), chunk_size=30)
b = ones((100, 100), chunk_size=30)
c = a.dot(b)
res = self.executor.execute_tensor(c, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testReshapeExecution(self):
x = ones((1, 2, 3), chunk_size=[4, 3, 5])
y = x.reshape(3, 2)
res = self.executor.execute_tensor(y)[0]
self.assertEqual(y.shape, (3, 2))
np.testing.assert_equal(res, np.ones((3, 2)))
data = np.random.rand(6, 4)
x2 = tensor(data, chunk_size=2)
y2 = x2.reshape(3, 8, order='F')
res = self.executor.execute_tensor(y2, concat=True)[0]
expected = data.reshape((3, 8), order='F')
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['F_CONTIGUOUS'])
self.assertFalse(res.flags['C_CONTIGUOUS'])
data2 = np.asfortranarray(np.random.rand(6, 4))
x3 = tensor(data2, chunk_size=2)
y3 = x3.reshape(3, 8)
res = self.executor.execute_tensor(y3, concat=True)[0]
expected = data2.reshape((3, 8))
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['C_CONTIGUOUS'])
self.assertFalse(res.flags['F_CONTIGUOUS'])
data2 = np.asfortranarray(np.random.rand(6, 4))
x3 = tensor(data2, chunk_size=2)
y3 = x3.reshape(3, 8, order='F')
res = self.executor.execute_tensor(y3, concat=True)[0]
expected = data2.reshape((3, 8), order='F')
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['F_CONTIGUOUS'])
self.assertFalse(res.flags['C_CONTIGUOUS'])
def testShuffleReshapeExecution(self):
a = ones((31, 27), chunk_size=10)
b = a.reshape(27, 31)
b.op.extra_params['_reshape_with_shuffle'] = True
res = self.executor.execute_tensor(b, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((27, 31)))
b2 = a.reshape(27, 31, order='F')
b.op.extra_params['_reshape_with_shuffle'] = True
res = self.executor.execute_tensor(b2)[0]
self.assertTrue(res.flags['F_CONTIGUOUS'])
self.assertFalse(res.flags['C_CONTIGUOUS'])
data = np.random.rand(6, 4)
x2 = tensor(data, chunk_size=2)
y2 = x2.reshape(4, 6, order='F')
y2.op.extra_params['_reshape_with_shuffle'] = True
res = self.executor.execute_tensor(y2, concat=True)[0]
expected = data.reshape((4, 6), order='F')
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['F_CONTIGUOUS'])
self.assertFalse(res.flags['C_CONTIGUOUS'])
data2 = np.asfortranarray(np.random.rand(6, 4))
x3 = tensor(data2, chunk_size=2)
y3 = x3.reshape(4, 6)
y3.op.extra_params['_reshape_with_shuffle'] = True
res = self.executor.execute_tensor(y3, concat=True)[0]
expected = data2.reshape((4, 6))
np.testing.assert_array_equal(res, expected)
self.assertTrue(res.flags['C_CONTIGUOUS'])
self.assertFalse(res.flags['F_CONTIGUOUS'])
class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
@unittest.skipIf(tiledb is None, 'tiledb not installed')
def testStoreTileDBExecution(self):
ctx = tiledb.Ctx()
tempdir = tempfile.mkdtemp()
try:
# store TileDB dense array
expected = np.random.rand(8, 4, 3)
a = tensor(expected, chunk_size=(3, 3, 2))
save = totiledb(tempdir, a, ctx=ctx)
self.executor.execute_tensor(save)
with tiledb.DenseArray(uri=tempdir, ctx=ctx) as arr:
np.testing.assert_allclose(expected, arr.read_direct())
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
try:
# store tensor with 1 chunk to TileDB dense array
a = arange(12)
save = totiledb(tempdir, a, ctx=ctx)
self.executor.execute_tensor(save)
with tiledb.DenseArray(uri=tempdir, ctx=ctx) as arr:
np.testing.assert_allclose(np.arange(12), arr.read_direct())
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
try:
# store 2-d TileDB sparse array
expected = sps.random(8, 7, density=0.1)
a = tensor(expected, chunk_size=(3, 5))
save = totiledb(tempdir, a, ctx=ctx)
self.executor.execute_tensor(save)
with tiledb.SparseArray(uri=tempdir, ctx=ctx) as arr:
data = arr[:, :]
coords = data['coords']
value = data[arr.attr(0).name]
ij = tuple(coords[arr.domain.dim(k).name]
for k in range(arr.ndim))
result = sps.coo_matrix((value, ij), shape=arr.shape)
np.testing.assert_allclose(expected.toarray(),
result.toarray())
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
try:
# store TileDB dense array
expected = np.asfortranarray(np.random.rand(8, 4, 3))
a = tensor(expected, chunk_size=(3, 3, 2))
save = totiledb(tempdir, a, ctx=ctx)
self.executor.execute_tensor(save)
with tiledb.DenseArray(uri=tempdir, ctx=ctx) as arr:
np.testing.assert_allclose(expected, arr.read_direct())
self.assertEqual(arr.schema.cell_order, 'col-major')
finally:
shutil.rmtree(tempdir)
def setup(self):
self.executor = Executor()
class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
def testCreateSparseExecution(self):
mat = sps.csr_matrix([[0, 0, 2], [2, 0, 0]])
t = tensor(mat, dtype='f8', chunk_size=2)
res = self.executor.execute_tensor(t)
self.assertIsInstance(res[0], SparseNDArray)
self.assertEqual(res[0].dtype, np.float64)
np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray())
np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray())
t2 = ones_like(t, dtype='f4')
res = self.executor.execute_tensor(t2)
expected = sps.csr_matrix([[0, 0, 1], [1, 0, 0]])
self.assertIsInstance(res[0], SparseNDArray)
self.assertEqual(res[0].dtype, np.float32)
np.testing.assert_array_equal(res[0].toarray(), expected[..., :2].toarray())
np.testing.assert_array_equal(res[1].toarray(), expected[..., 2:].toarray())
t3 = tensor(np.array([[0, 0, 2], [2, 0, 0]]), chunk_size=2).tosparse()
res = self.executor.execute_tensor(t3)
self.assertIsInstance(res[0], SparseNDArray)
self.assertEqual(res[0].dtype, np.int_)
np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray())
np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray())
def testZerosExecution(self):
t = zeros((20, 30), dtype='i8', chunk_size=5)
res = self.executor.execute_tensor(t, concat=True)
self.assertTrue(np.array_equal(res[0], np.zeros((20, 30), dtype='i8')))
self.assertEqual(res[0].dtype, np.int64)
t2 = zeros_like(t)
res = self.executor.execute_tensor(t2, concat=True)
self.assertTrue(np.array_equal(res[0], np.zeros((20, 30), dtype='i8')))
self.assertEqual(res[0].dtype, np.int64)
t = zeros((20, 30), dtype='i4', chunk_size=5, sparse=True)
res = self.executor.execute_tensor(t, concat=True)
self.assertEqual(res[0].nnz, 0)
def testEmptyExecution(self):
t = empty((20, 30), dtype='i8', chunk_size=5)
res = self.executor.execute_tensor(t, concat=True)
self.assertEqual(res[0].shape, (20, 30))
self.assertEqual(res[0].dtype, np.int64)
t = empty((20, 30), chunk_size=5)
res = self.executor.execute_tensor(t, concat=True)
self.assertEqual(res[0].shape, (20, 30))
self.assertEqual(res[0].dtype, np.float64)
t2 = empty_like(t)
res = self.executor.execute_tensor(t2, concat=True)
self.assertEqual(res[0].shape, (20, 30))
self.assertEqual(res[0].dtype, np.float64)
def testFullExecution(self):
t = full((2, 2), 1, dtype='f4', chunk_size=1)
res = self.executor.execute_tensor(t, concat=True)
self.assertTrue(np.array_equal(res[0], np.full((2, 2), 1, dtype='f4')))
t = full((2, 2), [1, 2], dtype='f8', chunk_size=1)
res = self.executor.execute_tensor(t, concat=True)
self.assertTrue(np.array_equal(res[0], np.full((2, 2), [1, 2], dtype='f8')))
def testArangeExecution(self):
t = arange(1, 20, 3, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertTrue(np.array_equal(res, np.arange(1, 20, 3)))
t = arange(1, 20, .3, chunk_size=4)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.arange(1, 20, .3)
self.assertTrue(np.allclose(res, expected))
t = arange(1.0, 1.8, .3, chunk_size=4)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.arange(1.0, 1.8, .3)
self.assertTrue(np.allclose(res, expected))
t = arange('1066-10-13', '1066-10-31', dtype=np.datetime64, chunk_size=3)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.arange('1066-10-13', '1066-10-31', dtype=np.datetime64)
self.assertTrue(np.array_equal(res, expected))
def testDiagExecution(self):
# 2-d 6 * 6
a = arange(36, chunk_size=2).reshape(6, 6)
d = diag(a)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(36).reshape(6, 6))
np.testing.assert_equal(res, expected)
d = diag(a, k=1)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(36).reshape(6, 6), k=1)
np.testing.assert_equal(res, expected)
d = diag(a, k=3)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(36).reshape(6, 6), k=3)
np.testing.assert_equal(res, expected)
d = diag(a, k=-2)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(36).reshape(6, 6), k=-2)
np.testing.assert_equal(res, expected)
d = diag(a, k=-5)
res = self.executor.execute_tensor(d)[0]
expected = np.diag(np.arange(36).reshape(6, 6), k=-5)
np.testing.assert_equal(res, expected)
# 2-d 4 * 9
a = arange(36, chunk_size=2).reshape(4, 9)
d = diag(a)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(36).reshape(4, 9))
np.testing.assert_equal(res, expected)
d = diag(a, k=1)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(36).reshape(4, 9), k=1)
np.testing.assert_equal(res, expected)
d = diag(a, k=3)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(36).reshape(4, 9), k=3)
np.testing.assert_equal(res, expected)
d = diag(a, k=-2)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(36).reshape(4, 9), k=-2)
np.testing.assert_equal(res, expected)
d = diag(a, k=-3)
res = self.executor.execute_tensor(d)[0]
expected = np.diag(np.arange(36).reshape(4, 9), k=-3)
np.testing.assert_equal(res, expected)
# 1-d
a = arange(5, chunk_size=2)
d = diag(a)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5))
np.testing.assert_equal(res, expected)
d = diag(a, k=1)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5), k=1)
np.testing.assert_equal(res, expected)
d = diag(a, k=3)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5), k=3)
np.testing.assert_equal(res, expected)
d = diag(a, k=-2)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5), k=-2)
np.testing.assert_equal(res, expected)
d = diag(a, k=-3)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5), k=-3)
np.testing.assert_equal(res, expected)
d = diag(a, sparse=True)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5))
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
d = diag(a, k=1, sparse=True)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5), k=1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
d = diag(a, k=2, sparse=True)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5), k=2)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
d = diag(a, k=-2, sparse=True)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5), k=-2)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
d = diag(a, k=-3, sparse=True)
res = self.executor.execute_tensor(d, concat=True)[0]
expected = np.diag(np.arange(5), k=-3)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
def testDiagflatExecution(self):
a = diagflat([[1, 2], [3, 4]], chunk_size=1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.diagflat([[1, 2], [3, 4]])
np.testing.assert_equal(res, expected)
d = tensor([[1, 2], [3, 4]], chunk_size=1)
a = diagflat(d)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.diagflat([[1, 2], [3, 4]])
np.testing.assert_equal(res, expected)
a = diagflat([1, 2], 1, chunk_size=1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.diagflat([1, 2], 1)
np.testing.assert_equal(res, expected)
d = tensor([[1, 2]], chunk_size=1)
a = diagflat(d, 1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.diagflat([1, 2], 1)
np.testing.assert_equal(res, expected)
def testEyeExecution(self):
t = eye(5, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5)
np.testing.assert_equal(res, expected)
t = eye(5, k=1, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, k=1)
np.testing.assert_equal(res, expected)
t = eye(5, k=2, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, k=2)
np.testing.assert_equal(res, expected)
t = eye(5, k=-1, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, k=-1)
np.testing.assert_equal(res, expected)
t = eye(5, k=-3, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, k=-3)
np.testing.assert_equal(res, expected)
t = eye(5, M=3, k=1, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, M=3, k=1)
np.testing.assert_equal(res, expected)
t = eye(5, M=3, k=-3, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, M=3, k=-3)
np.testing.assert_equal(res, expected)
t = eye(5, M=7, k=1, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, M=7, k=1)
np.testing.assert_equal(res, expected)
t = eye(5, M=8, k=-3, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, M=8, k=-3)
np.testing.assert_equal(res, expected)
t = eye(2, dtype=int)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.dtype, np.int_)
# test sparse
t = eye(5, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
t = eye(5, k=1, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, k=1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
t = eye(5, k=2, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, k=2)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
t = eye(5, k=-1, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, k=-1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
t = eye(5, k=-3, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, k=-3)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
t = eye(5, M=3, k=1, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, M=3, k=1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
t = eye(5, M=3, k=-3, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, M=3, k=-3)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
t = eye(5, M=7, k=1, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, M=7, k=1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
t = eye(5, M=8, k=-3, sparse=True, chunk_size=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.eye(5, M=8, k=-3)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res.toarray(), expected)
def testLinspaceExecution(self):
a = linspace(2.0, 9.0, num=11, chunk_size=3)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.linspace(2.0, 9.0, num=11)
np.testing.assert_allclose(res, expected)
a = linspace(2.0, 9.0, num=11, endpoint=False, chunk_size=3)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.linspace(2.0, 9.0, num=11, endpoint=False)
np.testing.assert_allclose(res, expected)
a = linspace(2.0, 9.0, num=11, chunk_size=3, dtype=int)
res = self.executor.execute_tensor(a, concat=True)[0]
self.assertEqual(res.dtype, np.int_)
def testMeshgridExecution(self):
a = arange(5, chunk_size=2)
b = arange(6, 12, chunk_size=3)
c = arange(12, 19, chunk_size=4)
A, B, C = meshgrid(a, b, c)
A_res = self.executor.execute_tensor(A, concat=True)[0]
A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[0]
np.testing.assert_equal(A_res, A_expected)
B_res = self.executor.execute_tensor(B, concat=True)[0]
B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[1]
np.testing.assert_equal(B_res, B_expected)
C_res = self.executor.execute_tensor(C, concat=True)[0]
C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[2]
np.testing.assert_equal(C_res, C_expected)
A, B, C = meshgrid(a, b, c, indexing='ij')
A_res = self.executor.execute_tensor(A, concat=True)[0]
A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[0]
np.testing.assert_equal(A_res, A_expected)
B_res = self.executor.execute_tensor(B, concat=True)[0]
B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[1]
np.testing.assert_equal(B_res, B_expected)
C_res = self.executor.execute_tensor(C, concat=True)[0]
C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[2]
np.testing.assert_equal(C_res, C_expected)
A, B, C = meshgrid(a, b, c, sparse=True)
A_res = self.executor.execute_tensor(A, concat=True)[0]
A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[0]
np.testing.assert_equal(A_res, A_expected)
B_res = self.executor.execute_tensor(B, concat=True)[0]
B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[1]
np.testing.assert_equal(B_res, B_expected)
C_res = self.executor.execute_tensor(C, concat=True)[0]
C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[2]
np.testing.assert_equal(C_res, C_expected)
A, B, C = meshgrid(a, b, c, indexing='ij', sparse=True)
A_res = self.executor.execute_tensor(A, concat=True)[0]
A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19),
indexing='ij', sparse=True)[0]
np.testing.assert_equal(A_res, A_expected)
B_res = self.executor.execute_tensor(B, concat=True)[0]
B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19),
indexing='ij', sparse=True)[1]
np.testing.assert_equal(B_res, B_expected)
C_res = self.executor.execute_tensor(C, concat=True)[0]
C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19),
indexing='ij', sparse=True)[2]
np.testing.assert_equal(C_res, C_expected)
def testIndicesExecution(self):
grid = indices((2, 3), chunk_size=1)
res = self.executor.execute_tensor(grid, concat=True)[0]
expected = np.indices((2, 3))
np.testing.assert_equal(res, expected)
res = self.executor.execute_tensor(grid[0], concat=True)[0]
np.testing.assert_equal(res, expected[0])
res = self.executor.execute_tensor(grid[1], concat=True)[0]
np.testing.assert_equal(res, expected[1])
def testTriuExecution(self):
a = arange(24, chunk_size=2).reshape(2, 3, 4)
t = triu(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(24).reshape(2, 3, 4))
np.testing.assert_equal(res, expected)
t = triu(a, k=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(24).reshape(2, 3, 4), k=1)
np.testing.assert_equal(res, expected)
t = triu(a, k=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(24).reshape(2, 3, 4), k=2)
np.testing.assert_equal(res, expected)
t = triu(a, k=-1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(24).reshape(2, 3, 4), k=-1)
np.testing.assert_equal(res, expected)
t = triu(a, k=-2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(24).reshape(2, 3, 4), k=-2)
np.testing.assert_equal(res, expected)
# test sparse
a = arange(12, chunk_size=2).reshape(3, 4).tosparse()
t = triu(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(12).reshape(3, 4))
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
t = triu(a, k=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(12).reshape(3, 4), k=1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
t = triu(a, k=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(12).reshape(3, 4), k=2)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
t = triu(a, k=-1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(12).reshape(3, 4), k=-1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
t = triu(a, k=-2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.triu(np.arange(12).reshape(3, 4), k=-2)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
def testTrilExecution(self):
a = arange(24, chunk_size=2).reshape(2, 3, 4)
t = tril(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(24).reshape(2, 3, 4))
np.testing.assert_equal(res, expected)
t = tril(a, k=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(24).reshape(2, 3, 4), k=1)
np.testing.assert_equal(res, expected)
t = tril(a, k=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(24).reshape(2, 3, 4), k=2)
np.testing.assert_equal(res, expected)
t = tril(a, k=-1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(24).reshape(2, 3, 4), k=-1)
np.testing.assert_equal(res, expected)
t = tril(a, k=-2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(24).reshape(2, 3, 4), k=-2)
np.testing.assert_equal(res, expected)
a = arange(12, chunk_size=2).reshape(3, 4).tosparse()
t = tril(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(12).reshape(3, 4))
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
t = tril(a, k=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(12).reshape(3, 4), k=1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
t = tril(a, k=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(12).reshape(3, 4), k=2)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
t = tril(a, k=-1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(12).reshape(3, 4), k=-1)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
t = tril(a, k=-2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tril(np.arange(12).reshape(3, 4), k=-2)
self.assertIsInstance(res, SparseNDArray)
np.testing.assert_equal(res, expected)
def testIndexTrickExecution(self):
mgrid = nd_grid()
t = mgrid[0:5, 0:5]
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.lib.index_tricks.nd_grid()[0:5, 0:5]
np.testing.assert_equal(res, expected)
t = mgrid[-1:1:5j]
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.lib.index_tricks.nd_grid()[-1:1:5j]
np.testing.assert_equal(res, expected)
ogrid = nd_grid(sparse=True)
t = ogrid[0:5, 0:5]
res = [self.executor.execute_tensor(o, concat=True)[0] for o in t]
expected = np.lib.index_tricks.nd_grid(sparse=True)[0:5, 0:5]
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
@unittest.skipIf(tiledb is None, 'tiledb not installed')
def testReadTileDBExecution(self):
ctx = tiledb.Ctx()
tempdir = tempfile.mkdtemp()
try:
# create TileDB dense array
dom = tiledb.Domain(ctx,
tiledb.Dim(ctx, domain=(1, 100), tile=30, dtype=np.int32),
tiledb.Dim(ctx, domain=(0, 90), tile=22, dtype=np.int32),
tiledb.Dim(ctx, domain=(0, 9), tile=8, dtype=np.int32),
)
schema = tiledb.ArraySchema(ctx, domain=dom, sparse=False,
attrs=[tiledb.Attr(ctx, dtype=np.float64)])
tiledb.DenseArray.create(tempdir, schema)
expected = np.random.rand(100, 91, 10)
with tiledb.DenseArray(ctx, tempdir, mode='w') as arr:
arr.write_direct(expected)
a = fromtiledb(tempdir, ctx=ctx)
result = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_allclose(expected, result)
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
try:
# create 2-d TileDB sparse array
dom = tiledb.Domain(ctx,
tiledb.Dim(ctx, domain=(0, 99), tile=30, dtype=np.int32),
tiledb.Dim(ctx, domain=(2, 11), tile=8, dtype=np.int32),
)
schema = tiledb.ArraySchema(ctx, domain=dom, sparse=True,
attrs=[tiledb.Attr(ctx, dtype=np.float64)])
tiledb.SparseArray.create(tempdir, schema)
expected = sps.rand(100, 10, density=0.01)
with tiledb.SparseArray(ctx, tempdir, mode='w') as arr:
I, J = expected.row, expected.col + 2
arr[I, J] = {arr.attr(0).name: expected.data}
a = fromtiledb(tempdir, ctx=ctx)
result = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_allclose(expected.toarray(), result.toarray())
finally:
shutil.rmtree(tempdir)
tempdir = tempfile.mkdtemp()
try:
# create 1-d TileDB sparse array
dom = tiledb.Domain(ctx,
tiledb.Dim(ctx, domain=(1, 100), tile=30, dtype=np.int32),
)
schema = tiledb.ArraySchema(ctx, domain=dom, sparse=True,
attrs=[tiledb.Attr(ctx, dtype=np.float64)])
tiledb.SparseArray.create(tempdir, schema)
expected = sps.rand(1, 100, density=0.05)
with tiledb.SparseArray(ctx, tempdir, mode='w') as arr:
I= expected.col + 1
arr[I] = expected.data
a = fromtiledb(tempdir, ctx=ctx)
result = self.executor.execute_tensor(a, concat=True)[0]
np.testing.assert_allclose(expected.toarray()[0], result.toarray())
finally:
shutil.rmtree(tempdir)
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testRechunkExecution(self):
raw = np.random.random((11, 8))
arr = tensor(raw, chunk_size=3)
arr2 = arr.rechunk(4)
res = self.executor.execute_tensor(arr2)
self.assertTrue(np.array_equal(res[0], raw[:4, :4]))
self.assertTrue(np.array_equal(res[1], raw[:4, 4:]))
self.assertTrue(np.array_equal(res[2], raw[4:8, :4]))
self.assertTrue(np.array_equal(res[3], raw[4:8, 4:]))
self.assertTrue(np.array_equal(res[4], raw[8:, :4]))
self.assertTrue(np.array_equal(res[5], raw[8:, 4:]))
def testCopytoExecution(self):
a = ones((2, 3), chunk_size=1)
b = tensor([3, -1, 3], chunk_size=2)
copyto(a, b, where=b > 1)
res = self.executor.execute_tensor(a, concat=True)[0]
expected = np.array([[3, 1, 3], [3, 1, 3]])
np.testing.assert_equal(res, expected)
def testAstypeExecution(self):
raw = np.random.random((10, 5))
arr = tensor(raw, chunk_size=3)
arr2 = arr.astype('i8')
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.array_equal(res[0], raw.astype('i8')))
raw = sps.random(10, 5, density=.2)
arr = tensor(raw, chunk_size=3)
arr2 = arr.astype('i8')
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(
np.array_equal(res[0].toarray(),
raw.astype('i8').toarray()))
def testTransposeExecution(self):
raw = np.random.random((11, 8, 5))
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr)
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.array_equal(res[0], raw.T))
arr3 = transpose(arr, axes=(-2, -1, -3))
res = self.executor.execute_tensor(arr3, concat=True)
self.assertTrue(np.array_equal(res[0], raw.transpose(1, 2, 0)))
raw = sps.random(11, 8)
arr = tensor(raw, chunk_size=3)
arr2 = transpose(arr)
self.assertTrue(arr2.issparse())
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.array_equal(res[0].toarray(), raw.T.toarray()))
def testSwapaxesExecution(self):
raw = np.random.random((11, 8, 5))
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(2, 0)
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(np.array_equal(res[0], raw.swapaxes(2, 0)))
raw = sps.random(11, 8, density=.2)
arr = tensor(raw, chunk_size=3)
arr2 = arr.swapaxes(1, 0)
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(
np.array_equal(res[0].toarray(),
raw.toarray().swapaxes(1, 0)))
def testMoveaxisExecution(self):
x = zeros((3, 4, 5), chunk_size=2)
t = moveaxis(x, 0, -1)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (4, 5, 3))
t = moveaxis(x, -1, 0)
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 3, 4))
t = moveaxis(x, [0, 1], [-1, -2])
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 4, 3))
t = moveaxis(x, [0, 1, 2], [-1, -2, -3])
res = self.executor.execute_tensor(t, concat=True)[0]
self.assertEqual(res.shape, (5, 4, 3))
def testBroadcastToExecution(self):
raw = np.random.random((10, 5, 1))
arr = tensor(raw, chunk_size=2)
arr2 = broadcast_to(arr, (5, 10, 5, 6))
res = self.executor.execute_tensor(arr2, concat=True)
self.assertTrue(
np.array_equal(res[0], np.broadcast_to(raw, (5, 10, 5, 6))))
def testBroadcastArraysExecutions(self):
x_data = [[1, 2, 3]]
x = tensor(x_data, chunk_size=1)
y_data = [[1], [2], [3]]
y = tensor(y_data, chunk_size=2)
a = broadcast_arrays(x, y)
res = [self.executor.execute_tensor(arr, concat=True)[0] for arr in a]
expected = np.broadcast_arrays(x_data, y_data)
for r, e in zip(res, expected):
np.testing.assert_equal(r, e)
def testWhereExecution(self):
raw_cond = np.random.randint(0, 2, size=(4, 4), dtype='?')
raw_x = np.random.rand(4, 1)
raw_y = np.random.rand(4, 4)
cond, x, y = tensor(raw_cond, chunk_size=2), tensor(
raw_x, chunk_size=2), tensor(raw_y, chunk_size=2)
arr = where(cond, x, y)
res = self.executor.execute_tensor(arr, concat=True)
self.assertTrue(
np.array_equal(res[0], np.where(raw_cond, raw_x, raw_y)))
raw_cond = sps.csr_matrix(
np.random.randint(0, 2, size=(4, 4), dtype='?'))
raw_x = sps.random(4, 1, density=.1)
raw_y = sps.random(4, 4, density=.1)
cond, x, y = tensor(raw_cond, chunk_size=2), tensor(
raw_x, chunk_size=2), tensor(raw_y, chunk_size=2)
arr = where(cond, x, y)
res = self.executor.execute_tensor(arr, concat=True)[0]
self.assertTrue(
np.array_equal(
res.toarray(),
np.where(raw_cond.toarray(), raw_x.toarray(),
raw_y.toarray())))
def testReshapeExecution(self):
raw_data = np.random.rand(10, 20, 30)
x = tensor(raw_data, chunk_size=6)
y = x.reshape(-1, 30)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], raw_data.reshape(-1, 30)))
y2 = x.reshape(10, -1)
res = self.executor.execute_tensor(y2, concat=True)
self.assertTrue(np.array_equal(res[0], raw_data.reshape(10, -1)))
y3 = x.reshape(-1)
res = self.executor.execute_tensor(y3, concat=True)
self.assertTrue(np.array_equal(res[0], raw_data.reshape(-1)))
y4 = x.ravel()
res = self.executor.execute_tensor(y4, concat=True)
self.assertTrue(np.array_equal(res[0], raw_data.ravel()))
raw_data = np.random.rand(30, 100, 20)
x = tensor(raw_data, chunk_size=6)
y = x.reshape(-1, 20, 5, 5, 4)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(
np.array_equal(res[0], raw_data.reshape(-1, 20, 5, 5, 4)))
y2 = x.reshape(3000, 10, 2)
res = self.executor.execute_tensor(y2, concat=True)
self.assertTrue(np.array_equal(res[0], raw_data.reshape(3000, 10, 2)))
y3 = x.reshape(60, 25, 40)
res = self.executor.execute_tensor(y3, concat=True)
self.assertTrue(np.array_equal(res[0], raw_data.reshape(60, 25, 40)))
y4 = x.reshape(60, 25, 40)
y4.op.extra_params['_reshape_with_shuffle'] = True
size_res = self.executor.execute_tensor(y4, mock=True)
res = self.executor.execute_tensor(y4, concat=True)
self.assertEqual(res[0].nbytes, sum(v[0] for v in size_res))
self.assertTrue(np.array_equal(res[0], raw_data.reshape(60, 25, 40)))
def testExpandDimsExecution(self):
raw_data = np.random.rand(10, 20, 30)
x = tensor(raw_data, chunk_size=6)
y = expand_dims(x, 1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 1)))
y = expand_dims(x, 0)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 0)))
y = expand_dims(x, 3)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 3)))
y = expand_dims(x, -1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -1)))
y = expand_dims(x, -4)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -4)))
with self.assertRaises(np.AxisError):
expand_dims(x, -5)
with self.assertRaises(np.AxisError):
expand_dims(x, 4)
def testRollAxisExecution(self):
x = ones((3, 4, 5, 6), chunk_size=1)
y = rollaxis(x, 3, 1)
res = self.executor.execute_tensor(y, concat=True)
self.assertTrue(
np.array_equal(res[0], np.rollaxis(np.ones((3, 4, 5, 6)), 3, 1)))
def testAtleast1dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_1d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.array([1])))
self.assertTrue(np.array_equal(res[1], np.ones(3)))
self.assertTrue(np.array_equal(res[2], np.ones((3, 4))))
def testAtleast2dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_2d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.array([[1]])))
self.assertTrue(np.array_equal(res[1], np.atleast_2d(np.ones(3))))
self.assertTrue(np.array_equal(res[2], np.ones((3, 4))))
def testAtleast3dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_3d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.atleast_3d(x)))
self.assertTrue(np.array_equal(res[1], np.atleast_3d(np.ones(3))))
self.assertTrue(np.array_equal(res[2], np.atleast_3d(np.ones((3, 4)))))
def testArgwhereExecution(self):
x = arange(6, chunk_size=2).reshape(2, 3)
t = argwhere(x > 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.argwhere(np.arange(6).reshape(2, 3) > 1)
self.assertTrue(np.array_equal(res, expected))
def testArraySplitExecution(self):
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = array_split(x, 3, axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.array_split(np.arange(48).reshape(2, 3, 8), 3, axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
ss = array_split(x, [3, 5, 6, 10], axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.array_split(np.arange(48).reshape(2, 3, 8),
[3, 5, 6, 10],
axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
def testSplitExecution(self):
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = split(x, 4, axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(np.arange(48).reshape(2, 3, 8), 4, axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
ss = split(x, [3, 5, 6, 10], axis=2)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10],
axis=2)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# hsplit
x = arange(120, chunk_size=3).reshape(2, 12, 5)
ss = hsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.hsplit(np.arange(120).reshape(2, 12, 5), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# vsplit
x = arange(48, chunk_size=3).reshape(8, 3, 2)
ss = vsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.vsplit(np.arange(48).reshape(8, 3, 2), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
# dsplit
x = arange(48, chunk_size=3).reshape(2, 3, 8)
ss = dsplit(x, 4)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.dsplit(np.arange(48).reshape(2, 3, 8), 4)
self.assertEqual(len(res), len(expected))
[np.testing.assert_equal(r, e) for r, e in zip(res, expected)]
x_data = sps.random(12, 8, density=.1)
x = tensor(x_data, chunk_size=3)
ss = split(x, 4, axis=0)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss]
expected = np.split(x_data.toarray(), 4, axis=0)
self.assertEqual(len(res), len(expected))
[
np.testing.assert_equal(r.toarray(), e)
for r, e in zip(res, expected)
]
def testRollExecution(self):
x = arange(10, chunk_size=2)
t = roll(x, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10), 2)
np.testing.assert_equal(res, expected)
x2 = x.reshape(2, 5)
t = roll(x2, 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1)
np.testing.assert_equal(res, expected)
t = roll(x2, 1, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=0)
np.testing.assert_equal(res, expected)
t = roll(x2, 1, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=1)
np.testing.assert_equal(res, expected)
def testSqueezeExecution(self):
data = np.array([[[0], [1], [2]]])
x = tensor(data, chunk_size=1)
t = squeeze(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.squeeze(data)
np.testing.assert_equal(res, expected)
t = squeeze(x, axis=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.squeeze(data, axis=2)
np.testing.assert_equal(res, expected)
def testPtpExecution(self):
x = arange(4, chunk_size=1).reshape(2, 2)
t = ptp(x, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ptp(np.arange(4).reshape(2, 2), axis=0)
np.testing.assert_equal(res, expected)
t = ptp(x, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ptp(np.arange(4).reshape(2, 2), axis=1)
np.testing.assert_equal(res, expected)
t = ptp(x)
res = self.executor.execute_tensor(t)[0]
expected = np.ptp(np.arange(4).reshape(2, 2))
np.testing.assert_equal(res, expected)
def testDiffExecution(self):
data = np.array([1, 2, 4, 7, 0])
x = tensor(data, chunk_size=2)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data)
np.testing.assert_equal(res, expected)
t = diff(x, n=2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data, n=2)
np.testing.assert_equal(res, expected)
data = np.array([[1, 3, 6, 10], [0, 5, 6, 8]])
x = tensor(data, chunk_size=2)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data)
np.testing.assert_equal(res, expected)
t = diff(x, axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(data, axis=0)
np.testing.assert_equal(res, expected)
x = mt.arange('1066-10-13', '1066-10-16', dtype=mt.datetime64)
t = diff(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.diff(
np.arange('1066-10-13', '1066-10-16', dtype=np.datetime64))
np.testing.assert_equal(res, expected)
def testEdiff1d(self):
data = np.array([1, 2, 4, 7, 0])
x = tensor(data, chunk_size=2)
t = ediff1d(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data)
np.testing.assert_equal(res, expected)
to_begin = tensor(-99, chunk_size=2)
to_end = tensor([88, 99], chunk_size=2)
t = ediff1d(x, to_begin=to_begin, to_end=to_end)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data, to_begin=-99, to_end=np.array([88, 99]))
np.testing.assert_equal(res, expected)
data = [[1, 2, 4], [1, 6, 24]]
t = ediff1d(tensor(data, chunk_size=2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.ediff1d(data)
np.testing.assert_equal(res, expected)
def testDigitizeExecution(self):
data = np.array([0.2, 6.4, 3.0, 1.6])
x = tensor(data, chunk_size=2)
bins = np.array([0.0, 1.0, 2.5, 4.0, 10.0])
inds = digitize(x, bins)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data, bins)
np.testing.assert_equal(res, expected)
b = tensor(bins, chunk_size=2)
inds = digitize(x, b)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data, bins)
np.testing.assert_equal(res, expected)
data = np.array([1.2, 10.0, 12.4, 15.5, 20.])
x = tensor(data, chunk_size=2)
bins = np.array([0, 5, 10, 15, 20])
inds = digitize(x, bins, right=True)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data, bins, right=True)
np.testing.assert_equal(res, expected)
inds = digitize(x, bins, right=False)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data, bins, right=False)
np.testing.assert_equal(res, expected)
data = sps.random(10, 1, density=.1) * 12
x = tensor(data, chunk_size=2)
bins = np.array([1.0, 2.0, 2.5, 4.0, 10.0])
inds = digitize(x, bins)
res = self.executor.execute_tensor(inds, concat=True)[0]
expected = np.digitize(data.toarray(), bins, right=False)
np.testing.assert_equal(res.toarray(), expected)
def testAverageExecution(self):
data = arange(1, 5, chunk_size=1)
t = average(data)
res = self.executor.execute_tensor(t)[0]
expected = np.average(np.arange(1, 5))
self.assertEqual(res, expected)
t = average(arange(1, 11, chunk_size=2),
weights=arange(10, 0, -1, chunk_size=2))
res = self.executor.execute_tensor(t)[0]
expected = np.average(range(1, 11), weights=range(10, 0, -1))
self.assertEqual(res, expected)
data = arange(6, chunk_size=2).reshape((3, 2))
t = average(data,
axis=1,
weights=tensor([1. / 4, 3. / 4], chunk_size=2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.average(np.arange(6).reshape(3, 2),
axis=1,
weights=(1. / 4, 3. / 4))
np.testing.assert_equal(res, expected)
with self.assertRaises(TypeError):
average(data, weights=tensor([1. / 4, 3. / 4], chunk_size=2))
def testCovExecution(self):
data = np.array([[0, 2], [1, 1], [2, 0]]).T
x = tensor(data, chunk_size=1)
t = cov(x)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.cov(data)
np.testing.assert_equal(res, expected)
data_x = [-2.1, -1, 4.3]
data_y = [3, 1.1, 0.12]
x = tensor(data_x, chunk_size=1)
y = tensor(data_y, chunk_size=1)
X = stack((x, y), axis=0)
t = cov(x, y)
r = tall(t == cov(X))
self.assertTrue(self.executor.execute_tensor(r)[0])
def testCorrcoefExecution(self):
data_x = [-2.1, -1, 4.3]
data_y = [3, 1.1, 0.12]
x = tensor(data_x, chunk_size=1)
y = tensor(data_y, chunk_size=1)
t = corrcoef(x, y)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.corrcoef(data_x, data_y)
np.testing.assert_equal(res, expected)
def testFlipExecution(self):
a = arange(8, chunk_size=2).reshape((2, 2, 2))
t = flip(a, 0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flip(np.arange(8).reshape(2, 2, 2), 0)
np.testing.assert_equal(res, expected)
t = flip(a, 1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flip(np.arange(8).reshape(2, 2, 2), 1)
np.testing.assert_equal(res, expected)
t = flipud(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.flipud(np.arange(8).reshape(2, 2, 2))
np.testing.assert_equal(res, expected)
t = fliplr(a)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.fliplr(np.arange(8).reshape(2, 2, 2))
np.testing.assert_equal(res, expected)
def testRepeatExecution(self):
a = repeat(3, 4)
res = self.executor.execute_tensor(a)[0]
expected = np.repeat(3, 4)
np.testing.assert_equal(res, expected)
x_data = np.random.randn(20, 30)
x = tensor(x_data, chunk_size=(3, 4))
t = repeat(x, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, 2)
np.testing.assert_equal(res, expected)
t = repeat(x, 3, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, 3, axis=1)
np.testing.assert_equal(res, expected)
t = repeat(x, np.arange(20), axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, np.arange(20), axis=0)
np.testing.assert_equal(res, expected)
t = repeat(x, arange(20, chunk_size=5), axis=0)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data, np.arange(20), axis=0)
np.testing.assert_equal(res, expected)
x_data = sps.random(20, 30, density=.1)
x = tensor(x_data, chunk_size=(3, 4))
t = repeat(x, 2, axis=1)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.repeat(x_data.toarray(), 2, axis=1)
np.testing.assert_equal(res.toarray(), expected)
def testTileExecution(self):
a_data = np.array([0, 1, 2])
a = tensor(a_data, chunk_size=2)
t = tile(a, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, 2)
np.testing.assert_equal(res, expected)
t = tile(a, (2, 2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, (2, 2))
np.testing.assert_equal(res, expected)
t = tile(a, (2, 1, 2))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(a_data, (2, 1, 2))
np.testing.assert_equal(res, expected)
b_data = np.array([[1, 2], [3, 4]])
b = tensor(b_data, chunk_size=1)
t = tile(b, 2)
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(b_data, 2)
np.testing.assert_equal(res, expected)
t = tile(b, (2, 1))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(b_data, (2, 1))
np.testing.assert_equal(res, expected)
c_data = np.array([1, 2, 3, 4])
c = tensor(c_data, chunk_size=3)
t = tile(c, (4, 1))
res = self.executor.execute_tensor(t, concat=True)[0]
expected = np.tile(c_data, (4, 1))
np.testing.assert_equal(res, expected)
def testIsInExecution(self):
element = 2 * arange(4, chunk_size=1).reshape((2, 2))
test_elements = [1, 2, 4, 8]
mask = isin(element, test_elements)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements)
np.testing.assert_equal(res, expected)
res = self.executor.execute_tensor(element[mask], concat=True)[0]
expected = np.array([2, 4])
np.testing.assert_equal(res, expected)
mask = isin(element, test_elements, invert=True)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)),
test_elements,
invert=True)
np.testing.assert_equal(res, expected)
res = self.executor.execute_tensor(element[mask], concat=True)[0]
expected = np.array([0, 6])
np.testing.assert_equal(res, expected)
test_set = {1, 2, 4, 8}
mask = isin(element, test_set)
res = self.executor.execute_tensor(mask, concat=True)[0]
expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_set)
np.testing.assert_equal(res, expected)
def setUp(self):
self.executor = Executor('cupy')
def setUp(self):
self.executor = Executor('numpy')
class Test(unittest.TestCase):
def setUp(self):
self.executor = Executor('numpy')
def testFFTExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 4, 30))
r = fft(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft(raw)
np.testing.assert_allclose(res, expected)
r = fft(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = fft(t, n=11)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft(raw, n=11)
np.testing.assert_allclose(res, expected)
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 4, 4))
r = fft(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft(raw)
np.testing.assert_allclose(res, expected)
r = fft(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = fft(t, n=11)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft(raw, n=11)
np.testing.assert_allclose(res, expected)
def testIFFTExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 4, 30))
r = ifft(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft(raw)
np.testing.assert_allclose(res, expected)
r = ifft(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = ifft(t, n=11)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft(raw, n=11)
np.testing.assert_allclose(res, expected)
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 4, 5))
r = ifft(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft(raw)
np.testing.assert_allclose(res, expected)
r = ifft(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = ifft(t, n=11)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft(raw, n=11)
np.testing.assert_allclose(res, expected)
def testFFT2Execution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 20, 30))
r = fft2(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft2(raw)
np.testing.assert_allclose(res, expected)
r = fft2(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft2(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = fft2(t, s=(11, 12))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft2(raw, s=(11, 12))
np.testing.assert_allclose(res, expected)
r = fft2(t, s=(11, 12), axes=(-1, -2))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft2(raw, s=(11, 12), axes=(-1, -2))
np.testing.assert_allclose(res, expected)
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 5, 6))
r = fft2(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft2(raw)
np.testing.assert_allclose(res, expected)
r = fft2(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft2(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = fft2(t, s=(11, 12))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft2(raw, s=(11, 12))
np.testing.assert_allclose(res, expected)
r = fft2(t, s=(11, 12), axes=(-1, -2))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fft2(raw, s=(11, 12), axes=(-1, -2))
np.testing.assert_allclose(res, expected)
def testIFFT2Execution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 20, 30))
r = ifft2(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft2(raw)
np.testing.assert_allclose(res, expected)
r = ifft2(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft2(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = ifft2(t, s=(11, 12))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft2(raw, s=(11, 12))
np.testing.assert_allclose(res, expected)
r = ifft2(t, s=(11, 12), axes=(-1, -2))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft2(raw, s=(11, 12), axes=(-1, -2))
np.testing.assert_allclose(res, expected)
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 3, 5))
r = ifft2(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft2(raw)
np.testing.assert_allclose(res, expected)
r = ifft2(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft2(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = ifft2(t, s=(11, 12))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft2(raw, s=(11, 12))
np.testing.assert_allclose(res, expected)
r = ifft2(t, s=(11, 12), axes=(-1, -2))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifft2(raw, s=(11, 12), axes=(-1, -2))
np.testing.assert_allclose(res, expected)
def testFFTNExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(10, 20, 30))
r = fftn(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftn(raw)
np.testing.assert_allclose(res, expected)
r = fftn(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftn(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = fftn(t, s=(11, 12, 5))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftn(raw, s=(11, 12, 5))
np.testing.assert_allclose(res, expected)
r = fftn(t, s=(11, 12, 5), axes=(-1, -2, -3))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftn(raw, s=(11, 12, 5), axes=(-1, -2, -3))
np.testing.assert_allclose(res, expected)
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(3, 3, 4))
r = fftn(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftn(raw)
np.testing.assert_allclose(res, expected)
r = fftn(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftn(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = fftn(t, s=(11, 12, 5))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftn(raw, s=(11, 12, 5))
np.testing.assert_allclose(res, expected)
r = fftn(t, s=(11, 12, 5), axes=(-1, -2, -3))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftn(raw, s=(11, 12, 5), axes=(-1, -2, -3))
np.testing.assert_allclose(res, expected)
def testIFFTNExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(10, 20, 30))
r = ifftn(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftn(raw)
np.testing.assert_allclose(res, expected)
r = ifftn(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftn(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = ifftn(t, s=(11, 12, 5))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftn(raw, s=(11, 12, 5))
np.testing.assert_allclose(res, expected)
r = ifftn(t, s=(11, 12, 5), axes=(-1, -2, -3))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftn(raw, s=(11, 12, 5), axes=(-1, -2, -3))
np.testing.assert_allclose(res, expected)
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(3, 4, 7))
r = ifftn(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftn(raw)
np.testing.assert_allclose(res, expected)
r = ifftn(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftn(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = ifftn(t, s=(11, 12, 5))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftn(raw, s=(11, 12, 5))
np.testing.assert_allclose(res, expected)
r = ifftn(t, s=(11, 12, 5), axes=(-1, -2, -3))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftn(raw, s=(11, 12, 5), axes=(-1, -2, -3))
np.testing.assert_allclose(res, expected)
def testRFFTExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 4, 30))
r = rfft(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfft(raw)
np.testing.assert_allclose(res, expected)
r = rfft(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfft(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = rfft(t, n=11)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfft(raw, n=11)
np.testing.assert_allclose(res, expected)
def testIRFFTExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 4, 30))
r = irfft(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfft(raw)
np.testing.assert_allclose(res, expected)
r = irfft(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfft(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = irfft(t, n=11)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfft(raw, n=11)
np.testing.assert_allclose(res, expected)
def testRFFT2Execution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 20, 30))
r = rfft2(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfft2(raw)
np.testing.assert_allclose(res, expected)
r = rfft2(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfft2(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = rfft2(t, s=(11, 12))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfft2(raw, s=(11, 12))
np.testing.assert_allclose(res, expected)
r = rfft2(t, s=(11, 12), axes=(-1, -2))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfft2(raw, s=(11, 12), axes=(-1, -2))
np.testing.assert_allclose(res, expected)
def testIRFFT2Execution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 20, 30))
r = irfft2(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfft2(raw)
np.testing.assert_allclose(res, expected)
r = irfft2(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfft2(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = irfft2(t, s=(11, 12))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfft2(raw, s=(11, 12))
np.testing.assert_allclose(res, expected)
r = irfft2(t, s=(11, 12), axes=(-1, -2))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfft2(raw, s=(11, 12), axes=(-1, -2))
np.testing.assert_allclose(res, expected)
def testRFFTNExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(10, 20, 30))
r = rfftn(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfftn(raw)
np.testing.assert_allclose(res, expected)
r = rfftn(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfftn(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = rfftn(t, s=(11, 12, 5))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfftn(raw, s=(11, 12, 5))
np.testing.assert_allclose(res, expected)
r = rfftn(t, s=(11, 12, 11), axes=(-1, -2, -3))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.rfftn(raw, s=(11, 12, 11), axes=(-1, -2, -3))
np.testing.assert_allclose(res, expected)
def testIRFFTNExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(10, 20, 30))
r = irfftn(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfftn(raw)
np.testing.assert_allclose(res, expected)
r = irfftn(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfftn(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = irfftn(t, s=(11, 21, 5))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfftn(raw, s=(11, 21, 5))
np.testing.assert_allclose(res, expected)
r = irfftn(t, s=(11, 21, 30), axes=(-1, -2, -3))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.irfftn(raw, s=(11, 21, 30), axes=(-1, -2, -3))
np.testing.assert_allclose(res, expected)
def testHFFTExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 4, 30))
r = hfft(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.hfft(raw)
np.testing.assert_allclose(res, expected)
r = hfft(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.hfft(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = hfft(t, n=11)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.hfft(raw, n=11)
np.testing.assert_allclose(res, expected)
def testIHFFTExecution(self):
raw = np.random.rand(10, 20, 30)
t = tensor(raw, chunk_size=(4, 4, 30))
r = ihfft(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ihfft(raw)
np.testing.assert_allclose(res, expected)
r = ihfft(t, norm='ortho')
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ihfft(raw, norm='ortho')
np.testing.assert_allclose(res, expected)
r = ihfft(t, n=11)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ihfft(raw, n=11)
np.testing.assert_allclose(res, expected)
r = ihfft(t, n=12)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ihfft(raw, n=12)
np.testing.assert_allclose(res, expected)
def testFFTFreqExecution(self):
t = fftfreq(10, .1, chunk_size=3)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, np.fft.fftfreq(10, .1))
t = fftfreq(11, .01, chunk_size=3)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, np.fft.fftfreq(11, .01))
def testRFFTFreqExecution(self):
t = rfftfreq(20, .1, chunk_size=3)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, np.fft.rfftfreq(20, .1))
t = rfftfreq(21, .01, chunk_size=3)
res = self.executor.execute_tensor(t, concat=True)[0]
np.testing.assert_allclose(res, np.fft.rfftfreq(21, .01))
def testFFTShiftExecution(self):
t = fftfreq(10, .1, chunk_size=3)
r = fftshift(t)
res = self.executor.execute_tensor(r, concat=True)[0]
np.testing.assert_allclose(res, np.fft.fftshift(np.fft.fftfreq(10,
.1)))
freqs = fftfreq(9, d=1. / 9, chunk_size=2).reshape(3, 3)
r = fftshift(freqs, axes=(1, ))
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.fftshift(np.fft.fftfreq(9, d=1. / 9).reshape(3, 3),
axes=(1, ))
np.testing.assert_allclose(res, expected)
def testIFFTShiftExecution(self):
t = fftfreq(9, d=1. / 9, chunk_size=2).reshape(3, 3)
r = ifftshift(t)
res = self.executor.execute_tensor(r, concat=True)[0]
expected = np.fft.ifftshift(np.fft.fftfreq(9, d=1. / 9).reshape(3, 3))
np.testing.assert_allclose(res, expected)
class Test(TestBase):
def setUp(self):
super(Test, self).setUp()
self.executor = Executor()
def testAddWithoutShuffleExecution(self):
# all the axes are monotonic
# data1 with index split into [0...4], [5...9],
# columns [3...7], [8...12]
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=np.arange(3, 13))
df1 = from_pandas(data1, chunk_size=5)
# data2 with index split into [6...11], [2, 5],
# columns [4...9], [10, 13]
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=np.arange(4, 14))
df2 = from_pandas(data2, chunk_size=6)
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testAddWithOneShuffleExecution(self):
# only 1 axis is monotonic
# data1 with index split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=5)
# data2 with index split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=6)
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# only 1 axis is monotonic
# data1 with columns split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7],
columns=np.arange(10))
df1 = from_pandas(data1, chunk_size=5)
# data2 with columns split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2],
columns=np.arange(11, 1, -1))
df2 = from_pandas(data2, chunk_size=6)
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testAddWithAllShuffleExecution(self):
# no axis is monotonic
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[0, 10, 2, 3, 4, 5, 6, 7, 8, 9],
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[11, 1, 2, 5, 7, 6, 8, 9, 10, 3],
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=6)
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testAddBothWithOneChunk(self):
# only 1 axis is monotonic
# data1 with index split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=10)
# data2 with index split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=10)
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# only 1 axis is monotonic
# data1 with columns split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7],
columns=np.arange(10))
df1 = from_pandas(data1, chunk_size=10)
# data2 with columns split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2],
columns=np.arange(11, 1, -1))
df2 = from_pandas(data2, chunk_size=10)
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testAddWithoutShuffleAndWithOneChunk(self):
# only 1 axis is monotonic
# data1 with index split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=(5, 10))
# data2 with index split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=(6, 10))
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# only 1 axis is monotonic
# data1 with columns split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7],
columns=np.arange(10))
df1 = from_pandas(data1, chunk_size=(10, 5))
# data2 with columns split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2],
columns=np.arange(11, 1, -1))
df2 = from_pandas(data2, chunk_size=(10, 6))
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testAddWithShuffleAndWithOneChunk(self):
# only 1 axis is monotonic
# data1 with index split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(10),
columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7])
df1 = from_pandas(data1, chunk_size=(10, 5))
# data2 with index split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=np.arange(11, 1, -1),
columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2])
df2 = from_pandas(data2, chunk_size=(10, 6))
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
# only 1 axis is monotonic
# data1 with columns split into [0...4], [5...9],
data1 = pd.DataFrame(np.random.rand(10, 10),
index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7],
columns=np.arange(10))
df1 = from_pandas(data1, chunk_size=(5, 10))
# data2 with columns split into [6...11], [2, 5],
data2 = pd.DataFrame(np.random.rand(10, 10),
index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2],
columns=np.arange(11, 1, -1))
df2 = from_pandas(data2, chunk_size=(6, 10))
df3 = add(df1, df2)
expected = data1 + data2
result = self.executor.execute_dataframe(df3, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testAddWithAdded(self):
data1 = pd.DataFrame(np.random.rand(10, 10))
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(np.random.rand(10, 10))
df2 = from_pandas(data2, chunk_size=6)
df3 = add(df1, df2)
data4 = pd.DataFrame(np.random.rand(10, 10))
df4 = from_pandas(data4, chunk_size=6)
df5 = add(df3, df4)
result = self.executor.execute_dataframe(df5, concat=True)[0]
expected = data1 + data2 + data4
pd.testing.assert_frame_equal(expected, result)
def testRadd(self):
data1 = pd.DataFrame(np.random.rand(10, 10))
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(np.random.rand(10, 10))
df2 = from_pandas(data2, chunk_size=6)
radd = getattr(df2, '__radd__')
df3 = radd(df1, df2)
result = self.executor.execute_dataframe(df3, concat=True)[0]
expected = data1 + data2
pd.testing.assert_frame_equal(expected, result)
def testAddWithMultiForms(self):
# test multiple forms of add
# such as self+other, self.add(other), add(self,other)
data1 = pd.DataFrame(np.random.rand(10, 10))
df1 = from_pandas(data1, chunk_size=5)
data2 = pd.DataFrame(np.random.rand(10, 10))
df2 = from_pandas(data2, chunk_size=6)
expected = data1 + data2
result = self.executor.execute_dataframe(df1 + df2, concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
result = self.executor.execute_dataframe(add(df1, df2), concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
result = self.executor.execute_dataframe(df1.add(df2), concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
result = self.executor.execute_dataframe(df1.radd(df2), concat=True)[0]
pd.testing.assert_frame_equal(expected, result)
def testAbs(self):
data1 = pd.DataFrame(np.random.uniform(low=-1, high=1, size=(10, 10)))
df1 = from_pandas(data1, chunk_size=5)
result = self.executor.execute_dataframe(abs(df1), concat=True)[0]
expected = data1.abs()
pd.testing.assert_frame_equal(expected, result)
def setUp(self):
self.executor = Executor('numpy', storage=MockStorage(), prefetch=True)
