class Test(TestBase): def setUp(self): super(Test, self).setUp() self.executor = Executor() def testFromPandasDataFrameExecution(self): pdf = pd.DataFrame(np.random.rand(20, 30), index=[np.arange(20), np.arange(20, 0, -1)]) df = from_pandas_df(pdf, chunk_size=(13, 21)) result = self.executor.execute_dataframe(df, concat=True)[0] pd.testing.assert_frame_equal(pdf, result) def testFromPandasSeriesExecution(self): ps = pd.Series(np.random.rand(20), index=[np.arange(20), np.arange(20, 0, -1)], name='a') series = from_pandas_series(ps, chunk_size=13) result = self.executor.execute_dataframe(series, concat=True)[0] pd.testing.assert_series_equal(ps, result) def testInitializerExecution(self): pdf = pd.DataFrame(np.random.rand(20, 30), index=[np.arange(20), np.arange(20, 0, -1)]) df = md.DataFrame(pdf, chunk_size=(15, 10)) result = self.executor.execute_dataframe(df, concat=True)[0] pd.testing.assert_frame_equal(pdf, result) ps = pd.Series(np.random.rand(20), index=[np.arange(20), np.arange(20, 0, -1)], name='a') series = md.Series(ps, chunk_size=7) result = self.executor.execute_dataframe(series, concat=True)[0] pd.testing.assert_series_equal(ps, result) def testSeriesFromTensor(self): data = np.random.rand(10) series = md.Series(mt.tensor(data), name='a') pd.testing.assert_series_equal(series.execute(), pd.Series(data, name='a')) series = md.Series(mt.tensor(data, chunk_size=3)) pd.testing.assert_series_equal(series.execute(), pd.Series(data)) series = md.Series(mt.ones((10, ), chunk_size=4)) pd.testing.assert_series_equal(series.execute(), pd.Series(np.ones(10, ))) def testFromTensorExecution(self): tensor = mt.random.rand(10, 10, chunk_size=5) df = dataframe_from_tensor(tensor) tensor_res = self.executor.execute_tensor(tensor, concat=True)[0] pdf_expected = pd.DataFrame(tensor_res) df_result = self.executor.execute_dataframe(df, concat=True)[0] pd.testing.assert_index_equal(df_result.index, pd.RangeIndex(0, 10)) pd.testing.assert_index_equal(df_result.columns, pd.RangeIndex(0, 10)) pd.testing.assert_frame_equal(df_result, pdf_expected) # test converted with specified index_value and columns tensor2 = mt.random.rand(2, 2, chunk_size=1) df2 = dataframe_from_tensor(tensor2, index=pd.Index(['a', 'b']), columns=pd.Index([3, 4])) df_result = self.executor.execute_dataframe(df2, concat=True)[0] pd.testing.assert_index_equal(df_result.index, pd.Index(['a', 'b'])) pd.testing.assert_index_equal(df_result.columns, pd.Index([3, 4])) # test converted from 1-d tensor tensor3 = mt.array([1, 2, 3]) df3 = dataframe_from_tensor(tensor3) result3 = self.executor.execute_dataframe(df3, concat=True)[0] pdf_expected = pd.DataFrame(np.array([1, 2, 3])) pd.testing.assert_frame_equal(pdf_expected, result3) # test converted from identical chunks tensor4 = mt.ones((10, 10), chunk_size=3) df4 = dataframe_from_tensor(tensor4) result4 = self.executor.execute_dataframe(df4, concat=True)[0] pdf_expected = pd.DataFrame( self.executor.execute_tensor(tensor4, concat=True)[0]) pd.testing.assert_frame_equal(pdf_expected, result4) # from tensor with given index tensor5 = mt.ones((10, 10), chunk_size=3) df5 = dataframe_from_tensor(tensor5, index=np.arange(0, 20, 2)) result5 = self.executor.execute_dataframe(df5, concat=True)[0] pdf_expected = pd.DataFrame(self.executor.execute_tensor( tensor5, concat=True)[0], index=np.arange(0, 20, 2)) pd.testing.assert_frame_equal(pdf_expected, result5) # from tensor with given columns tensor6 = mt.ones((10, 10), chunk_size=3) df6 = dataframe_from_tensor(tensor6, columns=list('abcdefghij')) result6 = self.executor.execute_dataframe(df6, concat=True)[0] pdf_expected = pd.DataFrame(self.executor.execute_tensor( tensor6, concat=True)[0], columns=list('abcdefghij')) pd.testing.assert_frame_equal(pdf_expected, result6) def testFromRecordsExecution(self): dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')]) ndarr = np.ones((10, ), dtype=dtype) pdf_expected = pd.DataFrame.from_records(ndarr, index=pd.RangeIndex(10)) # from structured array of mars tensor = mt.ones((10, ), dtype=dtype, chunk_size=3) df1 = from_records(tensor) df1_result = self.executor.execute_dataframe(df1, concat=True)[0] pd.testing.assert_frame_equal(df1_result, pdf_expected) # from structured array of numpy df2 = from_records(ndarr) df2_result = self.executor.execute_dataframe(df2, concat=True)[0] pd.testing.assert_frame_equal(df2_result, pdf_expected) def testReadCSVExecution(self): tempdir = tempfile.mkdtemp() file_path = os.path.join(tempdir, 'test.csv') try: df = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]), columns=['a', 'b', 'c']) df.to_csv(file_path) pdf = pd.read_csv(file_path, index_col=0) mdf = self.executor.execute_dataframe(md.read_csv(file_path, index_col=0), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf) mdf2 = self.executor.execute_dataframe(md.read_csv(file_path, index_col=0, chunk_bytes=10), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf2) finally: shutil.rmtree(tempdir) # test sep tempdir = tempfile.mkdtemp() file_path = os.path.join(tempdir, 'test.csv') try: df = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]), columns=['a', 'b', 'c']) df.to_csv(file_path, sep=';') pdf = pd.read_csv(file_path, sep=';', index_col=0) mdf = self.executor.execute_dataframe(md.read_csv(file_path, sep=';', index_col=0), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf) mdf2 = self.executor.execute_dataframe(md.read_csv(file_path, sep=';', index_col=0, chunk_bytes=10), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf2) finally: shutil.rmtree(tempdir) # test missing value tempdir = tempfile.mkdtemp() file_path = os.path.join(tempdir, 'test.csv') try: df = pd.DataFrame({ 'c1': [np.nan, 'a', 'b', 'c'], 'c2': [1, 2, 3, np.nan], 'c3': [np.nan, np.nan, 3.4, 2.2] }) df.to_csv(file_path) pdf = pd.read_csv(file_path, index_col=0) mdf = self.executor.execute_dataframe(md.read_csv(file_path, index_col=0), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf) mdf2 = self.executor.execute_dataframe(md.read_csv(file_path, index_col=0, chunk_bytes=12), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf2) finally: shutil.rmtree(tempdir) tempdir = tempfile.mkdtemp() file_path = os.path.join(tempdir, 'test.csv') try: index = pd.date_range(start='1/1/2018', periods=100) df = pd.DataFrame( { 'col1': np.random.rand(100), 'col2': np.random.choice(['a', 'b', 'c'], (100, )), 'col3': np.arange(100) }, index=index) df.to_csv(file_path) pdf = pd.read_csv(file_path, index_col=0) mdf = self.executor.execute_dataframe(md.read_csv(file_path, index_col=0), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf) mdf2 = self.executor.execute_dataframe(md.read_csv( file_path, index_col=0, chunk_bytes=100), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf2) finally: shutil.rmtree(tempdir) # test compression tempdir = tempfile.mkdtemp() file_path = os.path.join(tempdir, 'test.gzip') try: index = pd.date_range(start='1/1/2018', periods=100) df = pd.DataFrame( { 'col1': np.random.rand(100), 'col2': np.random.choice(['a', 'b', 'c'], (100, )), 'col3': np.arange(100) }, index=index) df.to_csv(file_path, compression='gzip') pdf = pd.read_csv(file_path, compression='gzip', index_col=0) mdf = self.executor.execute_dataframe(md.read_csv( file_path, compression='gzip', index_col=0), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf) mdf2 = self.executor.execute_dataframe(md.read_csv( file_path, compression='gzip', index_col=0, chunk_bytes='1k'), concat=True)[0] pd.testing.assert_frame_equal(pdf, mdf2) finally: shutil.rmtree(tempdir) # test multiply files tempdir = tempfile.mkdtemp() try: df = pd.DataFrame(np.random.rand(300, 3), columns=['a', 'b', 'c']) file_paths = [ os.path.join(tempdir, 'test{}.csv'.format(i)) for i in range(3) ] df[:100].to_csv(file_paths[0]) df[100:200].to_csv(file_paths[1]) df[200:].to_csv(file_paths[2]) mdf = self.executor.execute_dataframe(md.read_csv(file_paths, index_col=0), concat=True)[0] pd.testing.assert_frame_equal(df, mdf) mdf2 = self.executor.execute_dataframe(md.read_csv(file_paths, index_col=0, chunk_bytes=50), concat=True)[0] pd.testing.assert_frame_equal(df, mdf2) finally: shutil.rmtree(tempdir)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testConcatenateExecution(self): a_data = np.random.rand(10, 20, 30) b_data = np.random.rand(10, 20, 40) c_data = np.random.rand(10, 20, 50) a = tensor(a_data, chunk_size=5) b = tensor(b_data, chunk_size=6) c = tensor(c_data, chunk_size=7) d = concatenate([a, b, c], axis=-1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.concatenate([a_data, b_data, c_data], axis=-1) self.assertTrue(np.array_equal(res, expected)) a_data = sps.random(10, 30) b_data = sps.rand(10, 40) c_data = sps.rand(10, 50) a = tensor(a_data, chunk_size=5) b = tensor(b_data, chunk_size=6) c = tensor(c_data, chunk_size=7) d = concatenate([a, b, c], axis=-1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.concatenate([a_data.A, b_data.A, c_data.A], axis=-1) self.assertTrue(np.array_equal(res.toarray(), expected)) def testStackExecution(self): raw = [np.random.randn(3, 4) for _ in range(10)] arrs = [tensor(a, chunk_size=3) for a in raw] arr2 = stack(arrs) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], np.stack(raw))) arr3 = stack(arrs, axis=1) res = self.executor.execute_tensor(arr3, concat=True) self.assertTrue(np.array_equal(res[0], np.stack(raw, axis=1))) arr4 = stack(arrs, axis=2) res = self.executor.execute_tensor(arr4, concat=True) self.assertTrue(np.array_equal(res[0], np.stack(raw, axis=2))) def testHStackExecution(self): a_data = np.random.rand(10) b_data = np.random.rand(20) a = tensor(a_data, chunk_size=4) b = tensor(b_data, chunk_size=4) c = hstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.hstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) a_data = np.random.rand(10, 20) b_data = np.random.rand(10, 5) a = tensor(a_data, chunk_size=3) b = tensor(b_data, chunk_size=4) c = hstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.hstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) def testVStackExecution(self): a_data = np.random.rand(10) b_data = np.random.rand(10) a = tensor(a_data, chunk_size=4) b = tensor(b_data, chunk_size=4) c = vstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.vstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) a_data = np.random.rand(10, 20) b_data = np.random.rand(5, 20) a = tensor(a_data, chunk_size=3) b = tensor(b_data, chunk_size=4) c = vstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.vstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) def testDStackExecution(self): a_data = np.random.rand(10) b_data = np.random.rand(10) a = tensor(a_data, chunk_size=4) b = tensor(b_data, chunk_size=4) c = dstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.dstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) a_data = np.random.rand(10, 20) b_data = np.random.rand(10, 20) a = tensor(a_data, chunk_size=3) b = tensor(b_data, chunk_size=4) c = dstack([a, b]) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.dstack([a_data, b_data]) self.assertTrue(np.array_equal(res, expected)) def testColumnStackExecution(self): a_data = np.array((1, 2, 3)) b_data = np.array((2, 3, 4)) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=2) c = column_stack((a, b)) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.column_stack((a_data, b_data)) np.testing.assert_equal(res, expected) a_data = np.random.rand(4, 2, 3) b_data = np.random.rand(4, 2, 3) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=2) c = column_stack((a, b)) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.column_stack((a_data, b_data)) np.testing.assert_equal(res, expected)
def setUp(self): super(Test, self).setUp() self.executor = Executor()
class Test(TestBase): def setUp(self): super(Test, self).setUp() self.executor = Executor() def testFromPandasDataFrameExecution(self): pdf = pd.DataFrame(np.random.rand(20, 30), index=[np.arange(20), np.arange(20, 0, -1)]) df = from_pandas_df(pdf, chunk_size=(13, 21)) result = self.executor.execute_dataframe(df, concat=True)[0] pd.testing.assert_frame_equal(pdf, result) def testFromPandasSeriesExecution(self): ps = pd.Series(np.random.rand(20), index=[np.arange(20), np.arange(20, 0, -1)], name='a') series = from_pandas_series(ps, chunk_size=13) result = self.executor.execute_dataframe(series, concat=True)[0] pd.testing.assert_series_equal(ps, result) def testInitializerExecution(self): pdf = pd.DataFrame(np.random.rand(20, 30), index=[np.arange(20), np.arange(20, 0, -1)]) df = md.DataFrame(pdf, chunk_size=(15, 10)) result = self.executor.execute_dataframe(df, concat=True)[0] pd.testing.assert_frame_equal(pdf, result) ps = pd.Series(np.random.rand(20), index=[np.arange(20), np.arange(20, 0, -1)], name='a') series = md.Series(ps, chunk_size=7) result = self.executor.execute_dataframe(series, concat=True)[0] pd.testing.assert_series_equal(ps, result) def testFromTensorExecution(self): tensor = mt.random.rand(10, 10, chunk_size=5) df = from_tensor(tensor) tensor_res = self.executor.execute_tensor(tensor, concat=True)[0] pdf_expected = pd.DataFrame(tensor_res) df_result = self.executor.execute_dataframe(df, concat=True)[0] pd.testing.assert_index_equal(df_result.index, pd.RangeIndex(0, 10)) pd.testing.assert_index_equal(df_result.columns, pd.RangeIndex(0, 10)) pd.testing.assert_frame_equal(df_result, pdf_expected) # test converted with specified index_value and columns tensor2 = mt.random.rand(2, 2, chunk_size=1) df2 = from_tensor(tensor2, index=pd.Index(['a', 'b']), columns=pd.Index([3, 4])) df_result = self.executor.execute_dataframe(df2, concat=True)[0] pd.testing.assert_index_equal(df_result.index, pd.Index(['a', 'b'])) pd.testing.assert_index_equal(df_result.columns, pd.Index([3, 4])) # test converted from 1-d tensor tensor3 = mt.array([1, 2, 3]) df3 = from_tensor(tensor3) result3 = self.executor.execute_dataframe(df3, concat=True)[0] pdf_expected = pd.DataFrame(np.array([1, 2, 3])) pd.testing.assert_frame_equal(pdf_expected, result3) def testFromRecordsExecution(self): dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')]) ndarr = np.ones((10, ), dtype=dtype) pdf_expected = pd.DataFrame.from_records(ndarr, index=pd.RangeIndex(10)) # from structured array of mars tensor = mt.ones((10, ), dtype=dtype, chunk_size=3) df1 = from_records(tensor) df1_result = self.executor.execute_dataframe(df1, concat=True)[0] pd.testing.assert_frame_equal(df1_result, pdf_expected) # from structured array of numpy df2 = from_records(ndarr) df2_result = self.executor.execute_dataframe(df2, concat=True)[0] pd.testing.assert_frame_equal(df2_result, pdf_expected)
def __init__(self): self._executor = Executor( sync_provider_type=Executor.SyncProviderType.GEVENT)
class TestBinary(TestBase): def setUp(self): self.executor = Executor() @property def rfunc_name(self): return 'r' + self.func_name def testWithoutShuffleExecution(self): # all the axes are monotonic # data1 with index split into [0...4], [5...9], # columns [3...7], [8...12] data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=np.arange(3, 13)) df1 = from_pandas(data1, chunk_size=5) # data2 with index split into [6...11], [2, 5], # columns [4...9], [10, 13] data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=np.arange(4, 14)) df2 = from_pandas(data2, chunk_size=6) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testWithOneShuffleExecution(self): # only 1 axis is monotonic # data1 with index split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=5) # data2 with index split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=6) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # only 1 axis is monotonic # data1 with columns split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7], columns=np.arange(10)) df1 = from_pandas(data1, chunk_size=5) # data2 with columns split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2], columns=np.arange(11, 1, -1)) df2 = from_pandas(data2, chunk_size=6) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testWithAllShuffleExecution(self): # no axis is monotonic data1 = pd.DataFrame(np.random.rand(10, 10), index=[0, 10, 2, 3, 4, 5, 6, 7, 8, 9], columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame(np.random.rand(10, 10), index=[11, 1, 2, 5, 7, 6, 8, 9, 10, 3], columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=6) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testBothWithOneChunk(self): # only 1 axis is monotonic # data1 with index split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=10) # data2 with index split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=10) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # only 1 axis is monotonic # data1 with columns split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7], columns=np.arange(10)) df1 = from_pandas(data1, chunk_size=10) # data2 with columns split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2], columns=np.arange(11, 1, -1)) df2 = from_pandas(data2, chunk_size=10) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testWithoutShuffleAndWithOneChunk(self): # only 1 axis is monotonic # data1 with index split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=(5, 10)) # data2 with index split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=(6, 10)) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # only 1 axis is monotonic # data1 with columns split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7], columns=np.arange(10)) df1 = from_pandas(data1, chunk_size=(10, 5)) # data2 with columns split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2], columns=np.arange(11, 1, -1)) df2 = from_pandas(data2, chunk_size=(10, 6)) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testWithShuffleAndWithOneChunk(self): # only 1 axis is monotonic # data1 with index split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=(10, 5)) # data2 with index split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=(10, 6)) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # only 1 axis is monotonic # data1 with columns split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7], columns=np.arange(10)) df1 = from_pandas(data1, chunk_size=(5, 10)) # data2 with columns split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2], columns=np.arange(11, 1, -1)) df2 = from_pandas(data2, chunk_size=(6, 10)) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testSameIndex(self): data = pd.DataFrame(np.random.rand(10, 10), index=np.random.randint(0, 2, size=(10, )), columns=['c' + str(i) for i in range(10)]) df = from_pandas(data, chunk_size=3) df2 = self.func(df, df) expected = self.func(data, data) result = self.executor.execute_dataframe(df2, concat=True)[0] pd.testing.assert_frame_equal(expected, result) series = from_pandas_series(data.iloc[0], chunk_size=3) df3 = self.func(df, series) expected = self.func(data, data.iloc[0]) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) series = from_pandas_series(data.iloc[:, 0], chunk_size=3) df4 = getattr(df, self.func_name)(series, axis=0) expected = getattr(data, self.func_name)(data.iloc[:, 0], axis=0) result = self.executor.execute_dataframe(df4, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testChained(self): data1 = pd.DataFrame(np.random.rand(10, 10)) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame(np.random.rand(10, 10)) df2 = from_pandas(data2, chunk_size=6) df3 = self.func(df1, df2) data4 = pd.DataFrame(np.random.rand(10, 10)) df4 = from_pandas(data4, chunk_size=6) df5 = self.func(df3, df4) result = self.executor.execute_dataframe(df5, concat=True)[0] expected = self.func(self.func(data1, data2), data4) pd.testing.assert_frame_equal(expected, result) def testRfunc(self): data1 = pd.DataFrame(np.random.rand(10, 10)) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame(np.random.rand(10, 10)) df2 = from_pandas(data2, chunk_size=6) df3 = getattr(df1, self.rfunc_name)(df2) result = self.executor.execute_dataframe(df3, concat=True)[0] expected = self.func(data2, data1) pd.testing.assert_frame_equal(expected, result) data3 = pd.DataFrame(np.random.rand(10, 10)) df4 = from_pandas(data3, chunk_size=5) df5 = getattr(df4, self.rfunc_name)(1) result = self.executor.execute_dataframe(df5, concat=True)[0] expected2 = self.func(1, data3) pd.testing.assert_frame_equal(expected2, result) def testWithMultiForms(self): # test multiple forms # such as self+other, self.add(other), add(self,other) data1 = pd.DataFrame(np.random.rand(10, 10)) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame(np.random.rand(10, 10)) df2 = from_pandas(data2, chunk_size=6) expected = self.func(data1, data2) result = self.executor.execute_dataframe(self.func(df1, df2), concat=True)[0] pd.testing.assert_frame_equal(expected, result) result = self.executor.execute_dataframe(self.func(df1, df2), concat=True)[0] pd.testing.assert_frame_equal(expected, result) result = self.executor.execute_dataframe(getattr(df1, self.func_name)(df2), concat=True)[0] pd.testing.assert_frame_equal(expected, result) result = self.executor.execute_dataframe(getattr(df1, self.rfunc_name)(df2), concat=True)[0] pd.testing.assert_frame_equal(self.func(data2, data1), result) def testDataframeAndScalar(self): # test dataframe and scalar pdf = pd.DataFrame(np.random.rand(10, 10)) df = from_pandas(pdf, chunk_size=2) expected = self.func(pdf, 1) result = self.executor.execute_dataframe(self.func(df, 1), concat=True)[0] pd.testing.assert_frame_equal(expected, result) result2 = self.executor.execute_dataframe(self.func(df, 1), concat=True)[0] pd.testing.assert_frame_equal(expected, result2) result3 = self.executor.execute_dataframe(getattr(df, self.func_name)(1), concat=True)[0] pd.testing.assert_frame_equal(expected, result3) # test scalar and dataframe result4 = self.executor.execute_dataframe(self.func(df, 1), concat=True)[0] pd.testing.assert_frame_equal(expected, result4) expected2 = self.func(1, pdf) result5 = self.executor.execute_dataframe(self.func(1, df), concat=True)[0] pd.testing.assert_frame_equal(expected2, result5) result6 = self.executor.execute_dataframe(getattr(df, self.rfunc_name)(1), concat=True)[0] pd.testing.assert_frame_equal(expected2, result6) def testWithShuffleOnStringIndex(self): # no axis is monotonic, and the index values are strings. data1 = pd.DataFrame( np.random.rand(10, 10), index=[str(x) for x in [0, 10, 2, 3, 4, 5, 6, 7, 8, 9]], columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame( np.random.rand(10, 10), index=[str(x) for x in [11, 1, 2, 5, 7, 6, 8, 9, 10, 3]], columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=6) df3 = self.func(df1, df2) expected = self.func(data1, data2) result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testDataframeAndSeries(self): data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) s1 = from_pandas_series(data2[1], chunk_size=(6, )) # operate on single-column dataframe and series df1 = from_pandas(data1[[1]], chunk_size=(5, 5)) r1 = getattr(df1, self.func_name)(s1, axis='index') expected = getattr(data1[[1]], self.func_name)(data2[1], axis='index') result = self.executor.execute_dataframe(r1, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # operate on dataframe and series without shuffle df2 = from_pandas(data1, chunk_size=(5, 5)) r2 = getattr(df2, self.func_name)(s1, axis='index') expected = getattr(data1, self.func_name)(data2[1], axis='index') result = self.executor.execute_dataframe(r2, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # operate on dataframe and series with shuffle df3 = from_pandas(data1, chunk_size=(5, 5)) r3 = getattr(df3, self.func_name)(s1, axis='columns') expected = getattr(data1, self.func_name)(data2[1], axis='columns') result = self.executor.execute_dataframe(r3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # test both one chunk, axis=0 pdf = pd.DataFrame({ 'ca': [1, 3, 2], 'cb': [360, 180, 2] }, index=[1, 2, 3]) df = from_pandas(pdf) series = pd.Series([0, 1, 2], index=[1, 2, 3]) mars_series = from_pandas_series(series) result = self.executor.execute_dataframe(getattr(df, self.func_name)( mars_series, axis=0), concat=True)[0] expected = getattr(pdf, self.func_name)(series, axis=0) pd.testing.assert_frame_equal(expected, result) # test different number of chunks, axis=0 pdf = pd.DataFrame({ 'ca': [1, 3, 2], 'cb': [360, 180, 2] }, index=[1, 2, 3]) df = from_pandas(pdf, chunk_size=1) series = pd.Series([0, 1, 2], index=[1, 2, 3]) mars_series = from_pandas_series(series) result = self.executor.execute_dataframe(getattr(df, self.func_name)( mars_series, axis=0), concat=True)[0] expected = getattr(pdf, self.func_name)(series, axis=0) pd.testing.assert_frame_equal(expected, result) # test with row shuffle, axis=0 pdf = pd.DataFrame({ 'ca': [1, 3, 2], 'cb': [360, 180, 2] }, index=[2, 1, 3]) df = from_pandas(pdf, chunk_size=1) series = pd.Series([0, 1, 2], index=[3, 1, 2]) mars_series = from_pandas_series(series) result = self.executor.execute_dataframe(getattr(df, self.func_name)( mars_series, axis=0), concat=True)[0] expected = getattr(pdf, self.func_name)(series, axis=0).reindex([3, 1, 2]) # modify the order of rows result = result.reindex(index=[3, 1, 2]) pd.testing.assert_frame_equal(expected, result) # test both one chunk, axis=1 pdf = pd.DataFrame({ 1: [1, 3, 2], 2: [360, 180, 2], 3: [1, 2, 3] }, index=['ra', 'rb', 'rc']) df = from_pandas(pdf) series = pd.Series([0, 1, 2], index=[1, 2, 3]) mars_series = from_pandas_series(series) result = self.executor.execute_dataframe(getattr(df, self.func_name)( mars_series, axis=1), concat=True)[0] expected = getattr(pdf, self.func_name)(series, axis=1) pd.testing.assert_frame_equal(expected, result) # test different number of chunks, axis=1 pdf = pd.DataFrame({ 1: [1, 3, 2], 2: [360, 180, 2], 3: [1, 2, 3] }, index=['ra', 'rb', 'rc']) df = from_pandas(pdf, chunk_size=1) series = pd.Series([0, 1, 2], index=[1, 2, 3]) mars_series = from_pandas_series(series) result = self.executor.execute_dataframe(getattr(df, self.func_name)( mars_series, axis=1), concat=True)[0] expected = getattr(pdf, self.func_name)(series, axis=1) pd.testing.assert_frame_equal(expected, result) # test with row shuffle, axis=1 pdf = pd.DataFrame({ 1: [1, 3, 2], 3: [1, 2, 3], 2: [360, 180, 2] }, index=['ra', 'rb', 'rc']) df = from_pandas(pdf, chunk_size=1) series = pd.Series([0, 1, 2], index=[3, 1, 2]) mars_series = from_pandas_series(series) result = self.executor.execute_dataframe(getattr(df, self.func_name)( mars_series, axis=1), concat=True)[0] expected = getattr(pdf, self.func_name)(series, axis=1) # modify the order of columns result = result[[1, 2, 3]] pd.testing.assert_frame_equal(expected, result) def testSeries(self): # only one chunk s1 = pd.Series(np.arange(10) + 1) s2 = pd.Series(np.arange(10) + 1) r = self.func(from_pandas_series(s1, chunk_size=10), from_pandas_series(s2, chunk_size=10)) result = self.executor.execute_dataframe(r, concat=True)[0] expected = self.func(s1, s2) pd.testing.assert_series_equal(expected, result) # same index s1 = pd.Series(np.arange(10) + 1) s2 = pd.Series(np.arange(10) + 1) r = self.func(from_pandas_series(s1, chunk_size=4), from_pandas_series(s2, chunk_size=6)) result = self.executor.execute_dataframe(r, concat=True)[0] expected = self.func(s1, s2) pd.testing.assert_series_equal(expected, result) # no shuffle s1 = pd.Series(np.arange(10) + 1, index=range(10)) s2 = pd.Series(np.arange(10) + 1, index=range(10, 0, -1)) r = self.func(from_pandas_series(s1, chunk_size=4), from_pandas_series(s2, chunk_size=6)) result = self.executor.execute_dataframe(r, concat=True)[0] expected = self.func(s1, s2) pd.testing.assert_series_equal(expected, result) # shuffle s1 = pd.Series(np.arange(10) + 1, index=np.random.permutation(range(10))) s2 = pd.Series(np.arange(10) + 1, index=np.random.permutation(range(10, 0, -1))) r = self.func(from_pandas_series(s1, chunk_size=4), from_pandas_series(s2, chunk_size=6)) result = self.executor.execute_dataframe(r, concat=True)[0] expected = self.func(s1, s2) pd.testing.assert_series_equal(expected, result) # operate with scalar s1 = pd.Series(np.arange(10) + 1, index=np.random.permutation(range(10))) r = self.func(from_pandas_series(s1, chunk_size=4), 4) result = self.executor.execute_dataframe(r, concat=True)[0] expected = self.func(s1, 4) pd.testing.assert_series_equal(expected, result) # reverse with scalar s1 = pd.Series(np.arange(10) + 1, index=np.random.permutation(range(10))) r = self.func(4, from_pandas_series(s1, chunk_size=4)) result = self.executor.execute_dataframe(r, concat=True)[0] expected = self.func(4, s1) pd.testing.assert_series_equal(expected, result) def testWithPlainValue(self): data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=6) s1 = df1[2] r = getattr(df1, self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], axis=0) result = self.executor.execute_dataframe(r, concat=True)[0] expected = getattr(data1, self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], axis=0) pd.testing.assert_frame_equal(expected, result) r = getattr(df1, self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10), axis=0) result = self.executor.execute_dataframe(r, concat=True)[0] expected = getattr(data1, self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10), axis=0) pd.testing.assert_frame_equal(expected, result) r = getattr(s1, self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) result = self.executor.execute_dataframe(r, concat=True)[0] expected = getattr(data1[2], self.func_name)([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) pd.testing.assert_series_equal(expected, result) r = getattr(s1, self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) result = self.executor.execute_dataframe(r, concat=True)[0] expected = getattr(data1[2], self.func_name)((1, 2, 3, 4, 5, 6, 7, 8, 9, 10)) pd.testing.assert_series_equal(expected, result) # specify index, not the default range index data1 = pd.DataFrame(np.random.rand(10, 7), index=np.arange(5, 15), columns=[4, 1, 3, 2, 5, 6, 7]) df1 = from_pandas(data1, chunk_size=6) s1 = df1[2] r = getattr(df1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0) result = self.executor.execute_dataframe(r, concat=True)[0] expected = getattr(data1, self.func_name)(np.array( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0) pd.testing.assert_frame_equal(expected, result) r = getattr(df1, self.func_name)(from_array( np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])), axis=0) result = self.executor.execute_dataframe(r, concat=True)[0] expected = getattr(data1, self.func_name)(np.array( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), axis=0) pd.testing.assert_frame_equal(expected, result) r = getattr(s1, self.func_name)(np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])) result = self.executor.execute_dataframe(r, concat=True)[0] expected = getattr(data1[2], self.func_name)(np.array( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])) pd.testing.assert_series_equal(expected, result) r = getattr(s1, self.func_name)(from_array( np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]))) result = self.executor.execute_dataframe(r, concat=True)[0] expected = getattr(data1[2], self.func_name)(np.array( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])) pd.testing.assert_series_equal(expected, result)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testRechunkExecution(self): raw = np.random.random((11, 8)) arr = tensor(raw, chunk_size=3) arr2 = arr.rechunk(4) res = self.executor.execute_tensor(arr2) self.assertTrue(np.array_equal(res[0], raw[:4, :4])) self.assertTrue(np.array_equal(res[1], raw[:4, 4:])) self.assertTrue(np.array_equal(res[2], raw[4:8, :4])) self.assertTrue(np.array_equal(res[3], raw[4:8, 4:])) self.assertTrue(np.array_equal(res[4], raw[8:, :4])) self.assertTrue(np.array_equal(res[5], raw[8:, 4:])) def testCopytoExecution(self): a = ones((2, 3), chunk_size=1) b = tensor([3, -1, 3], chunk_size=2) copyto(a, b, where=b > 1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.array([[3, 1, 3], [3, 1, 3]]) np.testing.assert_equal(res, expected) a = ones((2, 3), chunk_size=1) b = tensor(np.asfortranarray(np.random.rand(2, 3)), chunk_size=2) copyto(b, a) res = self.executor.execute_tensor(b, concat=True)[0] expected = np.asfortranarray(np.ones((2, 3))) np.testing.assert_array_equal(res, expected) self.assertTrue(res.flags['F_CONTIGUOUS']) self.assertFalse(res.flags['C_CONTIGUOUS']) def testAstypeExecution(self): raw = np.random.random((10, 5)) arr = tensor(raw, chunk_size=3) arr2 = arr.astype('i8') res = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_array_equal(res[0], raw.astype('i8')) raw = sps.random(10, 5, density=.2) arr = tensor(raw, chunk_size=3) arr2 = arr.astype('i8') res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue( np.array_equal(res[0].toarray(), raw.astype('i8').toarray())) raw = np.asfortranarray(np.random.random((10, 5))) arr = tensor(raw, chunk_size=3) arr2 = arr.astype('i8', order='C') res = self.executor.execute_tensor(arr2, concat=True)[0] np.testing.assert_array_equal(res, raw.astype('i8')) self.assertTrue(res.flags['C_CONTIGUOUS']) self.assertFalse(res.flags['F_CONTIGUOUS']) def testTransposeExecution(self): raw = np.random.random((11, 8, 5)) arr = tensor(raw, chunk_size=3) arr2 = transpose(arr) res = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_array_equal(res[0], raw.T) arr3 = transpose(arr, axes=(-2, -1, -3)) res = self.executor.execute_tensor(arr3, concat=True) np.testing.assert_array_equal(res[0], raw.transpose(1, 2, 0)) raw = sps.random(11, 8) arr = tensor(raw, chunk_size=3) arr2 = transpose(arr) self.assertTrue(arr2.issparse()) res = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_array_equal(res[0].toarray(), raw.T.toarray()) # test order raw = np.asfortranarray(np.random.random((11, 8, 5))) arr = tensor(raw, chunk_size=3) arr2 = transpose(arr) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = np.transpose(raw).copy(order='A') np.testing.assert_array_equal(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) arr = tensor(raw, chunk_size=3) arr2 = transpose(arr, (1, 2, 0)) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = np.transpose(raw, (1, 2, 0)).copy(order='A') np.testing.assert_array_equal(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) def testSwapaxesExecution(self): raw = np.random.random((11, 8, 5)) arr = tensor(raw, chunk_size=3) arr2 = arr.swapaxes(2, 0) res = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_array_equal(res[0], raw.swapaxes(2, 0)) raw = sps.random(11, 8, density=.2) arr = tensor(raw, chunk_size=3) arr2 = arr.swapaxes(1, 0) res = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_array_equal(res[0].toarray(), raw.toarray().swapaxes(1, 0)) # test order raw = np.asfortranarray(np.random.rand(11, 8, 5)) arr = tensor(raw, chunk_size=3) arr2 = arr.swapaxes(2, 0) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = raw.swapaxes(2, 0).copy(order='A') np.testing.assert_array_equal(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) arr = tensor(raw, chunk_size=3) arr2 = arr.swapaxes(0, 2) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = raw.swapaxes(0, 2).copy(order='A') np.testing.assert_array_equal(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) arr = tensor(raw, chunk_size=3) arr2 = arr.swapaxes(1, 0) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = raw.swapaxes(1, 0).copy(order='A') np.testing.assert_array_equal(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) def testMoveaxisExecution(self): x = zeros((3, 4, 5), chunk_size=2) t = moveaxis(x, 0, -1) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (4, 5, 3)) t = moveaxis(x, -1, 0) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 3, 4)) t = moveaxis(x, [0, 1], [-1, -2]) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 4, 3)) t = moveaxis(x, [0, 1, 2], [-1, -2, -3]) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 4, 3)) def testBroadcastToExecution(self): raw = np.random.random((10, 5, 1)) arr = tensor(raw, chunk_size=2) arr2 = broadcast_to(arr, (5, 10, 5, 6)) res = self.executor.execute_tensor(arr2, concat=True)[0] np.testing.assert_array_equal(res, np.broadcast_to(raw, (5, 10, 5, 6))) # test chunk with unknown shape arr1 = mt.random.rand(3, 4, chunk_size=2) arr2 = mt.random.permutation(arr1) arr3 = broadcast_to(arr2, (2, 3, 4)) res = self.executor.execute_tensor(arr3, concat=True)[0] self.assertEqual(res.shape, (2, 3, 4)) def testBroadcastArraysExecutions(self): x_data = [[1, 2, 3]] x = tensor(x_data, chunk_size=1) y_data = [[1], [2], [3]] y = tensor(y_data, chunk_size=2) a = broadcast_arrays(x, y) res = [self.executor.execute_tensor(arr, concat=True)[0] for arr in a] expected = np.broadcast_arrays(x_data, y_data) for r, e in zip(res, expected): np.testing.assert_equal(r, e) def testWhereExecution(self): raw_cond = np.random.randint(0, 2, size=(4, 4), dtype='?') raw_x = np.random.rand(4, 1) raw_y = np.random.rand(4, 4) cond, x, y = tensor(raw_cond, chunk_size=2), tensor( raw_x, chunk_size=2), tensor(raw_y, chunk_size=2) arr = where(cond, x, y) res = self.executor.execute_tensor(arr, concat=True) self.assertTrue( np.array_equal(res[0], np.where(raw_cond, raw_x, raw_y))) raw_cond = sps.csr_matrix( np.random.randint(0, 2, size=(4, 4), dtype='?')) raw_x = sps.random(4, 1, density=.1) raw_y = sps.random(4, 4, density=.1) cond, x, y = tensor(raw_cond, chunk_size=2), tensor( raw_x, chunk_size=2), tensor(raw_y, chunk_size=2) arr = where(cond, x, y) res = self.executor.execute_tensor(arr, concat=True)[0] self.assertTrue( np.array_equal( res.toarray(), np.where(raw_cond.toarray(), raw_x.toarray(), raw_y.toarray()))) def testReshapeExecution(self): raw_data = np.random.rand(10, 20, 30) x = tensor(raw_data, chunk_size=6) y = x.reshape(-1, 30) res = self.executor.execute_tensor(y, concat=True) np.testing.assert_array_equal(res[0], raw_data.reshape(-1, 30)) y2 = x.reshape(10, -1) res = self.executor.execute_tensor(y2, concat=True) np.testing.assert_array_equal(res[0], raw_data.reshape(10, -1)) y3 = x.reshape(-1) res = self.executor.execute_tensor(y3, concat=True) np.testing.assert_array_equal(res[0], raw_data.reshape(-1)) y4 = x.ravel() res = self.executor.execute_tensor(y4, concat=True) np.testing.assert_array_equal(res[0], raw_data.ravel()) raw_data = np.random.rand(30, 100, 20) x = tensor(raw_data, chunk_size=6) y = x.reshape(-1, 20, 5, 5, 4) res = self.executor.execute_tensor(y, concat=True) np.testing.assert_array_equal(res[0], raw_data.reshape(-1, 20, 5, 5, 4)) y2 = x.reshape(3000, 10, 2) res = self.executor.execute_tensor(y2, concat=True) np.testing.assert_array_equal(res[0], raw_data.reshape(3000, 10, 2)) y3 = x.reshape(60, 25, 40) res = self.executor.execute_tensor(y3, concat=True) np.testing.assert_array_equal(res[0], raw_data.reshape(60, 25, 40)) y4 = x.reshape(60, 25, 40) y4.op.extra_params['_reshape_with_shuffle'] = True size_res = self.executor.execute_tensor(y4, mock=True) res = self.executor.execute_tensor(y4, concat=True) self.assertEqual(res[0].nbytes, sum(v[0] for v in size_res)) self.assertTrue(np.array_equal(res[0], raw_data.reshape(60, 25, 40))) y5 = x.ravel(order='F') res = self.executor.execute_tensor(y5, concat=True)[0] expected = raw_data.ravel(order='F') np.testing.assert_array_equal(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) def testExpandDimsExecution(self): raw_data = np.random.rand(10, 20, 30) x = tensor(raw_data, chunk_size=6) y = expand_dims(x, 1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 1))) y = expand_dims(x, 0) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 0))) y = expand_dims(x, 3) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 3))) y = expand_dims(x, -1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -1))) y = expand_dims(x, -4) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -4))) with self.assertRaises(np.AxisError): expand_dims(x, -5) with self.assertRaises(np.AxisError): expand_dims(x, 4) def testRollAxisExecution(self): x = ones((3, 4, 5, 6), chunk_size=1) y = rollaxis(x, 3, 1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue( np.array_equal(res[0], np.rollaxis(np.ones((3, 4, 5, 6)), 3, 1))) def testAtleast1dExecution(self): x = 1 y = ones(3, chunk_size=2) z = ones((3, 4), chunk_size=2) t = atleast_1d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.array([1]))) self.assertTrue(np.array_equal(res[1], np.ones(3))) self.assertTrue(np.array_equal(res[2], np.ones((3, 4)))) def testAtleast2dExecution(self): x = 1 y = ones(3, chunk_size=2) z = ones((3, 4), chunk_size=2) t = atleast_2d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.array([[1]]))) self.assertTrue(np.array_equal(res[1], np.atleast_2d(np.ones(3)))) self.assertTrue(np.array_equal(res[2], np.ones((3, 4)))) def testAtleast3dExecution(self): x = 1 y = ones(3, chunk_size=2) z = ones((3, 4), chunk_size=2) t = atleast_3d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.atleast_3d(x))) self.assertTrue(np.array_equal(res[1], np.atleast_3d(np.ones(3)))) self.assertTrue(np.array_equal(res[2], np.atleast_3d(np.ones((3, 4))))) def testArgwhereExecution(self): x = arange(6, chunk_size=2).reshape(2, 3) t = argwhere(x > 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.argwhere(np.arange(6).reshape(2, 3) > 1) np.testing.assert_array_equal(res, expected) data = np.asfortranarray(np.random.rand(10, 20)) x = tensor(data, chunk_size=10) t = argwhere(x > 0.5) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.argwhere(data > 0.5) np.testing.assert_array_equal(res, expected) self.assertTrue(res.flags['F_CONTIGUOUS']) self.assertFalse(res.flags['C_CONTIGUOUS']) def testArraySplitExecution(self): x = arange(48, chunk_size=3).reshape(2, 3, 8) ss = array_split(x, 3, axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.array_split(np.arange(48).reshape(2, 3, 8), 3, axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] ss = array_split(x, [3, 5, 6, 10], axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.array_split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10], axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] def testSplitExecution(self): x = arange(48, chunk_size=3).reshape(2, 3, 8) ss = split(x, 4, axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(np.arange(48).reshape(2, 3, 8), 4, axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] ss = split(x, [3, 5, 6, 10], axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10], axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # hsplit x = arange(120, chunk_size=3).reshape(2, 12, 5) ss = hsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.hsplit(np.arange(120).reshape(2, 12, 5), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # vsplit x = arange(48, chunk_size=3).reshape(8, 3, 2) ss = vsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.vsplit(np.arange(48).reshape(8, 3, 2), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # dsplit x = arange(48, chunk_size=3).reshape(2, 3, 8) ss = dsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.dsplit(np.arange(48).reshape(2, 3, 8), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] x_data = sps.random(12, 8, density=.1) x = tensor(x_data, chunk_size=3) ss = split(x, 4, axis=0) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(x_data.toarray(), 4, axis=0) self.assertEqual(len(res), len(expected)) [ np.testing.assert_equal(r.toarray(), e) for r, e in zip(res, expected) ] def testRollExecution(self): x = arange(10, chunk_size=2) t = roll(x, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10), 2) np.testing.assert_equal(res, expected) x2 = x.reshape(2, 5) t = roll(x2, 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1) np.testing.assert_equal(res, expected) t = roll(x2, 1, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=0) np.testing.assert_equal(res, expected) t = roll(x2, 1, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=1) np.testing.assert_equal(res, expected) def testSqueezeExecution(self): data = np.array([[[0], [1], [2]]]) x = tensor(data, chunk_size=1) t = squeeze(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.squeeze(data) np.testing.assert_equal(res, expected) t = squeeze(x, axis=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.squeeze(data, axis=2) np.testing.assert_equal(res, expected) def testPtpExecution(self): x = arange(4, chunk_size=1).reshape(2, 2) t = ptp(x, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ptp(np.arange(4).reshape(2, 2), axis=0) np.testing.assert_equal(res, expected) t = ptp(x, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ptp(np.arange(4).reshape(2, 2), axis=1) np.testing.assert_equal(res, expected) t = ptp(x) res = self.executor.execute_tensor(t)[0] expected = np.ptp(np.arange(4).reshape(2, 2)) np.testing.assert_equal(res, expected) def testDiffExecution(self): data = np.array([1, 2, 4, 7, 0]) x = tensor(data, chunk_size=2) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data) np.testing.assert_equal(res, expected) t = diff(x, n=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data, n=2) np.testing.assert_equal(res, expected) data = np.array([[1, 3, 6, 10], [0, 5, 6, 8]]) x = tensor(data, chunk_size=2) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data) np.testing.assert_equal(res, expected) t = diff(x, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data, axis=0) np.testing.assert_equal(res, expected) x = mt.arange('1066-10-13', '1066-10-16', dtype=mt.datetime64) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff( np.arange('1066-10-13', '1066-10-16', dtype=np.datetime64)) np.testing.assert_equal(res, expected) def testEdiff1d(self): data = np.array([1, 2, 4, 7, 0]) x = tensor(data, chunk_size=2) t = ediff1d(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data) np.testing.assert_equal(res, expected) to_begin = tensor(-99, chunk_size=2) to_end = tensor([88, 99], chunk_size=2) t = ediff1d(x, to_begin=to_begin, to_end=to_end) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data, to_begin=-99, to_end=np.array([88, 99])) np.testing.assert_equal(res, expected) data = [[1, 2, 4], [1, 6, 24]] t = ediff1d(tensor(data, chunk_size=2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data) np.testing.assert_equal(res, expected) def testDigitizeExecution(self): data = np.array([0.2, 6.4, 3.0, 1.6]) x = tensor(data, chunk_size=2) bins = np.array([0.0, 1.0, 2.5, 4.0, 10.0]) inds = digitize(x, bins) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins) np.testing.assert_equal(res, expected) b = tensor(bins, chunk_size=2) inds = digitize(x, b) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins) np.testing.assert_equal(res, expected) data = np.array([1.2, 10.0, 12.4, 15.5, 20.]) x = tensor(data, chunk_size=2) bins = np.array([0, 5, 10, 15, 20]) inds = digitize(x, bins, right=True) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins, right=True) np.testing.assert_equal(res, expected) inds = digitize(x, bins, right=False) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins, right=False) np.testing.assert_equal(res, expected) data = sps.random(10, 1, density=.1) * 12 x = tensor(data, chunk_size=2) bins = np.array([1.0, 2.0, 2.5, 4.0, 10.0]) inds = digitize(x, bins) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data.toarray(), bins, right=False) np.testing.assert_equal(res.toarray(), expected) def testAverageExecution(self): data = arange(1, 5, chunk_size=1) t = average(data) res = self.executor.execute_tensor(t)[0] expected = np.average(np.arange(1, 5)) self.assertEqual(res, expected) t = average(arange(1, 11, chunk_size=2), weights=arange(10, 0, -1, chunk_size=2)) res = self.executor.execute_tensor(t)[0] expected = np.average(range(1, 11), weights=range(10, 0, -1)) self.assertEqual(res, expected) data = arange(6, chunk_size=2).reshape((3, 2)) t = average(data, axis=1, weights=tensor([1. / 4, 3. / 4], chunk_size=2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.average(np.arange(6).reshape(3, 2), axis=1, weights=(1. / 4, 3. / 4)) np.testing.assert_equal(res, expected) with self.assertRaises(TypeError): average(data, weights=tensor([1. / 4, 3. / 4], chunk_size=2)) def testCovExecution(self): data = np.array([[0, 2], [1, 1], [2, 0]]).T x = tensor(data, chunk_size=1) t = cov(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.cov(data) np.testing.assert_equal(res, expected) data_x = [-2.1, -1, 4.3] data_y = [3, 1.1, 0.12] x = tensor(data_x, chunk_size=1) y = tensor(data_y, chunk_size=1) X = stack((x, y), axis=0) t = cov(x, y) r = tall(t == cov(X)) self.assertTrue(self.executor.execute_tensor(r)[0]) def testCorrcoefExecution(self): data_x = [-2.1, -1, 4.3] data_y = [3, 1.1, 0.12] x = tensor(data_x, chunk_size=1) y = tensor(data_y, chunk_size=1) t = corrcoef(x, y) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.corrcoef(data_x, data_y) np.testing.assert_equal(res, expected) def testFlipExecution(self): a = arange(8, chunk_size=2).reshape((2, 2, 2)) t = flip(a, 0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flip(np.arange(8).reshape(2, 2, 2), 0) np.testing.assert_equal(res, expected) t = flip(a, 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flip(np.arange(8).reshape(2, 2, 2), 1) np.testing.assert_equal(res, expected) t = flipud(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flipud(np.arange(8).reshape(2, 2, 2)) np.testing.assert_equal(res, expected) t = fliplr(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.fliplr(np.arange(8).reshape(2, 2, 2)) np.testing.assert_equal(res, expected) def testRepeatExecution(self): a = repeat(3, 4) res = self.executor.execute_tensor(a)[0] expected = np.repeat(3, 4) np.testing.assert_equal(res, expected) x_data = np.random.randn(20, 30) x = tensor(x_data, chunk_size=(3, 4)) t = repeat(x, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, 2) np.testing.assert_equal(res, expected) t = repeat(x, 3, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, 3, axis=1) np.testing.assert_equal(res, expected) t = repeat(x, np.arange(20), axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, np.arange(20), axis=0) np.testing.assert_equal(res, expected) t = repeat(x, arange(20, chunk_size=5), axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, np.arange(20), axis=0) np.testing.assert_equal(res, expected) x_data = sps.random(20, 30, density=.1) x = tensor(x_data, chunk_size=(3, 4)) t = repeat(x, 2, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data.toarray(), 2, axis=1) np.testing.assert_equal(res.toarray(), expected) def testTileExecution(self): a_data = np.array([0, 1, 2]) a = tensor(a_data, chunk_size=2) t = tile(a, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, 2) np.testing.assert_equal(res, expected) t = tile(a, (2, 2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, (2, 2)) np.testing.assert_equal(res, expected) t = tile(a, (2, 1, 2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, (2, 1, 2)) np.testing.assert_equal(res, expected) b_data = np.array([[1, 2], [3, 4]]) b = tensor(b_data, chunk_size=1) t = tile(b, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(b_data, 2) np.testing.assert_equal(res, expected) t = tile(b, (2, 1)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(b_data, (2, 1)) np.testing.assert_equal(res, expected) c_data = np.array([1, 2, 3, 4]) c = tensor(c_data, chunk_size=3) t = tile(c, (4, 1)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(c_data, (4, 1)) np.testing.assert_equal(res, expected) def testIsInExecution(self): element = 2 * arange(4, chunk_size=1).reshape((2, 2)) test_elements = [1, 2, 4, 8] mask = isin(element, test_elements) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements) np.testing.assert_equal(res, expected) res = self.executor.execute_tensor(element[mask], concat=True)[0] expected = np.array([2, 4]) np.testing.assert_equal(res, expected) mask = isin(element, test_elements, invert=True) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements, invert=True) np.testing.assert_equal(res, expected) res = self.executor.execute_tensor(element[mask], concat=True)[0] expected = np.array([0, 6]) np.testing.assert_equal(res, expected) test_set = {1, 2, 4, 8} mask = isin(element, test_set) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_set) np.testing.assert_equal(res, expected) def testRavelExecution(self): arr = ones((10, 5), chunk_size=2) flat_arr = mt.ravel(arr) res = self.executor.execute_tensor(flat_arr, concat=True)[0] self.assertEqual(len(res), 50) np.testing.assert_equal(res, np.ones(50)) def testSearchsortedExecution(self): raw = np.sort(np.random.randint(100, size=(16, ))) # test different chunk_size, 3 will have combine, 6 will skip combine for chunk_size in (3, 6): arr = tensor(raw, chunk_size=chunk_size) # test scalar, with value in the middle t1 = searchsorted(arr, 20) res = self.executor.execute_tensor(t1, concat=True)[0] expected = np.searchsorted(raw, 20) np.testing.assert_array_equal(res, expected) # test scalar, with value larger than 100 t2 = searchsorted(arr, 200) res = self.executor.execute_tensor(t2, concat=True)[0] expected = np.searchsorted(raw, 200) np.testing.assert_array_equal(res, expected) # test scalar, side left, with value exact in the middle of the array t3 = searchsorted(arr, raw[10], side='left') res = self.executor.execute_tensor(t3, concat=True)[0] expected = np.searchsorted(raw, raw[10], side='left') np.testing.assert_array_equal(res, expected) # test scalar, side right, with value exact in the middle of the array t4 = searchsorted(arr, raw[10], side='right') res = self.executor.execute_tensor(t4, concat=True)[0] expected = np.searchsorted(raw, raw[10], side='right') np.testing.assert_array_equal(res, expected) # test scalar, side left, with value exact in the end of the array t5 = searchsorted(arr, raw[15], side='left') res = self.executor.execute_tensor(t5, concat=True)[0] expected = np.searchsorted(raw, raw[15], side='left') np.testing.assert_array_equal(res, expected) # test scalar, side right, with value exact in the end of the array t6 = searchsorted(arr, raw[15], side='right') res = self.executor.execute_tensor(t6, concat=True)[0] expected = np.searchsorted(raw, raw[15], side='right') np.testing.assert_array_equal(res, expected) # test scalar, side left, with value exact in the start of the array t7 = searchsorted(arr, raw[0], side='left') res = self.executor.execute_tensor(t7, concat=True)[0] expected = np.searchsorted(raw, raw[0], side='left') np.testing.assert_array_equal(res, expected) # test scalar, side right, with value exact in the start of the array t8 = searchsorted(arr, raw[0], side='right') res = self.executor.execute_tensor(t8, concat=True)[0] expected = np.searchsorted(raw, raw[0], side='right') np.testing.assert_array_equal(res, expected) raw2 = np.random.randint(100, size=(3, 4)) # test tensor, side left t9 = searchsorted(arr, tensor(raw2, chunk_size=2), side='left') res = self.executor.execute_tensor(t9, concat=True)[0] expected = np.searchsorted(raw, raw2, side='left') np.testing.assert_array_equal(res, expected) # test tensor, side right t10 = searchsorted(arr, tensor(raw2, chunk_size=2), side='right') res = self.executor.execute_tensor(t10, concat=True)[0] expected = np.searchsorted(raw, raw2, side='right') np.testing.assert_array_equal(res, expected) # test one chunk arr = tensor(raw, chunk_size=16) # test scalar, tensor to search has 1 chunk t11 = searchsorted(arr, 20) res = self.executor.execute_tensor(t11, concat=True)[0] expected = np.searchsorted(raw, 20) np.testing.assert_array_equal(res, expected) # test tensor with 1 chunk, tensor to search has 1 chunk t12 = searchsorted(arr, tensor(raw2, chunk_size=4)) res = self.executor.execute_tensor(t12, concat=True)[0] expected = np.searchsorted(raw, raw2) np.testing.assert_array_equal(res, expected) # test tensor with more than 1 chunk, tensor to search has 1 chunk t13 = searchsorted(arr, tensor(raw2, chunk_size=2)) res = self.executor.execute_tensor(t13, concat=True)[0] expected = np.searchsorted(raw, raw2) np.testing.assert_array_equal(res, expected) # test sorter raw3 = np.random.randint(100, size=(16, )) arr = tensor(raw3, chunk_size=3) order = np.argsort(raw3) order_arr = tensor(order, chunk_size=4) t14 = searchsorted(arr, 20, sorter=order_arr) res = self.executor.execute_tensor(t14, concat=True)[0] expected = np.searchsorted(raw3, 20, sorter=order) np.testing.assert_array_equal(res, expected) def testUniqueExecution(self): rs = np.random.RandomState(0) raw = rs.randint(10, size=(10, )) for chunk_size in (10, 3): x = tensor(raw, chunk_size=chunk_size) y = unique(x) res = self.executor.execute_tensor(y, concat=True)[0] expected = np.unique(raw) np.testing.assert_array_equal(res, expected) y, indices = unique(x, return_index=True) res = self.executor.execute_tensors([y, indices]) expected = np.unique(raw, return_index=True) self.assertEqual(len(res), 2) self.assertEqual(len(expected), 2) np.testing.assert_array_equal(res[0], expected[0]) np.testing.assert_array_equal(res[1], expected[1]) y, inverse = unique(x, return_inverse=True) res = self.executor.execute_tensors([y, inverse]) expected = np.unique(raw, return_inverse=True) self.assertEqual(len(res), 2) self.assertEqual(len(expected), 2) np.testing.assert_array_equal(res[0], expected[0]) np.testing.assert_array_equal(res[1], expected[1]) y, counts = unique(x, return_counts=True) res = self.executor.execute_tensors([y, counts]) expected = np.unique(raw, return_counts=True) self.assertEqual(len(res), 2) self.assertEqual(len(expected), 2) np.testing.assert_array_equal(res[0], expected[0]) np.testing.assert_array_equal(res[1], expected[1]) y, indices, inverse, counts = unique(x, return_index=True, return_inverse=True, return_counts=True) res = self.executor.execute_tensors([y, indices, inverse, counts]) expected = np.unique(raw, return_index=True, return_inverse=True, return_counts=True) self.assertEqual(len(res), 4) self.assertEqual(len(expected), 4) np.testing.assert_array_equal(res[0], expected[0]) np.testing.assert_array_equal(res[1], expected[1]) np.testing.assert_array_equal(res[2], expected[2]) np.testing.assert_array_equal(res[3], expected[3]) y, indices, counts = unique(x, return_index=True, return_counts=True) res = self.executor.execute_tensors([y, indices, counts]) expected = np.unique(raw, return_index=True, return_counts=True) self.assertEqual(len(res), 3) self.assertEqual(len(expected), 3) np.testing.assert_array_equal(res[0], expected[0]) np.testing.assert_array_equal(res[1], expected[1]) np.testing.assert_array_equal(res[2], expected[2]) raw2 = rs.randint(10, size=(4, 5, 6)) x2 = tensor(raw2, chunk_size=chunk_size) y2 = unique(x2) res = self.executor.execute_tensor(y2, concat=True)[0] expected = np.unique(raw2) np.testing.assert_array_equal(res, expected) y2 = unique(x2, axis=1) res = self.executor.execute_tensor(y2, concat=True)[0] expected = np.unique(raw2, axis=1) np.testing.assert_array_equal(res, expected) y2 = unique(x2, axis=2) res = self.executor.execute_tensor(y2, concat=True)[0] expected = np.unique(raw2, axis=2) np.testing.assert_array_equal(res, expected) raw = rs.randint(10, size=(10, 20)) raw[:, 0] = raw[:, 11] = rs.randint(10, size=(10, )) x = tensor(raw, chunk_size=2) y, ind, inv, counts = unique(x, aggregate_size=3, axis=1, return_index=True, return_inverse=True, return_counts=True) res_unique, res_ind, res_inv, res_counts = self.executor.execute_tensors( (y, ind, inv, counts)) exp_unique, exp_ind, exp_counts = np.unique(raw, axis=1, return_index=True, return_counts=True) raw_res_unique = res_unique res_unique_df = pd.DataFrame(res_unique) res_unique_ind = np.asarray( res_unique_df.sort_values(list(range(res_unique.shape[0])), axis=1).columns) res_unique = res_unique[:, res_unique_ind] res_ind = res_ind[res_unique_ind] res_counts = res_counts[res_unique_ind] np.testing.assert_array_equal(res_unique, exp_unique) np.testing.assert_array_equal(res_ind, exp_ind) np.testing.assert_array_equal(raw_res_unique[:, res_inv], raw) np.testing.assert_array_equal(res_counts, exp_counts) x = (mt.random.RandomState(0).rand(1000, chunk_size=20) > 0.5).astype( np.int32) y = unique(x) res = np.sort(self.executor.execute_tensor(y, concat=True)[0]) np.testing.assert_array_equal(res, np.array([0, 1])) @require_cupy def testToGPUExecution(self): raw = np.random.rand(10, 10) x = tensor(raw, chunk_size=3) gx = to_gpu(x) res = self.executor.execute_tensor(gx, concat=True)[0] np.testing.assert_array_equal(res.get(), raw) @require_cupy def testToCPUExecution(self): raw = np.random.rand(10, 10) x = tensor(raw, chunk_size=3, gpu=True) cx = to_cpu(x) res = self.executor.execute_tensor(cx, concat=True)[0] np.testing.assert_array_equal(res, raw) def testSortExecution(self): # only 1 chunk when axis = -1 raw = np.random.rand(100, 10) x = tensor(raw, chunk_size=10) sx = sort(x) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw)) # 1-d chunk raw = np.random.rand(100) x = tensor(raw, chunk_size=10) sx = sort(x) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw)) # structured dtype raw = np.empty(100, dtype=[('id', np.int32), ('size', np.int64)]) raw['id'] = np.random.randint(1000, size=100, dtype=np.int32) raw['size'] = np.random.randint(1000, size=100, dtype=np.int64) x = tensor(raw, chunk_size=10) sx = sort(x, order=['size', 'id']) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw, order=['size', 'id'])) # test flatten case raw = np.random.rand(10, 10) x = tensor(raw, chunk_size=5) sx = sort(x, axis=None) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw, axis=None)) # test multi-dimension raw = np.random.rand(10, 100) x = tensor(raw, chunk_size=(2, 10)) sx = sort(x, psrs_kinds=['quicksort'] * 3) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw)) raw = np.random.rand(10, 99) x = tensor(raw, chunk_size=(2, 10)) sx = sort(x) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw)) # test 3-d raw = np.random.rand(20, 25, 28) x = tensor(raw, chunk_size=(10, 5, 7)) sx = sort(x) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw)) sx = sort(x, axis=0) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw, axis=0)) sx = sort(x, axis=1) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw, axis=1)) # test multi-dimension with structured type raw = np.empty((10, 100), dtype=[('id', np.int32), ('size', np.int64)]) raw['id'] = np.random.randint(1000, size=(10, 100), dtype=np.int32) raw['size'] = np.random.randint(1000, size=(10, 100), dtype=np.int64) x = tensor(raw, chunk_size=(3, 10)) sx = sort(x) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw)) sx = sort(x, order=['size', 'id']) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw, order=['size', 'id'])) sx = sort(x, order=['size']) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw, order=['size'])) sx = sort(x, axis=0, order=['size', 'id']) res = self.executor.execute_tensor(sx, concat=True)[0] np.testing.assert_array_equal( res, np.sort(raw, axis=0, order=['size', 'id'])) raw = np.random.rand(10, 12) a = tensor(raw, chunk_size=(5, 4)) a.sort(axis=1) res = self.executor.execute_tensor(a, concat=True)[0] np.testing.assert_array_equal(res, np.sort(raw, axis=1)) a.sort(axis=0) res = self.executor.execute_tensor(a, concat=True)[0] np.testing.assert_array_equal(res, np.sort(np.sort(raw, axis=1), axis=0))
def setUp(self): self.executor = Executor('numpy') self.old_chunk = options.tensor.chunk_size options.tensor.chunk_size = 10
def testExecutorWithGeventProvider(self): executor = Executor(sync_provider_type=Executor.SyncProviderType.GEVENT) a = mt.ones((10, 10), chunk_size=2) res = executor.execute_tensor(a, concat=True)[0] np.testing.assert_array_equal(res, np.ones((10, 10)))
def setUp(self): self.executor = Executor('numpy') local_session = LocalSession() local_session._executor = self.executor self.session = Session() self.session._sess = local_session
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') self.old_chunk = options.tensor.chunk_size options.tensor.chunk_size = 10 def tearDown(self): options.tensor.chunk_size = self.old_chunk def testBoolIndexingExecution(self): raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=3) index = arr < .5 arr2 = arr[index] size_res = self.executor.execute_tensor(arr2, mock=True) res = self.executor.execute_tensor(arr2) self.assertEqual(sum(s[0] for s in size_res), arr.nbytes) np.testing.assert_array_equal(np.sort(np.concatenate(res)), np.sort(raw[raw < .5])) index2 = tensor(raw[:, :, 0, 0], chunk_size=3) < .5 arr3 = arr[index2] res = self.executor.execute_tensor(arr3) self.assertEqual(sum(it.size for it in res), raw[raw[:, :, 0, 0] < .5].size) def testFancyIndexingNumpyExecution(self): # test fancy index of type numpy ndarray raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=(2, 3, 2, 3)) index = [8, 10, 3, 1, 9, 10] arr2 = arr[index] res = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_array_equal(res[0], raw[index]) index = np.random.permutation(8) arr3 = arr[:2, ..., index] res = self.executor.execute_tensor(arr3, concat=True) np.testing.assert_array_equal(res[0], raw[:2, ..., index]) index = [1, 3, 9, 10] arr4 = arr[..., index, :5] res = self.executor.execute_tensor(arr4, concat=True) np.testing.assert_array_equal(res[0], raw[..., index, :5]) index1 = [8, 10, 3, 1, 9, 10] index2 = [1, 3, 9, 10, 2, 7] arr5 = arr[index1, :, index2] res = self.executor.execute_tensor(arr5, concat=True) np.testing.assert_array_equal(res[0], raw[index1, :, index2]) index1 = [1, 3, 5, 7, 9, 10] index2 = [1, 9, 9, 10, 2, 7] arr6 = arr[index1, :, index2] res = self.executor.execute_tensor(arr6, concat=True) np.testing.assert_array_equal(res[0], raw[index1, :, index2]) # fancy index is ordered, no concat required self.assertGreater(len(arr6.nsplits[0]), 1) index1 = [[8, 10, 3], [1, 9, 10]] index2 = [[1, 3, 9], [10, 2, 7]] arr7 = arr[index1, :, index2] res = self.executor.execute_tensor(arr7, concat=True) np.testing.assert_array_equal(res[0], raw[index1, :, index2]) index1 = [[1, 3], [3, 7], [7, 7]] index2 = [1, 9] arr8 = arr[0, index1, :, index2] res = self.executor.execute_tensor(arr8, concat=True) np.testing.assert_array_equal(res[0], raw[0, index1, :, index2]) def testFancyIndexingTensorExecution(self): # test fancy index of type tensor raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=(2, 3, 2, 3)) raw_index = [8, 10, 3, 1, 9, 10] index = tensor(raw_index, chunk_size=4) arr2 = arr[index] res = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_array_equal(res[0], raw[raw_index]) raw_index = np.random.permutation(8) index = tensor(raw_index, chunk_size=3) arr3 = arr[:2, ..., index] res = self.executor.execute_tensor(arr3, concat=True) np.testing.assert_array_equal(res[0], raw[:2, ..., raw_index]) raw_index = [1, 3, 9, 10] index = tensor(raw_index) arr4 = arr[..., index, :5] res = self.executor.execute_tensor(arr4, concat=True) np.testing.assert_array_equal(res[0], raw[..., raw_index, :5]) raw_index1 = [8, 10, 3, 1, 9, 10] raw_index2 = [1, 3, 9, 10, 2, 7] index1 = tensor(raw_index1, chunk_size=4) index2 = tensor(raw_index2, chunk_size=3) arr5 = arr[index1, :, index2] res = self.executor.execute_tensor(arr5, concat=True) np.testing.assert_array_equal(res[0], raw[raw_index1, :, raw_index2]) raw_index1 = [1, 3, 5, 7, 9, 10] raw_index2 = [1, 9, 9, 10, 2, 7] index1 = tensor(raw_index1, chunk_size=3) index2 = tensor(raw_index2, chunk_size=4) arr6 = arr[index1, :, index2] res = self.executor.execute_tensor(arr6, concat=True) np.testing.assert_array_equal(res[0], raw[raw_index1, :, raw_index2]) raw_index1 = [[8, 10, 3], [1, 9, 10]] raw_index2 = [[1, 3, 9], [10, 2, 7]] index1 = tensor(raw_index1) index2 = tensor(raw_index2, chunk_size=2) arr7 = arr[index1, :, index2] res = self.executor.execute_tensor(arr7, concat=True) np.testing.assert_array_equal(res[0], raw[raw_index1, :, raw_index2]) raw_index1 = [[1, 3], [3, 7], [7, 7]] raw_index2 = [1, 9] index1 = tensor(raw_index1, chunk_size=(2, 1)) index2 = tensor(raw_index2) arr8 = arr[0, index1, :, index2] res = self.executor.execute_tensor(arr8, concat=True) np.testing.assert_array_equal(res[0], raw[0, raw_index1, :, raw_index2]) raw_a = np.random.rand(30, 30) a = tensor(raw_a, chunk_size=(13, 17)) b = a.argmax(axis=0) c = a[b, arange(30)] res = self.executor.execute_tensor(c, concat=True) np.testing.assert_array_equal( res[0], raw_a[raw_a.argmax(axis=0), np.arange(30)]) def testSliceExecution(self): raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=3) arr2 = arr[2:9:2, 3:7, -1:-9:-2, 12:-11:-4] res = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_array_equal(res[0], raw[2:9:2, 3:7, -1:-9:-2, 12:-11:-4]) arr3 = arr[-4, 2:] res = self.executor.execute_tensor(arr3, concat=True) np.testing.assert_equal(res[0], raw[-4, 2:]) raw = sps.random(12, 14, density=.1) arr = tensor(raw, chunk_size=3) arr2 = arr[-1:-9:-2, 12:-11:-4] res = self.executor.execute_tensor(arr2, concat=True)[0] np.testing.assert_equal(res.toarray(), raw.toarray()[-1:-9:-2, 12:-11:-4]) def testMixedIndexingExecution(self): raw = np.random.random((11, 8, 12, 13)) arr = tensor(raw, chunk_size=3) raw_cond = raw[0, :, 0, 0] < .5 cond = tensor(raw[0, :, 0, 0], chunk_size=3) < .5 arr2 = arr[10::-2, cond, None, ..., :5] size_res = self.executor.execute_tensor(arr2, mock=True) res = self.executor.execute_tensor(arr2, concat=True) new_shape = list(arr2.shape) new_shape[1] = cond.shape[0] self.assertEqual(sum(s[0] for s in size_res), int(np.prod(new_shape) * arr2.dtype.itemsize)) np.testing.assert_array_equal(res[0], raw[10::-2, raw_cond, None, ..., :5]) b_raw = np.random.random(8) cond = tensor(b_raw, chunk_size=2) < .5 arr3 = arr[-2::-3, cond, ...] res = self.executor.execute_tensor(arr3, concat=True) np.testing.assert_array_equal(res[0], raw[-2::-3, b_raw < .5, ...]) def testSetItemExecution(self): raw = data = np.random.randint(0, 10, size=(11, 8, 12, 13)) arr = tensor(raw.copy(), chunk_size=3) raw = raw.copy() idx = slice(2, 9, 2), slice(3, 7), slice(-1, -9, -2), 2 arr[idx] = 20 res = self.executor.execute_tensor(arr, concat=True) raw[idx] = 20 np.testing.assert_array_equal(res[0], raw) raw = data shape = raw[idx].shape arr2 = tensor(raw.copy(), chunk_size=3) raw = raw.copy() replace = np.random.randint(10, 20, size=shape[:-1] + (1, )).astype('f4') arr2[idx] = tensor(replace, chunk_size=4) res = self.executor.execute_tensor(arr2, concat=True) raw[idx] = replace np.testing.assert_array_equal(res[0], raw) def testSetItemStructuredExecution(self): rec_type = np.dtype([('a', np.int32), ('b', np.double), ('c', np.dtype([('a', np.int16), ('b', np.int64)]))]) raw = np.zeros((4, 5), dtype=rec_type) arr = tensor(raw.copy(), chunk_size=3) arr[1:4, 1] = (3, 4., (5, 6)) arr[1:4, 2] = 8 arr[1:3] = np.arange(5) arr[2:4] = np.arange(10).reshape(2, 5) arr[0] = np.arange(5) raw[1:4, 1] = (3, 4., (5, 6)) raw[1:4, 2] = 8 raw[1:3] = np.arange(5) raw[2:4] = np.arange(10).reshape(2, 5) raw[0] = np.arange(5) res = self.executor.execute_tensor(arr, concat=True) self.assertEqual(arr.dtype, raw.dtype) self.assertEqual(arr.shape, raw.shape) np.testing.assert_array_equal(res[0], raw) def testTakeExecution(self): data = np.random.rand(10, 20, 30) t = tensor(data, chunk_size=10) a = t.take([4, 1, 2, 6, 200]) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.take(data, [4, 1, 2, 6, 200]) np.testing.assert_array_equal(res, expected) a = take(t, [5, 19, 2, 13], axis=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.take(data, [5, 19, 2, 13], axis=1) np.testing.assert_array_equal(res, expected) def testCompressExecution(self): data = np.array([[1, 2], [3, 4], [5, 6]]) a = tensor(data, chunk_size=1) t = compress([0, 1], a, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1], data, axis=0) np.testing.assert_array_equal(res, expected) t = compress([0, 1], a, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1], data, axis=1) np.testing.assert_array_equal(res, expected) t = a.compress([0, 1, 1]) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1, 1], data) np.testing.assert_array_equal(res, expected) t = compress([False, True, True], a, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([False, True, True], data, axis=0) np.testing.assert_array_equal(res, expected) t = compress([False, True], a, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([False, True], data, axis=1) np.testing.assert_array_equal(res, expected) with self.assertRaises(np.AxisError): compress([0, 1, 1], a, axis=1) def testExtractExecution(self): data = np.arange(12).reshape((3, 4)) a = tensor(data, chunk_size=2) condition = mod(a, 3) == 0 t = extract(condition, a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.extract(np.mod(data, 3) == 0, data) np.testing.assert_array_equal(res, expected) def testChooseExecution(self): options.tensor.chunk_size = 2 choices = [[0, 1, 2, 3], [10, 11, 12, 13], [20, 21, 22, 23], [30, 31, 32, 33]] a = choose([2, 3, 1, 0], choices) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.choose([2, 3, 1, 0], choices) np.testing.assert_array_equal(res, expected) a = choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1) expected = np.choose([2, 4, 1, 0], choices, mode='clip') res = self.executor.execute_tensor(a, concat=True)[0] np.testing.assert_array_equal(res, expected) a = choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) expected = np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) res = self.executor.execute_tensor(a, concat=True)[0] np.testing.assert_array_equal(res, expected) a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]] choices = [-10, 10] b = choose(a, choices) expected = np.choose(a, choices) res = self.executor.execute_tensor(b, concat=True)[0] np.testing.assert_array_equal(res, expected) a = np.array([0, 1]).reshape((2, 1, 1)) c1 = np.array([1, 2, 3]).reshape((1, 3, 1)) c2 = np.array([-1, -2, -3, -4, -5]).reshape((1, 1, 5)) b = choose(a, (c1, c2)) expected = np.choose(a, (c1, c2)) res = self.executor.execute_tensor(b, concat=True)[0] np.testing.assert_array_equal(res, expected) def testUnravelExecution(self): a = tensor([22, 41, 37], chunk_size=1) t = stack(unravel_index(a, (7, 6))) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.stack(np.unravel_index([22, 41, 37], (7, 6))) np.testing.assert_array_equal(res, expected) def testNonzeroExecution(self): data = np.array([[1, 0, 0], [0, 2, 0], [1, 1, 0]]) x = tensor(data, chunk_size=2) t = hstack(nonzero(x)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.hstack(np.nonzero(data)) np.testing.assert_array_equal(res, expected) def testFlatnonzeroExecution(self): x = arange(-2, 3, chunk_size=2) t = flatnonzero(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flatnonzero(np.arange(-2, 3)) np.testing.assert_equal(res, expected)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') self.old_chunk = options.tensor.chunk_size options.tensor.chunk_size = 10 def tearDown(self): options.tensor.chunk_size = self.old_chunk def testBoolIndexingExecution(self): raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=3) index = arr < .5 arr2 = arr[index] size_res = self.executor.execute_tensor(arr2, mock=True) res = self.executor.execute_tensor(arr2) self.assertEqual(sum(s[0] for s in size_res), arr.nbytes) self.assertTrue(np.array_equal(np.sort(np.concatenate(res)), np.sort(raw[raw < .5]))) index2 = tensor(raw[:, :, 0, 0], chunk_size=3) < .5 arr3 = arr[index2] res = self.executor.execute_tensor(arr3) self.assertEqual(sum(it.size for it in res), raw[raw[:, :, 0, 0] < .5].size) def testFancyIndexingExecution(self): raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=(2, 3, 2, 3)) index = [8, 10, 3, 1, 9, 10] arr2 = arr[index] res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw[index])) index = np.random.permutation(8) arr3 = arr[:2, ..., index] res = self.executor.execute_tensor(arr3, concat=True) self.assertTrue(np.array_equal(res[0], raw[:2, ..., index])) index = [1, 3, 9, 10] arr4 = arr[..., index, :5] res = self.executor.execute_tensor(arr4, concat=True) self.assertTrue(np.array_equal(res[0], raw[..., index, :5])) def testSliceExecution(self): raw = np.random.random((11, 8, 12, 14)) arr = tensor(raw, chunk_size=3) arr2 = arr[2:9:2, 3:7, -1:-9:-2, 12:-11:-4] res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw[2:9:2, 3:7, -1:-9:-2, 12:-11:-4])) raw = sps.random(12, 14, density=.1) arr = tensor(raw, chunk_size=3) arr2 = arr[-1:-9:-2, 12:-11:-4] res = self.executor.execute_tensor(arr2, concat=True)[0] np.testing.assert_equal(res.toarray(), raw.toarray()[-1:-9:-2, 12:-11:-4]) def testMixedIndexingExecution(self): raw = np.random.random((11, 8, 12, 13)) arr = tensor(raw, chunk_size=3) raw_cond = raw[0, :, 0, 0] < .5 cond = tensor(raw[0, :, 0, 0], chunk_size=3) < .5 arr2 = arr[10::-2, cond, None, ..., :5] size_res = self.executor.execute_tensor(arr2, mock=True) res = self.executor.execute_tensor(arr2, concat=True) new_shape = list(arr2.shape) new_shape[1] = cond.shape[0] self.assertEqual(sum(s[0] for s in size_res), int(np.prod(new_shape) * arr2.dtype.itemsize)) self.assertTrue(np.array_equal(res[0], raw[10::-2, raw_cond, None, ..., :5])) b_raw = np.random.random(8) cond = tensor(b_raw, chunk_size=2) < .5 arr3 = arr[-2::-3, cond, ...] res = self.executor.execute_tensor(arr3, concat=True) self.assertTrue(np.array_equal(res[0], raw[-2::-3, b_raw < .5, ...])) def testSetItemExecution(self): raw = data = np.random.randint(0, 10, size=(11, 8, 12, 13)) arr = tensor(raw.copy(), chunk_size=3) raw = raw.copy() idx = slice(2, 9, 2), slice(3, 7), slice(-1, -9, -2), 2 arr[idx] = 20 res = self.executor.execute_tensor(arr, concat=True) raw[idx] = 20 self.assertTrue(np.array_equal(res[0], raw)) raw = data shape = raw[idx].shape arr2 = tensor(raw.copy(), chunk_size=3) raw = raw.copy() replace = np.random.randint(10, 20, size=shape[:-1] + (1,)).astype('f4') arr2[idx] = tensor(replace, chunk_size=4) res = self.executor.execute_tensor(arr2, concat=True) raw[idx] = replace self.assertTrue(np.array_equal(res[0], raw)) def testTakeExecution(self): data = np.random.rand(10, 20, 30) t = tensor(data, chunk_size=10) a = t.take([4, 1, 2, 6, 200]) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.take(data, [4, 1, 2, 6, 200]) self.assertTrue(np.array_equal(res, expected)) a = take(t, [5, 19, 2, 13], axis=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.take(data, [5, 19, 2, 13], axis=1) self.assertTrue(np.array_equal(res, expected)) def testCompressExecution(self): data = np.array([[1, 2], [3, 4], [5, 6]]) a = tensor(data, chunk_size=1) t = compress([0, 1], a, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1], data, axis=0) self.assertTrue(np.array_equal(res, expected)) t = compress([0, 1], a, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1], data, axis=1) self.assertTrue(np.array_equal(res, expected)) t = a.compress([0, 1, 1]) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([0, 1, 1], data) self.assertTrue(np.array_equal(res, expected)) t = compress([False, True, True], a, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([False, True, True], data, axis=0) self.assertTrue(np.array_equal(res, expected)) t = compress([False, True], a, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.compress([False, True], data, axis=1) self.assertTrue(np.array_equal(res, expected)) with self.assertRaises(np.AxisError): compress([0, 1, 1], a, axis=1) def testExtractExecution(self): data = np.arange(12).reshape((3, 4)) a = tensor(data, chunk_size=2) condition = mod(a, 3) == 0 t = extract(condition, a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.extract(np.mod(data, 3) == 0, data) self.assertTrue(np.array_equal(res, expected)) def testChooseExecution(self): options.tensor.chunk_size = 2 choices = [[0, 1, 2, 3], [10, 11, 12, 13], [20, 21, 22, 23], [30, 31, 32, 33]] a = choose([2, 3, 1, 0], choices) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.choose([2, 3, 1, 0], choices) self.assertTrue(np.array_equal(res, expected)) a = choose([2, 4, 1, 0], choices, mode='clip') # 4 goes to 3 (4-1) expected = np.choose([2, 4, 1, 0], choices, mode='clip') res = self.executor.execute_tensor(a, concat=True)[0] self.assertTrue(np.array_equal(res, expected)) a = choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) expected = np.choose([2, 4, 1, 0], choices, mode='wrap') # 4 goes to (4 mod 4) res = self.executor.execute_tensor(a, concat=True)[0] self.assertTrue(np.array_equal(res, expected)) a = [[1, 0, 1], [0, 1, 0], [1, 0, 1]] choices = [-10, 10] b = choose(a, choices) expected = np.choose(a, choices) res = self.executor.execute_tensor(b, concat=True)[0] self.assertTrue(np.array_equal(res, expected)) a = np.array([0, 1]).reshape((2, 1, 1)) c1 = np.array([1, 2, 3]).reshape((1, 3, 1)) c2 = np.array([-1, -2, -3, -4, -5]).reshape((1, 1, 5)) b = choose(a, (c1, c2)) expected = np.choose(a, (c1, c2)) res = self.executor.execute_tensor(b, concat=True)[0] self.assertTrue(np.array_equal(res, expected)) def testUnravelExecution(self): a = tensor([22, 41, 37], chunk_size=1) t = stack(unravel_index(a, (7, 6))) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.stack(np.unravel_index([22, 41, 37], (7, 6))) self.assertTrue(np.array_equal(res, expected)) def testNonzeroExecution(self): data = np.array([[1, 0, 0], [0, 2, 0], [1, 1, 0]]) x = tensor(data, chunk_size=2) t = hstack(nonzero(x)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.hstack(np.nonzero(data)) self.assertTrue(np.array_equal(res, expected)) def testFlatnonzeroExecution(self): x = arange(-2, 3, chunk_size=2) t = flatnonzero(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flatnonzero(np.arange(-2, 3)) np.testing.assert_equal(res, expected)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testRandExecution(self): arr = tensor.random.rand(10, 20, chunk_size=3, dtype='f4') res = self.executor.execute_tensor(arr, concat=True)[0] self.assertEqual(res.shape, (10, 20)) self.assertTrue(np.all(res < 1)) self.assertTrue(np.all(res > 0)) self.assertEqual(res.dtype, np.float32) def testRandnExecution(self): arr = tensor.random.randn(10, 20, chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.randn(10, 20, chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).randn(5, 5))) def testRandintExecution(self): size_executor = Executor( sync_provider_type=Executor.SyncProviderType.MOCK) arr = tensor.random.randint(0, 2, size=(10, 30), chunk_size=3) size_res = size_executor.execute_tensor(arr, mock=True) self.assertEqual(arr.nbytes, sum(tp[0] for tp in size_res)) res = self.executor.execute_tensor(arr, concat=True)[0] self.assertEqual(res.shape, (10, 30)) self.assertTrue(np.all(res >= 0)) self.assertTrue(np.all(res < 2)) def testRandomIntegersExecution(self): arr = tensor.random.random_integers(0, 10, size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.random_integers(0, 10, size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): np.testing.assert_equal( res, np.random.RandomState(0).random_integers(0, 10, size=(5, 5))) def testRandomSampleExecution(self): arr = tensor.random.random_sample(size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.random_sample(size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).random_sample(size=(5, 5)))) def testRandomExecution(self): arr = tensor.random.random(size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.random(size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).random_sample(size=(5, 5)))) def testRandfExecution(self): arr = tensor.random.ranf(size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.ranf(size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).random_sample(size=(5, 5)))) def testSampleExecution(self): arr = tensor.random.sample(size=(10, 20), chunk_size=3) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (10, 20)) arr = tensor.random.sample(size=(10, 20), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).random_sample(size=(5, 5)))) def testChoiceExecution(self): arr = tensor.random.choice(5, size=3, chunk_size=1) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (3, )) arr = tensor.random.choice(5, size=(15, ), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).choice(5, size=(5, )))) arr = tensor.random.choice([1, 4, 9], size=3, chunk_size=1) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (3, )) arr = tensor.random.choice([1, 4, 9], size=(15, ), chunk_size=5).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).choice([1, 4, 9], size=(5, )))) with self.assertRaises(ValueError): tensor.random.choice([1, 3, 4], size=5, replace=False, chunk_size=2) arr = tensor.random.choice([1, 4, 9], size=3, replace=False, chunk_size=1) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (3, )) arr = tensor.random.choice([1, 4, 9], size=(3, ), replace=False, chunk_size=1).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).choice([1, 4, 9], size=(1, ), replace=False))) arr = tensor.random.choice([1, 4, 9], size=3, p=[.2, .5, .3], chunk_size=1) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (3, )) arr = tensor.random.choice([1, 4, 9], size=(15, ), chunk_size=5, p=[.2, .5, .3]).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).choice([1, 4, 9], size=(5, ), p=[.2, .5, .3]))) def testSparseRandintExecution(self): size_executor = Executor( sync_provider_type=Executor.SyncProviderType.MOCK) arr = tensor.random.randint(1, 2, size=(30, 50), density=.1, chunk_size=10, dtype='f4') size_res = size_executor.execute_tensor(arr, mock=True) self.assertAlmostEqual(arr.nbytes * 0.1, sum(tp[0] for tp in size_res)) res = self.executor.execute_tensor(arr, concat=True)[0] self.assertTrue(issparse(res)) self.assertEqual(res.shape, (30, 50)) self.assertTrue(np.all(res.data >= 1)) self.assertTrue(np.all(res.data < 2)) self.assertAlmostEqual((res >= 1).toarray().sum(), 30 * 50 * .1, delta=20) def testBetaExecute(self): arr = tensor.random.beta(1, 2, chunk_size=2).tiles() arr.chunks[0].op._seed = 0 self.assertEqual( self.executor.execute_tensor(arr)[0], np.random.RandomState(0).beta(1, 2)) arr = tensor.random.beta([1, 2], [3, 4], chunk_size=2).tiles() arr.chunks[0].op._seed = 0 self.assertTrue( np.array_equal( self.executor.execute_tensor(arr)[0], np.random.RandomState(0).beta([1, 2], [3, 4]))) arr = tensor.random.beta([[2, 3]], from_ndarray([[4, 6], [5, 2]], chunk_size=2), chunk_size=1, size=(3, 2, 2)).tiles() for c in arr.chunks: c.op._seed = 0 res = self.executor.execute_tensor(arr, concat=True)[0] self.assertEqual(res[0, 0, 0], np.random.RandomState(0).beta(2, 4)) self.assertEqual(res[0, 0, 1], np.random.RandomState(0).beta(3, 6)) self.assertEqual(res[0, 1, 0], np.random.RandomState(0).beta(2, 5)) self.assertEqual(res[0, 1, 1], np.random.RandomState(0).beta(3, 2)) arr = tensor.random.RandomState(0).beta([[3, 4]], [[1], [2]], chunk_size=1) tensor.random.seed(0) arr2 = tensor.random.beta([[3, 4]], [[1], [2]], chunk_size=1) self.assertTrue( np.array_equal( self.executor.execute_tensor(arr, concat=True)[0], self.executor.execute_tensor(arr2, concat=True)[0])) def testBinomialExecute(self): arr = tensor.random.binomial(10, .5, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.binomial(10, .5, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).binomial(10, .5, 10))) def testChisquareExecute(self): arr = tensor.random.chisquare(2, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.chisquare(2, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).chisquare(2, 10))) def testDirichletExecute(self): arr = tensor.random.dirichlet((10, 5, 3), 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 3)) arr = tensor.random.dirichlet((10, 5, 3), 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).dirichlet((10, 5, 3), 10))) def testExponentialExecute(self): arr = tensor.random.exponential(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.exponential(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).exponential(1.0, 10))) def testFExecute(self): arr = tensor.random.f(1.0, 2.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.f(1.0, 2.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).f(1.0, 2.0, 10))) def testGammaExecute(self): arr = tensor.random.gamma(1.0, 2.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.gamma(1.0, 2.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).gamma(1.0, 2.0, 10))) def testGeometricExecution(self): arr = tensor.random.geometric(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.geometric(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).geometric(1.0, 10))) def testGumbelExecution(self): arr = tensor.random.gumbel(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.gumbel(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).gumbel(.5, 1.0, 10))) def testHypergeometricExecution(self): arr = tensor.random.hypergeometric(10, 20, 15, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.hypergeometric(10, 20, 15, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).hypergeometric(10, 20, 15, 10))) def testLaplaceExecution(self): arr = tensor.random.laplace(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.laplace(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).laplace(.5, 1.0, 10))) def testLogisticExecution(self): arr = tensor.random.logistic(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.logistic(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): np.testing.assert_equal( res, np.random.RandomState(0).logistic(.5, 1.0, 10)) def testLognormalExecution(self): arr = tensor.random.lognormal(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.lognormal(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).lognormal(.5, 1.0, 10))) def testLogseriesExecution(self): arr = tensor.random.logseries(.5, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.logseries(.5, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).logseries(.5, 10))) def testMultinomialExecution(self): arr = tensor.random.multinomial(10, [.2, .5, .3], 100, chunk_size=10) self.assertEqual(arr.shape, (100, 3)) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 3)) arr = tensor.random.multinomial(10, [.2, .5, .3], 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).multinomial(10, [.2, .5, .3], 10))) def testMultivariateNormalExecution(self): arr = tensor.random.multivariate_normal([1, 2], [[1, 0], [0, 1]], 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, 2)) arr = tensor.random.multivariate_normal([1, 2], [[1, 0], [0, 1]], 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).multivariate_normal( [1, 2], [[1, 0], [0, 1]], 10))) def testNegativeBinomialExecution(self): arr = tensor.random.negative_binomial(5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.negative_binomial(5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).negative_binomial(5, 1.0, 10))) def testNoncentralChisquareExecution(self): arr = tensor.random.noncentral_chisquare(.5, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.noncentral_chisquare(.5, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).noncentral_chisquare(.5, 1.0, 10))) def testNoncentralFExecution(self): arr = tensor.random.noncentral_f(1.5, 1.0, 1.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.noncentral_f(1.5, 1.0, 1.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).noncentral_f(1.5, 1.0, 1.1, 10))) def testNormalExecute(self): arr = tensor.random.normal(10, 1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.normal(10, 1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).normal(10, 1.0, 10))) def testParetoExecute(self): arr = tensor.random.pareto(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.pareto(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).pareto(1.0, 10))) def testPoissonExecute(self): arr = tensor.random.poisson(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.poisson(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).poisson(1.0, 10))) def testPowerExecute(self): arr = tensor.random.power(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.power(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).power(1.0, 10))) def testRayleighExecute(self): arr = tensor.random.rayleigh(1.0, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.rayleigh(1.0, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).rayleigh(1.0, 10))) def testStandardCauchyExecute(self): arr = tensor.random.standard_cauchy(100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_cauchy(100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).standard_cauchy(10))) def testStandardExponentialExecute(self): arr = tensor.random.standard_exponential(100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_exponential(100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).standard_exponential(10))) def testStandardGammaExecute(self): arr = tensor.random.standard_gamma(.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_gamma(.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).standard_gamma(.1, 10))) def testStandardNormalExecute(self): arr = tensor.random.standard_normal(100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_normal(100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).standard_normal(10))) def testStandardTExecute(self): arr = tensor.random.standard_t(.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.standard_t(.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).standard_t(.1, 10))) def testTriangularExecute(self): arr = tensor.random.triangular(.1, .2, .3, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.triangular(.1, .2, .3, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal( res, np.random.RandomState(0).triangular(.1, .2, .3, 10))) def testUniformExecute(self): arr = tensor.random.uniform(.1, .2, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.uniform(.1, .2, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).uniform(.1, .2, 10))) def testVonmisesExecute(self): arr = tensor.random.vonmises(.1, .2, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.vonmises(.1, .2, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).vonmises(.1, .2, 10))) def testWaldExecute(self): arr = tensor.random.wald(.1, .2, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.wald(.1, .2, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).wald(.1, .2, 10))) def testWeibullExecute(self): arr = tensor.random.weibull(.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.weibull(.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).weibull(.1, 10))) def testZipfExecute(self): arr = tensor.random.zipf(1.1, 100, chunk_size=10) self.assertEqual( self.executor.execute_tensor(arr, concat=True)[0].shape, (100, )) arr = tensor.random.zipf(1.1, 100, chunk_size=10).tiles() for chunk in arr.chunks: chunk.op._seed = 0 for res in self.executor.execute_tensor(arr): self.assertTrue( np.array_equal(res, np.random.RandomState(0).zipf(1.1, 10)))
class Test(TestBase): def setUp(self): super(Test, self).setUp() self.executor = Executor() def testMerge(self): df1 = pd.DataFrame(np.arange(20).reshape((4, 5)) + 1, columns=['a', 'b', 'c', 'd', 'e']) df2 = pd.DataFrame(np.arange(20).reshape((5, 4)) + 1, columns=['a', 'b', 'x', 'y']) mdf1 = from_pandas(df1, chunk_size=2) mdf2 = from_pandas(df2, chunk_size=2) # Note [Index of Merge] # # When `left_index` and `right_index` of `merge` is both false, pandas will generate an RangeIndex to # the final result dataframe. # # We chunked the `left` and `right` dataframe, thus every result chunk will have its own RangeIndex. # When they are contenated we don't generate a new RangeIndex for the result, thus we cannot obtain the # same index value with pandas. But we guarantee that the content of dataframe is correct. # merge on index expected0 = df1.merge(df2) jdf0 = mdf1.merge(mdf2) result0 = self.executor.execute_dataframe(jdf0, concat=True)[0] pd.testing.assert_frame_equal(sort_dataframe_inplace(expected0, 0), sort_dataframe_inplace(result0, 0)) # merge on left index and `right_on` expected1 = df1.merge(df2, how='left', right_on='x', left_index=True) jdf1 = mdf1.merge(mdf2, how='left', right_on='x', left_index=True) result1 = self.executor.execute_dataframe(jdf1, concat=True)[0] expected1.set_index('a_x', inplace=True) result1.set_index('a_x', inplace=True) pd.testing.assert_frame_equal(sort_dataframe_inplace(expected1, 0), sort_dataframe_inplace(result1, 0)) # merge on `left_on` and right index expected2 = df1.merge(df2, how='right', left_on='a', right_index=True) jdf2 = mdf1.merge(mdf2, how='right', left_on='a', right_index=True) result2 = self.executor.execute_dataframe(jdf2, concat=True)[0] expected2.set_index('a', inplace=True) result2.set_index('a', inplace=True) pd.testing.assert_frame_equal(sort_dataframe_inplace(expected2, 0), sort_dataframe_inplace(result2, 0)) # merge on `left_on` and `right_on` expected3 = df1.merge(df2, how='left', left_on='a', right_on='x') jdf3 = mdf1.merge(mdf2, how='left', left_on='a', right_on='x') result3 = self.executor.execute_dataframe(jdf3, concat=True)[0] expected3.set_index('a_x', inplace=True) result3.set_index('a_x', inplace=True) pd.testing.assert_frame_equal(sort_dataframe_inplace(expected3, 0), sort_dataframe_inplace(result3, 0)) # merge on `on` expected4 = df1.merge(df2, how='right', on='a') jdf4 = mdf1.merge(mdf2, how='right', on='a') result4 = self.executor.execute_dataframe(jdf4, concat=True)[0] expected4.set_index('a', inplace=True) result4.set_index('a', inplace=True) pd.testing.assert_frame_equal(sort_dataframe_inplace(expected4, 0), sort_dataframe_inplace(result4, 0)) # merge on multiple columns expected5 = df1.merge(df2, how='inner', on=['a', 'b']) jdf5 = mdf1.merge(mdf2, how='inner', on=['a', 'b']) result5 = self.executor.execute_dataframe(jdf5, concat=True)[0] pd.testing.assert_frame_equal(sort_dataframe_inplace(expected5, 0), sort_dataframe_inplace(result5, 0)) def testJoin(self): df1 = pd.DataFrame([[1, 3, 3], [4, 2, 6], [7, 8, 9]], index=['a1', 'a2', 'a3']) df2 = pd.DataFrame([[1, 2, 3], [1, 5, 6], [7, 8, 9]], index=['a1', 'b2', 'b3']) + 1 df2 = pd.concat([df2, df2 + 1]) mdf1 = from_pandas(df1, chunk_size=2) mdf2 = from_pandas(df2, chunk_size=2) # default `how` expected0 = df1.join(df2, lsuffix='l_', rsuffix='r_') jdf0 = mdf1.join(mdf2, lsuffix='l_', rsuffix='r_') result0 = self.executor.execute_dataframe(jdf0, concat=True)[0] pd.testing.assert_frame_equal(expected0.sort_index(), result0.sort_index()) # how = 'left' expected1 = df1.join(df2, how='left', lsuffix='l_', rsuffix='r_') jdf1 = mdf1.join(mdf2, how='left', lsuffix='l_', rsuffix='r_') result1 = self.executor.execute_dataframe(jdf1, concat=True)[0] pd.testing.assert_frame_equal(expected1.sort_index(), result1.sort_index()) # how = 'right' expected2 = df1.join(df2, how='right', lsuffix='l_', rsuffix='r_') jdf2 = mdf1.join(mdf2, how='right', lsuffix='l_', rsuffix='r_') result2 = self.executor.execute_dataframe(jdf2, concat=True)[0] pd.testing.assert_frame_equal(expected2.sort_index(), result2.sort_index()) # how = 'inner' expected3 = df1.join(df2, how='inner', lsuffix='l_', rsuffix='r_') jdf3 = mdf1.join(mdf2, how='inner', lsuffix='l_', rsuffix='r_') result3 = self.executor.execute_dataframe(jdf3, concat=True)[0] pd.testing.assert_frame_equal(expected3.sort_index(), result3.sort_index()) # how = 'outer' expected4 = df1.join(df2, how='outer', lsuffix='l_', rsuffix='r_') jdf4 = mdf1.join(mdf2, how='outer', lsuffix='l_', rsuffix='r_') result4 = self.executor.execute_dataframe(jdf4, concat=True)[0] pd.testing.assert_frame_equal(expected4.sort_index(), result4.sort_index()) def testJoinOn(self): df1 = pd.DataFrame([[1, 3, 3], [4, 2, 6], [7, 8, 9]], columns=['a1', 'a2', 'a3']) df2 = pd.DataFrame([[1, 2, 3], [1, 5, 6], [7, 8, 9]], columns=['a1', 'b2', 'b3']) + 1 df2 = pd.concat([df2, df2 + 1]) mdf1 = from_pandas(df1, chunk_size=2) mdf2 = from_pandas(df2, chunk_size=2) expected0 = df1.join(df2, on=None, lsuffix='_l', rsuffix='_r') jdf0 = mdf1.join(mdf2, on=None, lsuffix='_l', rsuffix='_r') result0 = self.executor.execute_dataframe(jdf0, concat=True)[0] pd.testing.assert_frame_equal(sort_dataframe_inplace(expected0, 0), sort_dataframe_inplace(result0, 0)) expected1 = df1.join(df2, how='left', on='a1', lsuffix='_l', rsuffix='_r') jdf1 = mdf1.join(mdf2, how='left', on='a1', lsuffix='_l', rsuffix='_r') result1 = self.executor.execute_dataframe(jdf1, concat=True)[0] # Note [Columns of Left Join] # # I believe we have no chance to obtain the entirely same result with pandas here: # # Look at the following example: # # >>> df1 # a1 a2 a3 # 0 1 3 3 # >>> df2 # a1 b2 b3 # 1 2 6 7 # >>> df3 # a1 b2 b3 # 1 2 6 7 # 1 2 6 7 # # >>> df1.merge(df2, how='left', left_on='a1', left_index=False, right_index=True) # a1_x a2 a3 a1_y b2 b3 # 0 1 3 3 2 6 7 # >>> df1.merge(df3, how='left', left_on='a1', left_index=False, right_index=True) # a1 a1_x a2 a3 a1_y b2 b3 # 0 1 1 3 3 2 6 7 # 0 1 1 3 3 2 6 7 # # Note that the result of `df1.merge(df3)` has an extra column `a` compared to `df1.merge(df2)`. # The value of column `a` is the same of `a1_x`, just because `1` occurs twice in index of `df3`. # I haven't invistagated why pandas has such behaviour... # # We cannot yeild the same result with pandas, because, the `df3` is chunked, then some of the # result chunk has 6 columns, others may have 7 columns, when concatenated into one DataFrame # some cells of column `a` will have value `NaN`, which is different from the result of pandas. # # But we can guarantee that other effective columns have absolutely same value with pandas. columns_to_compare = jdf1.columns_value.to_pandas() pd.testing.assert_frame_equal( sort_dataframe_inplace(expected1[columns_to_compare], 0, 1), sort_dataframe_inplace(result1[columns_to_compare], 0, 1)) # Note [Index of Join on EmptyDataFrame] # # It is tricky that it is non-trivial to get the same `index` result with pandas. # # Look at the following example: # # >>> df1 # a1 a2 a3 # 1 4 2 6 # >>> df2 # a1 b2 b3 # 1 2 6 7 # 2 8 9 10 # >>> df3 # Empty DataFrame # Columns: [a1, a2, a3] # Index: [] # >>> df1.join(df2, how='right', on='a2', lsuffix='_l', rsuffix='_r') # a1_l a2 a3 a1_r b2 b3 # 1.0 4.0 2 6.0 8 9 10 # NaN NaN 1 NaN 2 6 7 # >>> df3.join(df2, how='right', on='a2', lsuffix='_l', rsuffix='_r') # a1_l a2 a3 a1_r b2 b3 # 1 NaN 1 NaN 2 6 7 # 2 NaN 2 NaN 8 9 10 # # When the `left` dataframe is not empty, the mismatched rows in `right` will have index value `NaN`, # and the matched rows have index value from `right`. When the `left` dataframe is empty, the mismatched # rows have index value from `right`. # # Since we chunked the `left` dataframe, it is uneasy to obtain the same index value with pandas in the # final result dataframe, but we guaranteed that the dataframe content is correctly. expected2 = df1.join(df2, how='right', on='a2', lsuffix='_l', rsuffix='_r') jdf2 = mdf1.join(mdf2, how='right', on='a2', lsuffix='_l', rsuffix='_r') result2 = self.executor.execute_dataframe(jdf2, concat=True)[0] expected2.set_index('a2', inplace=True) result2.set_index('a2', inplace=True) pd.testing.assert_frame_equal(sort_dataframe_inplace(expected2, 0), sort_dataframe_inplace(result2, 0)) expected3 = df1.join(df2, how='inner', on='a2', lsuffix='_l', rsuffix='_r') jdf3 = mdf1.join(mdf2, how='inner', on='a2', lsuffix='_l', rsuffix='_r') result3 = self.executor.execute_dataframe(jdf3, concat=True)[0] pd.testing.assert_frame_equal(sort_dataframe_inplace(expected3, 0), sort_dataframe_inplace(result3, 0)) expected4 = df1.join(df2, how='outer', on='a2', lsuffix='_l', rsuffix='_r') jdf4 = mdf1.join(mdf2, how='outer', on='a2', lsuffix='_l', rsuffix='_r') result4 = self.executor.execute_dataframe(jdf4, concat=True)[0] expected4.set_index('a2', inplace=True) result4.set_index('a2', inplace=True) pd.testing.assert_frame_equal(sort_dataframe_inplace(expected4, 0), sort_dataframe_inplace(result4, 0))
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testSumProdExecution(self): arr = ones((10, 8), chunk_size=3) self.assertEqual([80], self.executor.execute_tensor(arr.sum())) self.assertEqual( (10, ) * 8, tuple(np.concatenate(self.executor.execute_tensor( arr.sum(axis=0))))) arr = ones((3, 3), chunk_size=2) self.assertEqual([512], self.executor.execute_tensor((arr * 2).prod())) self.assertEqual((8, ) * 3, tuple( np.concatenate( self.executor.execute_tensor( (arr * 2).prod(axis=0))))) raw = sps.random(10, 20, density=.1) arr = tensor(raw, chunk_size=3) res = self.executor.execute_tensor(arr.sum())[0] self.assertAlmostEqual(res, raw.sum()) # test order raw = np.asfortranarray(np.random.rand(10, 20, 30)) arr = tensor(raw, chunk_size=13) arr2 = arr.sum(axis=-1) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = raw.sum(axis=-1) np.testing.assert_allclose(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) def testMaxMinExecution(self): raw = np.random.randint(10000, size=(10, 10, 10)) arr = tensor(raw, chunk_size=3) self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max())) self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min())) np.testing.assert_array_equal( raw.max(axis=0), self.executor.execute_tensor(arr.max(axis=0), concat=True)[0]) np.testing.assert_array_equal( raw.min(axis=0), self.executor.execute_tensor(arr.min(axis=0), concat=True)[0]) np.testing.assert_array_equal( raw.max(axis=(1, 2)), self.executor.execute_tensor(arr.max(axis=(1, 2)), concat=True)[0]) np.testing.assert_array_equal( raw.min(axis=(1, 2)), self.executor.execute_tensor(arr.min(axis=(1, 2)), concat=True)[0]) raw = sps.random(10, 10, density=.5) arr = tensor(raw, chunk_size=3) self.assertEqual([raw.max()], self.executor.execute_tensor(arr.max())) self.assertEqual([raw.min()], self.executor.execute_tensor(arr.min())) def testAllAnyExecution(self): raw1 = np.zeros((10, 15)) raw2 = np.ones((10, 15)) raw3 = np.array([[True, False, True, False], [True, True, True, True], [False, False, False, False], [False, True, False, True]]) arr1 = tensor(raw1, chunk_size=3) arr2 = tensor(raw2, chunk_size=3) arr3 = tensor(raw3, chunk_size=4) self.assertFalse(self.executor.execute_tensor(arr1.all())[0]) self.assertTrue(self.executor.execute_tensor(arr2.all())[0]) self.assertFalse(self.executor.execute_tensor(arr1.any())[0]) self.assertTrue(self.executor.execute_tensor(arr1.any())) np.testing.assert_array_equal( raw3.all(axis=1), self.executor.execute_tensor(arr3.all(axis=1))[0]) np.testing.assert_array_equal( raw3.any(axis=0), self.executor.execute_tensor(arr3.any(axis=0))[0]) raw = sps.random(10, 10, density=.5) > .5 arr = tensor(raw, chunk_size=3) self.assertEqual(raw.A.all(), self.executor.execute_tensor(arr.all())[0]) self.assertEqual(raw.A.any(), self.executor.execute_tensor(arr.any())[0]) def testMeanExecution(self): raw1 = np.random.random((20, 25)) raw2 = np.random.randint(10, size=(20, 25)) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.mean()) expected1 = raw1.mean() self.assertTrue(np.allclose(res1[0], expected1)) res2 = self.executor.execute_tensor(arr1.mean(axis=0)) expected2 = raw1.mean(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr1.mean(axis=1, keepdims=True)) expected3 = raw1.mean(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor(arr2.mean()) expected1 = raw2.mean() self.assertEqual(res1[0], expected1) res2 = self.executor.execute_tensor(arr2.mean(axis=0)) expected2 = raw2.mean(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr2.mean(axis=1, keepdims=True)) expected3 = raw2.mean(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) raw1 = sps.random(20, 25, density=.1) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.mean()) expected1 = raw1.mean() self.assertTrue(np.allclose(res1[0], expected1)) arr2 = tensor(raw1, chunk_size=30) res1 = self.executor.execute_tensor(arr2.mean()) expected1 = raw1.mean() self.assertTrue(np.allclose(res1[0], expected1)) arr = mean(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 1) def testVarExecution(self): raw1 = np.random.random((20, 25)) raw2 = np.random.randint(10, size=(20, 25)) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.var()) expected1 = raw1.var() self.assertTrue(np.allclose(res1[0], expected1)) res2 = self.executor.execute_tensor(arr1.var(axis=0)) expected2 = raw1.var(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr1.var(axis=1, keepdims=True)) expected3 = raw1.var(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor(arr2.var()) expected1 = raw2.var() self.assertAlmostEqual(res1[0], expected1) res2 = self.executor.execute_tensor(arr2.var(axis=0)) expected2 = raw2.var(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr2.var(axis=1, keepdims=True)) expected3 = raw2.var(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) res4 = self.executor.execute_tensor(arr2.var(ddof=1)) expected4 = raw2.var(ddof=1) self.assertAlmostEqual(res4[0], expected4) raw1 = sps.random(20, 25, density=.1) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.var()) expected1 = raw1.toarray().var() self.assertTrue(np.allclose(res1[0], expected1)) arr2 = tensor(raw1, chunk_size=30) res1 = self.executor.execute_tensor(arr2.var()) expected1 = raw1.toarray().var() self.assertTrue(np.allclose(res1[0], expected1)) arr = var(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 0) def testStdExecution(self): raw1 = np.random.random((20, 25)) raw2 = np.random.randint(10, size=(20, 25)) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.std()) expected1 = raw1.std() self.assertTrue(np.allclose(res1[0], expected1)) res2 = self.executor.execute_tensor(arr1.std(axis=0)) expected2 = raw1.std(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr1.std(axis=1, keepdims=True)) expected3 = raw1.std(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor(arr2.std()) expected1 = raw2.std() self.assertAlmostEqual(res1[0], expected1) res2 = self.executor.execute_tensor(arr2.std(axis=0)) expected2 = raw2.std(axis=0) self.assertTrue(np.allclose(np.concatenate(res2), expected2)) res3 = self.executor.execute_tensor(arr2.std(axis=1, keepdims=True)) expected3 = raw2.std(axis=1, keepdims=True) self.assertTrue(np.allclose(np.concatenate(res3), expected3)) res4 = self.executor.execute_tensor(arr2.std(ddof=1)) expected4 = raw2.std(ddof=1) self.assertAlmostEqual(res4[0], expected4) raw1 = sps.random(20, 25, density=.1) arr1 = tensor(raw1, chunk_size=3) res1 = self.executor.execute_tensor(arr1.std()) expected1 = raw1.toarray().std() self.assertTrue(np.allclose(res1[0], expected1)) arr2 = tensor(raw1, chunk_size=30) res1 = self.executor.execute_tensor(arr2.std()) expected1 = raw1.toarray().std() self.assertTrue(np.allclose(res1[0], expected1)) arr = std(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 0) def testArgReduction(self): raw = np.random.random((20, 20, 20)) arr = tensor(raw, chunk_size=3) self.assertEqual(raw.argmax(), self.executor.execute_tensor(arr.argmax())[0]) self.assertEqual(raw.argmin(), self.executor.execute_tensor(arr.argmin())[0]) np.testing.assert_array_equal( raw.argmax(axis=0), self.executor.execute_tensor(arr.argmax(axis=0), concat=True)[0]) np.testing.assert_array_equal( raw.argmin(axis=0), self.executor.execute_tensor(arr.argmin(axis=0), concat=True)[0]) raw_format = sps.random(20, 20, density=.1, format='lil') random_min = np.random.randint(0, 200) random_max = np.random.randint(200, 400) raw_format[np.unravel_index(random_min, raw_format.shape)] = -1 raw_format[np.unravel_index(random_max, raw_format.shape)] = 2 raw = raw_format.tocoo() arr = tensor(raw, chunk_size=3) self.assertEqual(raw.argmax(), self.executor.execute_tensor(arr.argmax())[0]) self.assertEqual(raw.argmin(), self.executor.execute_tensor(arr.argmin())[0]) # test order raw = np.asfortranarray(np.random.rand(10, 20, 30)) arr = tensor(raw, chunk_size=13) arr2 = arr.argmax(axis=-1) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = raw.argmax(axis=-1) np.testing.assert_allclose(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) def testNanReduction(self): raw = np.random.choice(a=[0, 1, np.nan], size=(10, 10), p=[0.3, 0.4, 0.3]) arr = tensor(raw, chunk_size=3) self.assertEqual(np.nansum(raw), self.executor.execute_tensor(nansum(arr))[0]) self.assertEqual(np.nanprod(raw), self.executor.execute_tensor(nanprod(arr))[0]) self.assertEqual(np.nanmax(raw), self.executor.execute_tensor(nanmax(arr))[0]) self.assertEqual(np.nanmin(raw), self.executor.execute_tensor(nanmin(arr))[0]) self.assertEqual(np.nanmean(raw), self.executor.execute_tensor(nanmean(arr))[0]) self.assertAlmostEqual(np.nanvar(raw), self.executor.execute_tensor(nanvar(arr))[0]) self.assertAlmostEqual( np.nanvar(raw, ddof=1), self.executor.execute_tensor(nanvar(arr, ddof=1))[0]) self.assertAlmostEqual(np.nanstd(raw), self.executor.execute_tensor(nanstd(arr))[0]) self.assertAlmostEqual( np.nanstd(raw, ddof=1), self.executor.execute_tensor(nanstd(arr, ddof=1))[0]) arr = tensor(raw, chunk_size=10) self.assertEqual(np.nansum(raw), self.executor.execute_tensor(nansum(arr))[0]) self.assertEqual(np.nanprod(raw), self.executor.execute_tensor(nanprod(arr))[0]) self.assertEqual(np.nanmax(raw), self.executor.execute_tensor(nanmax(arr))[0]) self.assertEqual(np.nanmin(raw), self.executor.execute_tensor(nanmin(arr))[0]) self.assertEqual(np.nanmean(raw), self.executor.execute_tensor(nanmean(arr))[0]) self.assertAlmostEqual(np.nanvar(raw), self.executor.execute_tensor(nanvar(arr))[0]) self.assertAlmostEqual( np.nanvar(raw, ddof=1), self.executor.execute_tensor(nanvar(arr, ddof=1))[0]) self.assertAlmostEqual(np.nanstd(raw), self.executor.execute_tensor(nanstd(arr))[0]) self.assertAlmostEqual( np.nanstd(raw, ddof=1), self.executor.execute_tensor(nanstd(arr, ddof=1))[0]) raw = np.random.random((10, 10)) raw[:3, :3] = np.nan arr = tensor(raw, chunk_size=3) self.assertEqual(np.nanargmin(raw), self.executor.execute_tensor(nanargmin(arr))[0]) self.assertEqual(np.nanargmax(raw), self.executor.execute_tensor(nanargmax(arr))[0]) raw = np.full((10, 10), np.nan) arr = tensor(raw, chunk_size=3) self.assertEqual(0, self.executor.execute_tensor(nansum(arr))[0]) self.assertEqual(1, self.executor.execute_tensor(nanprod(arr))[0]) self.assertTrue(np.isnan(self.executor.execute_tensor(nanmax(arr))[0])) self.assertTrue(np.isnan(self.executor.execute_tensor(nanmin(arr))[0])) self.assertTrue(np.isnan( self.executor.execute_tensor(nanmean(arr))[0])) with self.assertRaises(ValueError): _ = self.executor.execute_tensor(nanargmin(arr))[0] # noqa: F841 with self.assertRaises(ValueError): _ = self.executor.execute_tensor(nanargmax(arr))[0] # noqa: F841 raw = sps.random(10, 10, density=.1, format='csr') raw[:3, :3] = np.nan arr = tensor(raw, chunk_size=3) self.assertAlmostEqual(np.nansum(raw.A), self.executor.execute_tensor(nansum(arr))[0]) self.assertAlmostEqual(np.nanprod(raw.A), self.executor.execute_tensor(nanprod(arr))[0]) self.assertAlmostEqual(np.nanmax(raw.A), self.executor.execute_tensor(nanmax(arr))[0]) self.assertAlmostEqual(np.nanmin(raw.A), self.executor.execute_tensor(nanmin(arr))[0]) self.assertAlmostEqual(np.nanmean(raw.A), self.executor.execute_tensor(nanmean(arr))[0]) self.assertAlmostEqual(np.nanvar(raw.A), self.executor.execute_tensor(nanvar(arr))[0]) self.assertAlmostEqual( np.nanvar(raw.A, ddof=1), self.executor.execute_tensor(nanvar(arr, ddof=1))[0]) self.assertAlmostEqual(np.nanstd(raw.A), self.executor.execute_tensor(nanstd(arr))[0]) self.assertAlmostEqual( np.nanstd(raw.A, ddof=1), self.executor.execute_tensor(nanstd(arr, ddof=1))[0]) arr = nansum(1) self.assertEqual(self.executor.execute_tensor(arr)[0], 1) def testCumReduction(self): raw = np.random.randint(5, size=(8, 8, 8)) arr = tensor(raw, chunk_size=3) res1 = self.executor.execute_tensor(arr.cumsum(axis=1), concat=True) res2 = self.executor.execute_tensor(arr.cumprod(axis=1), concat=True) expected1 = raw.cumsum(axis=1) expected2 = raw.cumprod(axis=1) np.testing.assert_array_equal(res1[0], expected1) np.testing.assert_array_equal(res2[0], expected2) raw = sps.random(8, 8, density=.1) arr = tensor(raw, chunk_size=3) res1 = self.executor.execute_tensor(arr.cumsum(axis=1), concat=True) res2 = self.executor.execute_tensor(arr.cumprod(axis=1), concat=True) expected1 = raw.A.cumsum(axis=1) expected2 = raw.A.cumprod(axis=1) self.assertTrue(np.allclose(res1[0], expected1)) self.assertTrue(np.allclose(res2[0], expected2)) # test order raw = np.asfortranarray(np.random.rand(10, 20, 30)) arr = tensor(raw, chunk_size=13) arr2 = arr.cumsum(axis=-1) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = raw.cumsum(axis=-1) np.testing.assert_allclose(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) def testNanCumReduction(self): raw = np.random.randint(5, size=(8, 8, 8)) raw[:2, 2:4, 4:6] = np.nan arr = tensor(raw, chunk_size=3) res1 = self.executor.execute_tensor(nancumsum(arr, axis=1), concat=True) res2 = self.executor.execute_tensor(nancumprod(arr, axis=1), concat=True) expected1 = np.nancumsum(raw, axis=1) expected2 = np.nancumprod(raw, axis=1) np.testing.assert_array_equal(res1[0], expected1) np.testing.assert_array_equal(res2[0], expected2) raw = sps.random(8, 8, density=.1, format='lil') raw[:2, 2:4] = np.nan arr = tensor(raw, chunk_size=3) res1 = self.executor.execute_tensor(nancumsum(arr, axis=1), concat=True)[0] res2 = self.executor.execute_tensor(nancumprod(arr, axis=1), concat=True)[0] expected1 = np.nancumsum(raw.A, axis=1) expected2 = np.nancumprod(raw.A, axis=1) self.assertTrue(np.allclose(res1, expected1)) self.assertTrue(np.allclose(res2, expected2)) def testOutReductionExecution(self): raw = np.random.randint(5, size=(8, 8, 8)) arr = tensor(raw, chunk_size=3) arr2 = ones((8, 8), dtype='i8', chunk_size=3) arr.sum(axis=1, out=arr2) res = self.executor.execute_tensor(arr2, concat=True)[0] expected = raw.sum(axis=1) np.testing.assert_array_equal(res, expected) def testOutCumReductionExecution(self): raw = np.random.randint(5, size=(8, 8, 8)) arr = tensor(raw, chunk_size=3) arr.cumsum(axis=0, out=arr) res = self.executor.execute_tensor(arr, concat=True)[0] expected = raw.cumsum(axis=0) np.testing.assert_array_equal(res, expected) def testCountNonzeroExecution(self): raw = [[0, 1, 7, 0, 0], [3, 0, 0, 2, 19]] arr = tensor(raw, chunk_size=2) t = count_nonzero(arr) res = self.executor.execute_tensor(t)[0] expected = np.count_nonzero(raw) np.testing.assert_equal(res, expected) t = count_nonzero(arr, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.count_nonzero(raw, axis=0) np.testing.assert_equal(res, expected) t = count_nonzero(arr, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.count_nonzero(raw, axis=1) np.testing.assert_equal(res, expected) raw = sps.csr_matrix(raw) arr = tensor(raw, chunk_size=2) t = count_nonzero(arr) res = self.executor.execute_tensor(t)[0] expected = np.count_nonzero(raw.A) np.testing.assert_equal(res, expected) t = count_nonzero(arr, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.count_nonzero(raw.A, axis=0) np.testing.assert_equal(res, expected) t = count_nonzero(arr, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.count_nonzero(raw.A, axis=1) np.testing.assert_equal(res, expected) def testAllcloseExecution(self): a = tensor([1e10, 1e-7], chunk_size=1) b = tensor([1.00001e10, 1e-8], chunk_size=1) t = allclose(a, b) res = self.executor.execute_tensor(t)[0] self.assertFalse(res) a = tensor([1e10, 1e-8], chunk_size=1) b = tensor([1.00001e10, 1e-9], chunk_size=1) t = allclose(a, b) res = self.executor.execute_tensor(t)[0] self.assertTrue(res) a = tensor([1.0, np.nan], chunk_size=1) b = tensor([1.0, np.nan], chunk_size=1) t = allclose(a, b, equal_nan=True) res = self.executor.execute_tensor(t)[0] self.assertTrue(res) a = tensor(sps.csr_matrix([[1e10, 1e-7], [0, 0]]), chunk_size=1) b = tensor(sps.csr_matrix([[1.00001e10, 1e-8], [0, 0]]), chunk_size=1) t = allclose(a, b) res = self.executor.execute_tensor(t)[0] self.assertFalse(res) def testArrayEqual(self): a = ones((10, 5), chunk_size=1) b = ones((10, 5), chunk_size=2) c = array_equal(a, b) res = bool(self.executor.execute_tensor(c)[0]) self.assertTrue(res)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testQRExecution(self): data = np.random.randn(18, 6) a = tensor(data, chunk_size=(3, 6)) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(9, 6)) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=3) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) # test for Short-and-Fat QR data = np.random.randn(6, 18) a = tensor(data, chunk_size=(6, 9)) q, r = qr(a, method='sfqr') t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(3, 3)) q, r = qr(a, method='sfqr') t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(6, 3)) q, r = qr(a, method='sfqr') t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) def testSVDExecution(self): data = np.random.randn(18, 6) + 1j * np.random.randn(18, 6) a = tensor(data, chunk_size=(9, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(18, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(2, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) data = np.random.randn(6, 18) + 1j * np.random.randn(6, 18) a = tensor(data) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) # test for matrix of ones data = np.ones((20, 10)) a = tensor(data, chunk_size=10) s = svd(a)[1] res = self.executor.execute_tensor(s, concat=True)[0] expected = np.linalg.svd(a)[1] np.testing.assert_array_almost_equal(res, expected) def testCholeskyExecution(self): data = np.array([[1, -2j], [2j, 5]]) a = tensor(data, chunk_size=1) L = cholesky(a, lower=True) t = L.dot(L.T.conj()) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=2) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=(1, 2)) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) def testLUExecution(self): np.random.seed(1) data = np.random.randint(1, 10, (6, 6)) a = tensor(data) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=2) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=(2, 3)) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=4) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) # test for sparse data = sps.csr_matrix([[2, 0, 0, 0, 5, 2], [0, 6, 1, 0, 0, 6], [8, 0, 9, 0, 0, 2], [0, 6, 0, 8, 7, 3], [7, 0, 6, 1, 7, 0], [0, 0, 0, 7, 0, 8]]) a = tensor(data) P, L, U = lu(a) result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] # check lower and upper triangular matrix np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) self.assertIsInstance(result_l, SparseNDArray) self.assertIsInstance(result_u, SparseNDArray) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_array_almost_equal(data.A, res) a = tensor(data, chunk_size=2) P, L, U = lu(a) result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] # check lower and upper triangular matrix np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) self.assertIsInstance(result_l, SparseNDArray) self.assertIsInstance(result_u, SparseNDArray) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_array_almost_equal(data.A, res) a = tensor(data, chunk_size=(2, 3)) P, L, U = lu(a) result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] # check lower and upper triangular matrix np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) self.assertIsInstance(result_l, SparseNDArray) self.assertIsInstance(result_u, SparseNDArray) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_array_almost_equal(data.A, res) a = tensor(data, chunk_size=4) P, L, U = lu(a) result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] # check lower and upper triangular matrix np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) self.assertIsInstance(result_l, SparseNDArray) self.assertIsInstance(result_u, SparseNDArray) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_array_almost_equal(data.A, res) def testSolveTriangular(self): from mars.tensor import tril, triu np.random.seed(1) data1 = np.random.randint(1, 10, (20, 20)) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=20) b = tensor(data2, chunk_size=20) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) data1 = np.random.randint(1, 10, (10, 10)) data2 = np.random.randint(1, 10, (10, 5)) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=3) b = tensor(data2, chunk_size=3) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) # test sparse data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10)))) data2 = np.random.random((10,)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve_triangular(A, b) result_x = self.executor.execute_tensor(x, concat=True)[0] result_b = data1.dot(result_x) self.assertIsInstance(result_x, SparseNDArray) np.testing.assert_allclose(result_b, data2) data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10)))) data2 = np.random.random((10, 2)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve_triangular(A, b) result_x = self.executor.execute_tensor(x, concat=True)[0] result_b = data1.dot(result_x) self.assertIsInstance(result_x, SparseNDArray) np.testing.assert_allclose(result_b, data2) def testSolve(self): import scipy.linalg np.random.seed(1) data1 = np.random.randint(1, 10, (20, 20)) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) data2 = np.random.randint(1, 10, (20, 5)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) data2 = np.random.randint(1, 10, (20, 20)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) # test for not all chunks are square in matrix A data2 = np.random.randint(1, 10, (20,)) A = tensor(data1, chunk_size=6) b = tensor(data2, chunk_size=6) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=(7, 6)) b = tensor(data2, chunk_size=6) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) # test sparse data1 = sps.csr_matrix(np.random.randint(1, 10, (20, 20))) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] self.assertIsInstance(res, SparseNDArray) np.testing.assert_allclose(data1.dot(res), data2) data2 = np.random.randint(1, 10, (20, 5)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] self.assertIsInstance(res, SparseNDArray) np.testing.assert_allclose(res, data2) data2 = np.random.randint(1, 10, (20, 20)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] self.assertIsInstance(res, SparseNDArray) np.testing.assert_allclose(res, data2) # test for not all chunks are square in matrix A data2 = np.random.randint(1, 10, (20,)) A = tensor(data1, chunk_size=6) b = tensor(data2, chunk_size=6) x = solve(A, b) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) def testSolveSymPos(self): import scipy.linalg np.random.seed(1) data = np.random.randint(1, 10, (20, 20)) data_l = np.tril(data) data1 = data_l.dot(data_l.T) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b, sym_pos=True) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) def testInv(self): import scipy.linalg np.random.seed(1) data = np.random.randint(1, 10, (20, 20)) A = tensor(data) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) A = tensor(data, chunk_size=5) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) B = A.T.dot(A) inv_B = inv(B) res = self.executor.execute_tensor(inv_B, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data.T.dot(data)))) res = self.executor.execute_tensor(B.dot(inv_B), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test for not all chunks are square in matrix A A = tensor(data, chunk_size=8) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test sparse data = np.random.randint(1, 10, (20, 20)) sp_data = sps.csr_matrix(data) A = tensor(sp_data, chunk_size=5) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertIsInstance(res, SparseNDArray) self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test for not all chunks are square in matrix A A = tensor(sp_data, chunk_size=8) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertIsInstance(res, SparseNDArray) self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) def testNormExecution(self): d = np.arange(9) - 4 d2 = d.reshape(3, 3) ma = [tensor(d, chunk_size=2), tensor(d, chunk_size=9), tensor(d2, chunk_size=(2, 3)), tensor(d2, chunk_size=3)] for i, a in enumerate(ma): data = d if i < 2 else d2 for ord in (None, 'nuc', np.inf, -np.inf, 0, 1, -1, 2, -2): for axis in (0, 1, (0, 1)): for keepdims in (True, False): try: expected = np.linalg.norm(data, ord=ord, axis=axis, keepdims=keepdims) t = norm(a, ord=ord, axis=axis, keepdims=keepdims) concat = t.ndim > 0 res = self.executor.execute_tensor(t, concat=concat)[0] np.testing.assert_allclose(res, expected, atol=.0001) except ValueError: continue m = norm(tensor(d)) expected = self.executor.execute_tensor(m)[0] res = np.linalg.norm(d) self.assertEqual(expected, res) d = uniform(-0.5, 0.5, size=(500, 2), chunk_size=50) inside = (norm(d, axis=1) < 0.5).sum().astype(float) t = inside / 500 * 4 res = self.executor.execute_tensor(t)[0] self.assertAlmostEqual(res, 3.14, delta=1) def testTensordotExecution(self): size_executor = Executor(sync_provider_type=Executor.SyncProviderType.MOCK) a_data = np.arange(60).reshape(3, 4, 5) a = tensor(a_data, chunk_size=2) b_data = np.arange(24).reshape(4, 3, 2) b = tensor(b_data, chunk_size=2) axes = ([1, 0], [0, 1]) c = tensordot(a, b, axes=axes) size_res = size_executor.execute_tensor(c, mock=True) res = self.executor.execute_tensor(c) expected = np.tensordot(a_data, b_data, axes=axes) self.assertEqual(sum(s[0] for s in size_res), c.nbytes) self.assertTrue(np.array_equal(res[0], expected[:2, :])) self.assertTrue(np.array_equal(res[1], expected[2:4, :])) self.assertTrue(np.array_equal(res[2], expected[4:, :])) a = ones((1000, 2000), chunk_size=500) b = ones((2000, 100), chunk_size=500) c = dot(a, b) res = self.executor.execute_tensor(c) expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((10, 8), chunk_size=2) b = ones((8, 10), chunk_size=2) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertEqual(len(res), 25) for r in res: self.assertTrue(np.array_equal(r, np.tile([8], [2, 2]))) a = ones((500, 500), chunk_size=500) b = ones((500, 100), chunk_size=500) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100]))) raw_a = np.random.random((100, 200, 50)) raw_b = np.random.random((200, 10, 100)) a = tensor(raw_a, chunk_size=50) b = tensor(raw_b, chunk_size=33) c = tensordot(a, b, axes=((0, 1), (2, 0))) res = self.executor.execute_tensor(c, concat=True) expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes)) self.assertTrue(np.allclose(res[0], expected)) a = ones((1000, 2000), chunk_size=500) b = ones((100, 2000), chunk_size=500) c = inner(a, b) res = self.executor.execute_tensor(c) expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((100, 100), chunk_size=30) b = ones((100, 100), chunk_size=30) c = a.dot(b) res = self.executor.execute_tensor(c, concat=True)[0] np.testing.assert_array_equal(res, np.ones((100, 100)) * 100) def testSparseDotExecution(self): size_executor = Executor(sync_provider_type=Executor.SyncProviderType.MOCK) a_data = sps.random(5, 9, density=.1) b_data = sps.random(9, 10, density=.2) a = tensor(a_data, chunk_size=2) b = tensor(b_data, chunk_size=3) c = dot(a, b) size_res = size_executor.execute_tensor(c, mock=True) res = self.executor.execute_tensor(c, concat=True)[0] self.assertEqual(sum(s[0] for s in size_res), 0) self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c2 = dot(a, b, sparse=False) res = self.executor.execute_tensor(c2, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c3 = tensordot(a, b.T, (-1, -1), sparse=False) res = self.executor.execute_tensor(c3, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c = inner(a, b.T) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c = inner(a, b.T, sparse=False) res = self.executor.execute_tensor(c, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) # test vector inner a_data = np.random.rand(5) b_data = np.random.rand(5) a = tensor(a_data, chunk_size=2).tosparse() b = tensor(b_data, chunk_size=2).tosparse() c = inner(a, b) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(np.isscalar(res)) np.testing.assert_allclose(res, np.inner(a_data, b_data)) def testVdotExecution(self): a_data = np.array([1 + 2j, 3 + 4j]) b_data = np.array([5 + 6j, 7 + 8j]) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=1) t = vdot(a, b) res = self.executor.execute_tensor(t)[0] expected = np.vdot(a_data, b_data) np.testing.assert_equal(res, expected) a_data = np.array([[1, 4], [5, 6]]) b_data = np.array([[4, 1], [2, 2]]) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=1) t = vdot(a, b) res = self.executor.execute_tensor(t)[0] expected = np.vdot(a_data, b_data) np.testing.assert_equal(res, expected) def testMatmulExecution(self): data_a = np.random.randn(10, 20) data_b = np.random.randn(20) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b) np.testing.assert_allclose(res, expected) data_a = np.random.randn(10, 20) data_b = np.random.randn(10) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(b, a) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_b, data_a) np.testing.assert_allclose(res, expected) data_a = np.random.randn(15, 1, 20, 30) data_b = np.random.randn(1, 11, 30, 20) a = tensor(data_a, chunk_size=12) b = tensor(data_b, chunk_size=13) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b) np.testing.assert_allclose(res, expected, atol=.0001) a = arange(2 * 2 * 4, chunk_size=1).reshape((2, 2, 4)) b = arange(2 * 2 * 4, chunk_size=1).reshape((2, 4, 2)) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(np.arange(2 * 2 * 4).reshape(2, 2, 4), np.arange(2 * 2 * 4).reshape(2, 4, 2)) np.testing.assert_allclose(res, expected, atol=.0001) data_a = sps.random(10, 20) data_b = sps.random(20, 5) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a.toarray(), data_b.toarray()) np.testing.assert_allclose(res.toarray(), expected)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor() def testBaseExecution(self): executor_numpy = Executor('numpy') raw1 = np.random.randint(10, size=(10, 10, 10)) raw2 = np.random.randint(10, size=(10, 10, 10)) arr1 = tensor(raw1, chunk_size=3) arr2 = tensor(raw2, chunk_size=3) arr3 = arr1 + arr2 + 10 arr4 = 10 + arr1 + arr2 res3 = executor_numpy.execute_tensor(arr3, concat=True) res3_cmp = self.executor.execute_tensor(arr4, concat=True) self.assertTrue(np.array_equal(res3[0], res3_cmp[0])) res5 = executor_numpy.execute_tensor((arr1 + arr1), concat=True) res5_cmp = self.executor.execute_tensor((arr1 + arr1), concat=True) self.assertTrue(np.array_equal(res5[0], res5_cmp[0])) def testFuseSizeExecution(self): executor_size = Executor() executor_numpy = Executor() raw1 = np.random.randint(10, size=(10, 10, 10)) arr1 = tensor(raw1, chunk_size=3) arr2 = arr1 + 10 arr3 = arr2 * 3 arr4 = arr3 + 5 res4_size = executor_size.execute_tensor(arr4, mock=True) res4 = executor_numpy.execute_tensor(arr4, concat=True) res4_cmp = self.executor.execute_tensor(arr4, concat=True) self.assertEqual(sum(s[0] for s in res4_size), arr4.nbytes) self.assertTrue(np.array_equal(res4[0], res4_cmp[0])) def testUnaryExecution(self): from mars.tensor.expressions.arithmetic import UNARY_UFUNC, arccosh, invert, sin, conj _sp_unary_ufunc = {arccosh, invert, conj} _new_unary_ufunc = list(UNARY_UFUNC - _sp_unary_ufunc) executor_numexpr = Executor() def _normalize_by_sin(func1, func2, arr): return func1(abs(sin((func2(arr))))) for i, j in itertools.permutations(range(len(_new_unary_ufunc)), 2): raw = np.random.random((8, 8, 8)) arr1 = tensor(raw, chunk_size=4) func1 = _new_unary_ufunc[i] func2 = _new_unary_ufunc[j] arr2 = _normalize_by_sin(func1, func2, arr1) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) np.testing.assert_allclose(res[0], res_cmp[0]) raw = np.random.randint(100, size=(8, 8, 8)) arr1 = tensor(raw, chunk_size=4) arr2 = arccosh(1 + abs(invert(arr1))) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0])) def testBinExecution(self): from mars.tensor.expressions.arithmetic import BIN_UFUNC, mod, fmod, \ bitand, bitor, bitxor, lshift, rshift, ldexp _sp_bin_ufunc = [mod, fmod, bitand, bitor, bitxor, lshift, rshift] _new_bin_ufunc = list(BIN_UFUNC - set(_sp_bin_ufunc) - {ldexp}) executor_numexpr = Executor() for i, j in itertools.permutations(range(len(_new_bin_ufunc)), 2): raw = np.random.random((9, 9, 9)) arr1 = tensor(raw, chunk_size=5) func1 = _new_bin_ufunc[i] func2 = _new_bin_ufunc[j] arr2 = func1(1, func2(2, arr1)) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0])) for i, j in itertools.permutations(range(len(_sp_bin_ufunc)), 2): raw = np.random.randint(1, 100, size=(10, 10, 10)) arr1 = tensor(raw, chunk_size=3) func1 = _sp_bin_ufunc[i] func2 = _sp_bin_ufunc[j] arr2 = func1(10, func2(arr1, 5)) res = executor_numexpr.execute_tensor(arr2, concat=True) res_cmp = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.allclose(res[0], res_cmp[0])) def testReductionExecution(self): raw1 = np.random.randint(5, size=(8, 8, 8)) raw2 = np.random.randint(5, size=(8, 8, 8)) arr1 = tensor(raw1, chunk_size=3) arr2 = tensor(raw2, chunk_size=3) res1 = self.executor.execute_tensor((arr1 + 1).sum(keepdims=True)) res2 = self.executor.execute_tensor((arr1 + 1).prod(keepdims=True)) self.assertTrue(np.array_equal((raw1 + 1).sum(keepdims=True), res1[0])) self.assertTrue(np.array_equal((raw1 + 1).prod(keepdims=True), res2[0])) res1 = self.executor.execute_tensor((arr1 + 1).sum(axis=1), concat=True) res2 = self.executor.execute_tensor((arr1 + 1).prod(axis=1), concat=True) res3 = self.executor.execute_tensor((arr1 + 1).max(axis=1), concat=True) res4 = self.executor.execute_tensor((arr1 + 1).min(axis=1), concat=True) self.assertTrue(np.array_equal((raw1 + 1).sum(axis=1), res1[0])) self.assertTrue(np.array_equal((raw1 + 1).prod(axis=1), res2[0])) self.assertTrue(np.array_equal((raw1 + 1).max(axis=1), res3[0])) self.assertTrue(np.array_equal((raw1 + 1).min(axis=1), res4[0])) raw3 = raw2 - raw1 + 10 arr3 = -arr1 + arr2 + 10 res1 = self.executor.execute_tensor(arr3.sum(axis=(0, 1)), concat=True) res2 = self.executor.execute_tensor(arr3.prod(axis=(0, 1)), concat=True) res3 = self.executor.execute_tensor(arr3.max(axis=(0, 1)), concat=True) res4 = self.executor.execute_tensor(arr3.min(axis=(0, 1)), concat=True) self.assertTrue(np.array_equal(raw3.sum(axis=(0, 1)), res1[0])) self.assertTrue(np.array_equal(raw3.prod(axis=(0, 1)), res2[0])) self.assertTrue(np.array_equal(raw3.max(axis=(0, 1)), res3[0])) self.assertTrue(np.array_equal(raw3.min(axis=(0, 1)), res4[0])) def testBoolReductionExecution(self): raw = np.random.randint(5, size=(8, 8, 8)) arr = tensor(raw, chunk_size=2) res = self.executor.execute_tensor((arr > 3).sum(axis=1), concat=True) np.testing.assert_array_equal(res[0], (raw > 3).sum(axis=1)) res = self.executor.execute_tensor((arr > 3).sum()) np.testing.assert_array_equal(res, (raw > 3).sum())
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testQRExecution(self): data = np.random.randn(18, 6) a = tensor(data, chunk_size=(3, 6)) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(9, 6)) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=3) q, r = qr(a) t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) # test for Short-and-Fat QR data = np.random.randn(6, 18) a = tensor(data, chunk_size=(6, 9)) q, r = qr(a, method='sfqr') t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(3, 3)) q, r = qr(a, method='sfqr') t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(6, 3)) q, r = qr(a, method='sfqr') t = q.dot(r) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) def testSVDExecution(self): data = np.random.randn(18, 6) + 1j * np.random.randn(18, 6) a = tensor(data, chunk_size=(9, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(18, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) a = tensor(data, chunk_size=(2, 6)) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) data = np.random.randn(6, 18) + 1j * np.random.randn(6, 18) a = tensor(data) U, s, V = svd(a) t = U.dot(diag(s).dot(V)) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, data)) # test for matrix of ones data = np.ones((20, 10)) a = tensor(data, chunk_size=10) s = svd(a)[1] res = self.executor.execute_tensor(s, concat=True)[0] expected = np.linalg.svd(a)[1] np.testing.assert_array_almost_equal(res, expected) def testRandomizedSVDExecution(self): n_samples = 100 n_features = 500 rank = 5 k = 10 for dtype in (np.int32, np.int64, np.float32, np.float64): # generate a matrix X of approximate effective rank `rank` and no noise # component (very structured signal): X = make_low_rank_matrix(n_samples=n_samples, n_features=n_features, effective_rank=rank, tail_strength=0.0, random_state=0).astype(dtype, copy=False) self.assertEqual(X.shape, (n_samples, n_features)) dtype = np.dtype(dtype) decimal = 5 if dtype == np.float32 else 7 # compute the singular values of X using the slow exact method X_res = self.executor.execute_tensor(X, concat=True)[0] U, s, V = np.linalg.svd(X_res, full_matrices=False) # Convert the singular values to the specific dtype U = U.astype(dtype, copy=False) s = s.astype(dtype, copy=False) V = V.astype(dtype, copy=False) for normalizer in ['auto', 'LU', 'QR']: # 'none' would not be stable # compute the singular values of X using the fast approximate method Ua, sa, Va = randomized_svd( X, k, power_iteration_normalizer=normalizer, random_state=0) # If the input dtype is float, then the output dtype is float of the # same bit size (f32 is not upcast to f64) # But if the input dtype is int, the output dtype is float64 if dtype.kind == 'f': self.assertEqual(Ua.dtype, dtype) self.assertEqual(sa.dtype, dtype) self.assertEqual(Va.dtype, dtype) else: self.assertEqual(Ua.dtype, np.float64) self.assertEqual(sa.dtype, np.float64) self.assertEqual(Va.dtype, np.float64) self.assertEqual(Ua.shape, (n_samples, k)) self.assertEqual(sa.shape, (k, )) self.assertEqual(Va.shape, (k, n_features)) # ensure that the singular values of both methods are equal up to the # real rank of the matrix sa_res = self.executor.execute_tensor(sa, concat=True)[0] np.testing.assert_almost_equal(s[:k], sa_res, decimal=decimal) # check the singular vectors too (while not checking the sign) dot_res = self.executor.execute_tensor(dot(Ua, Va), concat=True)[0] np.testing.assert_almost_equal(np.dot(U[:, :k], V[:k, :]), dot_res, decimal=decimal) def testCholeskyExecution(self): data = np.random.randint(1, 10, (10, 10)) symmetric_data = data.dot(data.T) a = tensor(symmetric_data, chunk_size=5) U = cholesky(a) t = U.T.dot(U) res_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.triu(res_u), res_u) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, symmetric_data)) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.allclose(res, symmetric_data)) a = tensor(symmetric_data, chunk_size=2) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, symmetric_data) a = tensor(symmetric_data, chunk_size=(1, 2)) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, symmetric_data) a = tensor(symmetric_data, chunk_size=4) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.triu(res_u), res_u) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, symmetric_data) a = tensor(symmetric_data, chunk_size=3) L = cholesky(a, lower=True) U = cholesky(a) t = L.dot(U) res_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.triu(res_u), res_u) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, symmetric_data) def testLUExecution(self): np.random.seed(1) # square matrix data = np.random.randint(1, 10, (6, 6)) a = tensor(data) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=2) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=(2, 3)) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=4) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) # shape[0] > shape[1] data = np.random.randint(1, 10, (10, 6)) a = tensor(data) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=2) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=(2, 3)) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=4) P, L, U = lu(a) # check lower and upper triangular matrix result_l, result_u = self.executor.execute_tensors([L, U]) np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) # shape[0] < shape[1] data = np.random.randint(1, 10, (6, 10)) a = tensor(data) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=2) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=(2, 3)) P, L, U = lu(a) # check lower and upper triangular matrix result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) a = tensor(data, chunk_size=4) P, L, U = lu(a) # check lower and upper triangular matrix result_l, result_u = self.executor.execute_tensors([L, U]) np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data) # test for sparse data = sps.csr_matrix([[2, 0, 0, 0, 5, 2], [0, 6, 1, 0, 0, 6], [8, 0, 9, 0, 0, 2], [0, 6, 0, 8, 7, 3], [7, 0, 6, 1, 7, 0], [0, 0, 0, 7, 0, 8]]) a = tensor(data) P, L, U = lu(a) result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] # check lower and upper triangular matrix np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) self.assertIsInstance(result_l, SparseNDArray) self.assertIsInstance(result_u, SparseNDArray) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_array_almost_equal(data.A, res) a = tensor(data, chunk_size=2) P, L, U = lu(a) result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] # check lower and upper triangular matrix np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) self.assertIsInstance(result_l, SparseNDArray) self.assertIsInstance(result_u, SparseNDArray) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_array_almost_equal(data.A, res) a = tensor(data, chunk_size=(2, 3)) P, L, U = lu(a) result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] # check lower and upper triangular matrix np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) self.assertIsInstance(result_l, SparseNDArray) self.assertIsInstance(result_u, SparseNDArray) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_array_almost_equal(data.A, res) a = tensor(data, chunk_size=4) P, L, U = lu(a) result_l = self.executor.execute_tensor(L, concat=True)[0] result_u = self.executor.execute_tensor(U, concat=True)[0] # check lower and upper triangular matrix np.testing.assert_allclose(np.tril(result_l), result_l) np.testing.assert_allclose(np.triu(result_u), result_u) self.assertIsInstance(result_l, SparseNDArray) self.assertIsInstance(result_u, SparseNDArray) t = P.dot(L).dot(U) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_array_almost_equal(data.A, res) def testSolveTriangular(self): from mars.tensor import tril, triu np.random.seed(1) data1 = np.random.randint(1, 10, (20, 20)) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=20) b = tensor(data2, chunk_size=20) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) data1 = np.random.randint(1, 10, (10, 10)) data2 = np.random.randint(1, 10, (10, 5)) A = tensor(data1, chunk_size=10) b = tensor(data2, chunk_size=10) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=3) b = tensor(data2, chunk_size=3) x = solve_triangular(A, b) t = triu(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) x = solve_triangular(A, b, lower=True) t = tril(A).dot(x) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, data2) # test sparse data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10)))) data2 = np.random.random((10, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve_triangular(A, b) result_x = self.executor.execute_tensor(x, concat=True)[0] result_b = data1.dot(result_x) self.assertIsInstance(result_x, SparseNDArray) np.testing.assert_allclose(result_b, data2) data1 = sps.csr_matrix(np.triu(np.random.randint(1, 10, (10, 10)))) data2 = np.random.random((10, 2)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve_triangular(A, b) result_x = self.executor.execute_tensor(x, concat=True)[0] result_b = data1.dot(result_x) self.assertIsInstance(result_x, SparseNDArray) np.testing.assert_allclose(result_b, data2) def testSolve(self): import scipy.linalg np.random.seed(1) data1 = np.random.randint(1, 10, (20, 20)) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) data2 = np.random.randint(1, 10, (20, 5)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) data2 = np.random.randint(1, 10, (20, 20)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) # test for not all chunks are square in matrix A data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=6) b = tensor(data2, chunk_size=6) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) A = tensor(data1, chunk_size=(7, 6)) b = tensor(data2, chunk_size=6) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) # test sparse data1 = sps.csr_matrix(np.random.randint(1, 10, (20, 20))) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(x, concat=True)[0] self.assertIsInstance(res, SparseNDArray) np.testing.assert_allclose(data1.dot(res), data2) data2 = np.random.randint(1, 10, (20, 5)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] self.assertIsInstance(res, SparseNDArray) np.testing.assert_allclose(res, data2) data2 = np.random.randint(1, 10, (20, 20)) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] self.assertIsInstance(res, SparseNDArray) np.testing.assert_allclose(res, data2) # test for not all chunks are square in matrix A data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=6) b = tensor(data2, chunk_size=6) x = solve(A, b) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) def testSolveSymPos(self): import scipy.linalg np.random.seed(1) data = np.random.randint(1, 10, (20, 20)) data_l = np.tril(data) data1 = data_l.dot(data_l.T) data2 = np.random.randint(1, 10, (20, )) A = tensor(data1, chunk_size=5) b = tensor(data2, chunk_size=5) x = solve(A, b, sym_pos=True) res = self.executor.execute_tensor(x, concat=True)[0] np.testing.assert_allclose(res, scipy.linalg.solve(data1, data2)) res = self.executor.execute_tensor(A.dot(x), concat=True)[0] np.testing.assert_allclose(res, data2) def testInv(self): import scipy.linalg np.random.seed(1) data = np.random.randint(1, 10, (20, 20)) A = tensor(data) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) A = tensor(data, chunk_size=5) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) B = A.T.dot(A) inv_B = inv(B) res = self.executor.execute_tensor(inv_B, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data.T.dot(data)))) res = self.executor.execute_tensor(B.dot(inv_B), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test for not all chunks are square in matrix A A = tensor(data, chunk_size=8) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test sparse data = np.random.randint(1, 10, (20, 20)) sp_data = sps.csr_matrix(data) A = tensor(sp_data, chunk_size=5) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertIsInstance(res, SparseNDArray) self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) # test for not all chunks are square in matrix A A = tensor(sp_data, chunk_size=8) inv_A = inv(A) res = self.executor.execute_tensor(inv_A, concat=True)[0] self.assertIsInstance(res, SparseNDArray) self.assertTrue(np.allclose(res, scipy.linalg.inv(data))) res = self.executor.execute_tensor(A.dot(inv_A), concat=True)[0] self.assertTrue(np.allclose(res, np.eye(data.shape[0], dtype=float))) def testNormExecution(self): d = np.arange(9) - 4 d2 = d.reshape(3, 3) ma = [ tensor(d, chunk_size=2), tensor(d, chunk_size=9), tensor(d2, chunk_size=(2, 3)), tensor(d2, chunk_size=3) ] for i, a in enumerate(ma): data = d if i < 2 else d2 for ord in (None, 'nuc', np.inf, -np.inf, 0, 1, -1, 2, -2): for axis in (0, 1, (0, 1)): for keepdims in (True, False): try: expected = np.linalg.norm(data, ord=ord, axis=axis, keepdims=keepdims) t = norm(a, ord=ord, axis=axis, keepdims=keepdims) concat = t.ndim > 0 res = self.executor.execute_tensor( t, concat=concat)[0] np.testing.assert_allclose(res, expected, atol=.0001) except ValueError: continue m = norm(tensor(d)) expected = self.executor.execute_tensor(m)[0] res = np.linalg.norm(d) self.assertEqual(expected, res) d = uniform(-0.5, 0.5, size=(500, 2), chunk_size=50) inside = (norm(d, axis=1) < 0.5).sum().astype(float) t = inside / 500 * 4 res = self.executor.execute_tensor(t)[0] self.assertAlmostEqual(res, 3.14, delta=1) def testTensordotExecution(self): size_executor = Executor( sync_provider_type=Executor.SyncProviderType.MOCK) a_data = np.arange(60).reshape(3, 4, 5) a = tensor(a_data, chunk_size=2) b_data = np.arange(24).reshape(4, 3, 2) b = tensor(b_data, chunk_size=2) axes = ([1, 0], [0, 1]) c = tensordot(a, b, axes=axes) size_res = size_executor.execute_tensor(c, mock=True) self.assertEqual(sum(s[0] for s in size_res), c.nbytes) self.assertEqual(sum(s[1] for s in size_res), c.nbytes) res = self.executor.execute_tensor(c) expected = np.tensordot(a_data, b_data, axes=axes) self.assertTrue(np.array_equal(res[0], expected[:2, :])) self.assertTrue(np.array_equal(res[1], expected[2:4, :])) self.assertTrue(np.array_equal(res[2], expected[4:, :])) a = ones((1000, 2000), chunk_size=500) b = ones((2000, 100), chunk_size=500) c = dot(a, b) res = self.executor.execute_tensor(c) expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((10, 8), chunk_size=2) b = ones((8, 10), chunk_size=2) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertEqual(len(res), 25) for r in res: self.assertTrue(np.array_equal(r, np.tile([8], [2, 2]))) a = ones((500, 500), chunk_size=500) b = ones((500, 100), chunk_size=500) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100]))) raw_a = np.random.random((100, 200, 50)) raw_b = np.random.random((200, 10, 100)) a = tensor(raw_a, chunk_size=50) b = tensor(raw_b, chunk_size=33) c = tensordot(a, b, axes=((0, 1), (2, 0))) res = self.executor.execute_tensor(c, concat=True) expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes)) self.assertTrue(np.allclose(res[0], expected)) a = ones((1000, 2000), chunk_size=500) b = ones((100, 2000), chunk_size=500) c = inner(a, b) res = self.executor.execute_tensor(c) expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((100, 100), chunk_size=30) b = ones((100, 100), chunk_size=30) c = a.dot(b) res = self.executor.execute_tensor(c, concat=True)[0] np.testing.assert_array_equal(res, np.ones((100, 100)) * 100) def testSparseDotSizeExecution(self): from mars.tensor.linalg.tensordot import TensorTensorDot from mars.executor import register, register_default chunk_sizes = dict() chunk_nbytes = dict() chunk_input_sizes = dict() chunk_input_nbytes = dict() def execute_size(t): def _tensordot_size_recorder(ctx, op): TensorTensorDot.estimate_size(ctx, op) chunk_key = op.outputs[0].key chunk_sizes[chunk_key] = ctx[chunk_key] chunk_nbytes[chunk_key] = op.outputs[0].nbytes input_sizes = dict( (inp.op.key, ctx[inp.key][0]) for inp in op.inputs) chunk_input_sizes[chunk_key] = sum(input_sizes.values()) input_nbytes = dict( (inp.op.key, inp.nbytes) for inp in op.inputs) chunk_input_nbytes[chunk_key] = sum(input_nbytes.values()) size_executor = Executor( sync_provider_type=Executor.SyncProviderType.MOCK) try: chunk_sizes.clear() chunk_nbytes.clear() chunk_input_sizes.clear() chunk_input_nbytes.clear() register(TensorTensorDot, size_estimator=_tensordot_size_recorder) size_executor.execute_tensor(t, mock=True) finally: register_default(TensorTensorDot) a_data = sps.random(5, 9, density=.1) b_data = sps.random(9, 10, density=.2) a = tensor(a_data, chunk_size=2) b = tensor(b_data, chunk_size=3) c = dot(a, b) execute_size(c) for key in chunk_input_sizes.keys(): self.assertGreaterEqual(chunk_sizes[key][1], chunk_input_sizes[key]) c2 = dot(a, b, sparse=False) execute_size(c2) for key in chunk_input_sizes.keys(): self.assertEqual(chunk_sizes[key][0], chunk_nbytes[key]) self.assertEqual(chunk_sizes[key][1], chunk_input_nbytes[key] + chunk_nbytes[key]) def testSparseDotExecution(self): a_data = sps.random(5, 9, density=.1) b_data = sps.random(9, 10, density=.2) a = tensor(a_data, chunk_size=2) b = tensor(b_data, chunk_size=3) c = dot(a, b) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c2 = dot(a, b, sparse=False) res = self.executor.execute_tensor(c2, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c3 = tensordot(a, b.T, (-1, -1), sparse=False) res = self.executor.execute_tensor(c3, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) c = inner(a, b.T) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(issparse(res)) np.testing.assert_allclose(res.toarray(), a_data.dot(b_data).toarray()) c = inner(a, b.T, sparse=False) res = self.executor.execute_tensor(c, concat=True)[0] self.assertFalse(issparse(res)) np.testing.assert_allclose(res, a_data.dot(b_data).toarray()) # test vector inner a_data = np.random.rand(5) b_data = np.random.rand(5) a = tensor(a_data, chunk_size=2).tosparse() b = tensor(b_data, chunk_size=2).tosparse() c = inner(a, b) res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(np.isscalar(res)) np.testing.assert_allclose(res, np.inner(a_data, b_data)) def testVdotExecution(self): a_data = np.array([1 + 2j, 3 + 4j]) b_data = np.array([5 + 6j, 7 + 8j]) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=1) t = vdot(a, b) res = self.executor.execute_tensor(t)[0] expected = np.vdot(a_data, b_data) np.testing.assert_equal(res, expected) a_data = np.array([[1, 4], [5, 6]]) b_data = np.array([[4, 1], [2, 2]]) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=1) t = vdot(a, b) res = self.executor.execute_tensor(t)[0] expected = np.vdot(a_data, b_data) np.testing.assert_equal(res, expected) def testMatmulExecution(self): data_a = np.random.randn(10, 20) data_b = np.random.randn(20) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b) np.testing.assert_allclose(res, expected) data_a = np.random.randn(10, 20) data_b = np.random.randn(10) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(b, a) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_b, data_a) np.testing.assert_allclose(res, expected) data_a = np.random.randn(15, 1, 20, 30) data_b = np.random.randn(1, 11, 30, 20) a = tensor(data_a, chunk_size=12) b = tensor(data_b, chunk_size=13) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b) np.testing.assert_allclose(res, expected, atol=.0001) a = arange(2 * 2 * 4, chunk_size=1).reshape((2, 2, 4)) b = arange(2 * 2 * 4, chunk_size=1).reshape((2, 4, 2)) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul( np.arange(2 * 2 * 4).reshape(2, 2, 4), np.arange(2 * 2 * 4).reshape(2, 4, 2)) np.testing.assert_allclose(res, expected, atol=.0001) data_a = sps.random(10, 20) data_b = sps.random(20, 5) a = tensor(data_a, chunk_size=2) b = tensor(data_b, chunk_size=3) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a.toarray(), data_b.toarray()) np.testing.assert_allclose(res.toarray(), expected) # test order data_a = np.asfortranarray(np.random.randn(10, 20)) data_b = np.asfortranarray(np.random.randn(20, 30)) a = tensor(data_a, chunk_size=12) b = tensor(data_b, chunk_size=13) c = matmul(a, b) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b) np.testing.assert_allclose(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) c = matmul(a, b, order='A') res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b, order='A') np.testing.assert_allclose(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS']) c = matmul(a, b, order='C') res = self.executor.execute_tensor(c, concat=True)[0] expected = np.matmul(data_a, data_b, order='C') np.testing.assert_allclose(res, expected) self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS']) self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS'])
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def _nan_equal(self, a, b): try: np.testing.assert_equal(a, b) except AssertionError: return False return True def testBaseExecution(self): arr = ones((10, 8), chunk_size=2) arr2 = arr + 1 res = self.executor.execute_tensor(arr2) self.assertTrue((res[0] == np.ones((2, 2)) + 1).all()) data = np.random.random((10, 8, 3)) arr = tensor(data, chunk_size=2) arr2 = arr + 1 res = self.executor.execute_tensor(arr2) self.assertTrue((res[0] == data[:2, :2, :2] + 1).all()) @staticmethod def _get_func(op): if isinstance(op, six.string_types): return getattr(np, op) return op def testUfuncExecution(self): from mars.tensor.expressions.arithmetic import UNARY_UFUNC, BIN_UFUNC, arccosh, \ invert, mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp from mars.tensor.execution.arithmetic import OP_TO_HANDLER _sp_unary_ufunc = {arccosh, invert} _sp_bin_ufunc = { mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp } data1 = np.random.random((5, 9, 4)) data2 = np.random.random((5, 9, 4)) rand = np.random.random() arr1 = tensor(data1, chunk_size=3) arr2 = tensor(data2, chunk_size=3) _new_unary_ufunc = UNARY_UFUNC - _sp_unary_ufunc for func in _new_unary_ufunc: res_tensor = func(arr1) res = self.executor.execute_tensor(res_tensor, concat=True) expected = self._get_func(OP_TO_HANDLER[type( res_tensor.op)])(data1) self.assertTrue(np.allclose(res[0], expected)) _new_bin_ufunc = BIN_UFUNC - _sp_bin_ufunc for func in _new_bin_ufunc: res_tensor1 = func(arr1, arr2) res_tensor2 = func(arr1, rand) res_tensor3 = func(rand, arr1) res1 = self.executor.execute_tensor(res_tensor1, concat=True) res2 = self.executor.execute_tensor(res_tensor2, concat=True) res3 = self.executor.execute_tensor(res_tensor3, concat=True) expected1 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( data1, data2) expected2 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( data1, rand) expected3 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( rand, data1) self.assertTrue(np.allclose(res1[0], expected1)) self.assertTrue(np.allclose(res2[0], expected2)) self.assertTrue(np.allclose(res3[0], expected3)) data1 = np.random.randint(2, 10, size=(10, 10, 10)) data2 = np.random.randint(2, 10, size=(10, 10, 10)) rand = np.random.randint(1, 10) arr1 = tensor(data1, chunk_size=3) arr2 = tensor(data2, chunk_size=3) for func in _sp_unary_ufunc: res_tensor = func(arr1) res = self.executor.execute_tensor(res_tensor, concat=True) expected = self._get_func(OP_TO_HANDLER[type( res_tensor.op)])(data1) self.assertTrue(np.allclose(res[0], expected)) for func in _sp_bin_ufunc: res_tensor1 = func(arr1, arr2) res_tensor2 = func(arr1, rand) res_tensor3 = func(rand, arr1) res1 = self.executor.execute_tensor(res_tensor1, concat=True) res2 = self.executor.execute_tensor(res_tensor2, concat=True) res3 = self.executor.execute_tensor(res_tensor3, concat=True) expected1 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( data1, data2) expected2 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( data1, rand) expected3 = self._get_func(OP_TO_HANDLER[type(res_tensor1.op)])( rand, data1) self.assertTrue(np.allclose(res1[0], expected1)) self.assertTrue(np.allclose(res2[0], expected2)) self.assertTrue(np.allclose(res3[0], expected3)) @staticmethod def _get_sparse_func(op): from mars.lib.sparse.core import issparse if isinstance(op, six.string_types): op = getattr(np, op) def func(*args): new_args = [] for arg in args: if issparse(arg): new_args.append(arg.toarray()) else: new_args.append(arg) return op(*new_args) return func @staticmethod def toarray(x): if hasattr(x, 'toarray'): return x.toarray() return x def testSparseUfuncExexution(self): from mars.tensor.expressions.arithmetic import UNARY_UFUNC, BIN_UFUNC, arccosh, \ invert, mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp from mars.tensor.execution.arithmetic import OP_TO_HANDLER _sp_unary_ufunc = {arccosh, invert} _sp_bin_ufunc = { mod, fmod, bitand, bitor, bitxor, lshift, rshift, ldexp } data1 = sps.random(5, 9, density=.1) data2 = sps.random(5, 9, density=.2) rand = np.random.random() arr1 = tensor(data1, chunk_size=3) arr2 = tensor(data2, chunk_size=3) _new_unary_ufunc = UNARY_UFUNC - _sp_unary_ufunc for func in _new_unary_ufunc: res_tensor = func(arr1) res = self.executor.execute_tensor(res_tensor, concat=True) expected = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor.op)])(data1) self._nan_equal(self.toarray(res[0]), expected) _new_bin_ufunc = BIN_UFUNC - _sp_bin_ufunc for func in _new_bin_ufunc: res_tensor1 = func(arr1, arr2) res_tensor2 = func(arr1, rand) res_tensor3 = func(rand, arr1) res1 = self.executor.execute_tensor(res_tensor1, concat=True) res2 = self.executor.execute_tensor(res_tensor2, concat=True) res3 = self.executor.execute_tensor(res_tensor3, concat=True) expected1 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(data1, data2) expected2 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(data1, rand) expected3 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(rand, data1) self._nan_equal(self.toarray(res1[0]), expected1) self._nan_equal(self.toarray(res2[0]), expected2) self._nan_equal(self.toarray(res3[0]), expected3) data1 = np.random.randint(2, 10, size=(10, 10)) data2 = np.random.randint(2, 10, size=(10, 10)) rand = np.random.randint(1, 10) arr1 = tensor(data1, chunk_size=3).tosparse() arr2 = tensor(data2, chunk_size=3).tosparse() for func in _sp_unary_ufunc: res_tensor = func(arr1) res = self.executor.execute_tensor(res_tensor, concat=True) expected = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor.op)])(data1) self._nan_equal(self.toarray(res[0]), expected) for func in _sp_bin_ufunc: res_tensor1 = func(arr1, arr2) res_tensor2 = func(arr1, rand) res_tensor3 = func(rand, arr1) res1 = self.executor.execute_tensor(res_tensor1, concat=True) res2 = self.executor.execute_tensor(res_tensor2, concat=True) res3 = self.executor.execute_tensor(res_tensor3, concat=True) expected1 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(data1, data2) expected2 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(data1, rand) expected3 = self._get_sparse_func(OP_TO_HANDLER[type( res_tensor1.op)])(rand, data1) self._nan_equal(self.toarray(res1[0]), expected1) self._nan_equal(self.toarray(res2[0]), expected2) self._nan_equal(self.toarray(res3[0]), expected3) def testAddWithOutExecution(self): data1 = np.random.random((5, 9, 4)) data2 = np.random.random((9, 4)) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) add(arr1, arr2, out=arr1) res = self.executor.execute_tensor(arr1, concat=True)[0] self.assertTrue(np.array_equal(res, data1 + data2)) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) arr3 = add(arr1, arr2, out=arr1.astype('i4'), casting='unsafe') res = self.executor.execute_tensor(arr3, concat=True)[0] np.testing.assert_array_equal(res, (data1 + data2).astype('i4')) arr1 = tensor(data1.copy(), chunk_size=3) arr2 = tensor(data2.copy(), chunk_size=3) arr3 = truediv(arr1, arr2, out=arr1, where=arr2 > .5) res = self.executor.execute_tensor(arr3, concat=True)[0] self.assertTrue( np.array_equal( res, np.true_divide(data1, data2, out=data1.copy(), where=data2 > .5))) arr1 = tensor(data1.copy(), chunk_size=4) arr2 = tensor(data2.copy(), chunk_size=4) arr3 = add(arr1, arr2, where=arr1 > .5) res = self.executor.execute_tensor(arr3, concat=True)[0] expected = np.add(data1, data2, where=data1 > .5) self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5])) arr1 = tensor(data1.copy(), chunk_size=4) arr3 = add(arr1, 1, where=arr1 > .5) res = self.executor.execute_tensor(arr3, concat=True)[0] expected = np.add(data1, 1, where=data1 > .5) self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5])) def testFrexpExecution(self): data1 = np.random.random((5, 9, 4)) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = frexp(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.frexp(data1)) self.assertTrue(np.allclose(res, expected)) arr1 = tensor(data1.copy(), chunk_size=3) o1 = zeros(data1.shape, chunk_size=3) o2 = zeros(data1.shape, dtype='i8', chunk_size=3) frexp(arr1, o1, o2) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.frexp(data1)) self.assertTrue(np.allclose(res, expected)) data1 = sps.random(5, 9, density=.1) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = frexp(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.frexp(data1.toarray())) np.testing.assert_equal(res.toarray(), expected) def testModfExecution(self): data1 = np.random.random((5, 9)) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = modf(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1)) self.assertTrue(np.allclose(res, expected)) arr1 = tensor(data1.copy(), chunk_size=3) o1 = zeros(data1.shape, chunk_size=3) o2 = zeros(data1.shape, chunk_size=3) modf(arr1, o1, o2) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1)) self.assertTrue(np.allclose(res, expected)) data1 = sps.random(5, 9, density=.1) arr1 = tensor(data1.copy(), chunk_size=3) o1, o2 = modf(arr1) o = o1 + o2 res = self.executor.execute_tensor(o, concat=True)[0] expected = sum(np.modf(data1.toarray())) np.testing.assert_equal(res.toarray(), expected) def testClipExecution(self): a_data = np.arange(10) a = tensor(a_data.copy(), chunk_size=3) b = clip(a, 1, 8) res = self.executor.execute_tensor(b, concat=True)[0] expected = np.clip(a_data, 1, 8) self.assertTrue(np.array_equal(res, expected)) a = tensor(a_data.copy(), chunk_size=3) clip(a, 3, 6, out=a) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.clip(a_data, 3, 6) self.assertTrue(np.array_equal(res, expected)) with option_context() as options: options.tensor.chunk_size = 3 a = tensor(a_data.copy(), chunk_size=3) b = clip(a, [3, 4, 1, 1, 1, 4, 4, 4, 4, 4], 8) res = self.executor.execute_tensor(b, concat=True)[0] expected = np.clip(a_data, [3, 4, 1, 1, 1, 4, 4, 4, 4, 4], 8) self.assertTrue(np.array_equal(res, expected)) # test sparse clip a_data = sps.csr_matrix([[0, 2, 8], [0, 0, -1]]) a = tensor(a_data, chunk_size=3) b_data = sps.csr_matrix([[0, 3, 0], [1, 0, -2]]) c = clip(a, b_data, 4) res = self.executor.execute_tensor(c, concat=True)[0] expected = np.clip(a_data.toarray(), b_data.toarray(), 4) self.assertTrue(np.array_equal(res.toarray(), expected)) def testAroundExecution(self): data = np.random.randn(10, 20) x = tensor(data, chunk_size=3) t = x.round(2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.around(data, decimals=2) np.testing.assert_allclose(res, expected) data = sps.random(10, 20, density=.2) x = tensor(data, chunk_size=3) t = x.round(2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.around(data.toarray(), decimals=2) np.testing.assert_allclose(res.toarray(), expected) def testIsCloseExecution(self): data = np.array([1.05, 1.0, 1.01, np.nan]) data2 = np.array([1.04, 1.0, 1.03, np.nan]) x = tensor(data, chunk_size=2) y = tensor(data2, chunk_size=3) z = isclose(x, y, atol=.01) res = self.executor.execute_tensor(z, concat=True)[0] expected = np.isclose(data, data2, atol=.01) np.testing.assert_equal(res, expected) z = isclose(x, y, atol=.01, equal_nan=True) res = self.executor.execute_tensor(z, concat=True)[0] expected = np.isclose(data, data2, atol=.01, equal_nan=True) np.testing.assert_equal(res, expected) # test sparse data = sps.csr_matrix(np.array([0, 1.0, 1.01, np.nan])) data2 = sps.csr_matrix(np.array([0, 1.0, 1.03, np.nan])) x = tensor(data, chunk_size=2) y = tensor(data2, chunk_size=3) z = isclose(x, y, atol=.01) res = self.executor.execute_tensor(z, concat=True)[0] expected = np.isclose(data.toarray(), data2.toarray(), atol=.01) np.testing.assert_equal(res, expected) z = isclose(x, y, atol=.01, equal_nan=True) res = self.executor.execute_tensor(z, concat=True)[0] expected = np.isclose(data.toarray(), data2.toarray(), atol=.01, equal_nan=True) np.testing.assert_equal(res, expected) def testDtypeExecution(self): a = ones((10, 20), dtype='f4', chunk_size=5) c = truediv(a, 2, dtype='f8') res = self.executor.execute_tensor(c, concat=True)[0] self.assertEqual(res.dtype, np.float64) c = truediv(a, 0, dtype='f8') res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(np.isinf(res[0, 0])) with self.assertRaises(FloatingPointError): with np.errstate(divide='raise'): c = truediv(a, 0, dtype='f8') _ = self.executor.execute_tensor(c, concat=True)[0] # noqa: F841 def testSetGetRealExecution(self): a_data = np.array([1 + 2j, 3 + 4j, 5 + 6j]) a = tensor(a_data, chunk_size=2) res = self.executor.execute_tensor(a.real, concat=True)[0] expected = a_data.real np.testing.assert_equal(res, expected) a.real = 9 res = self.executor.execute_tensor(a, concat=True)[0] expected = a_data.copy() expected.real = 9 np.testing.assert_equal(res, expected) a.real = np.array([9, 8, 7]) res = self.executor.execute_tensor(a, concat=True)[0] expected = a_data.copy() expected.real = np.array([9, 8, 7]) np.testing.assert_equal(res, expected) # test sparse a_data = np.array([[1 + 2j, 3 + 4j, 0], [0, 0, 0]]) a = tensor(sps.csr_matrix(a_data)) res = self.executor.execute_tensor(a.real, concat=True)[0].toarray() expected = a_data.real np.testing.assert_equal(res, expected) a.real = 9 res = self.executor.execute_tensor(a, concat=True)[0].toarray() expected = a_data.copy() expected.real = 9 np.testing.assert_equal(res, expected) a.real = np.array([9, 8, 7]) res = self.executor.execute_tensor(a, concat=True)[0].toarray() expected = a_data.copy() expected.real = np.array([9, 8, 7]) np.testing.assert_equal(res, expected) def testSetGetImagExecution(self): a_data = np.array([1 + 2j, 3 + 4j, 5 + 6j]) a = tensor(a_data, chunk_size=2) res = self.executor.execute_tensor(a.imag, concat=True)[0] expected = a_data.imag np.testing.assert_equal(res, expected) a.imag = 9 res = self.executor.execute_tensor(a, concat=True)[0] expected = a_data.copy() expected.imag = 9 np.testing.assert_equal(res, expected) a.imag = np.array([9, 8, 7]) res = self.executor.execute_tensor(a, concat=True)[0] expected = a_data.copy() expected.imag = np.array([9, 8, 7]) np.testing.assert_equal(res, expected) # test sparse a_data = np.array([[1 + 2j, 3 + 4j, 0], [0, 0, 0]]) a = tensor(sps.csr_matrix(a_data)) res = self.executor.execute_tensor(a.imag, concat=True)[0].toarray() expected = a_data.imag np.testing.assert_equal(res, expected) a.imag = 9 res = self.executor.execute_tensor(a, concat=True)[0].toarray() expected = a_data.copy() expected.imag = 9 np.testing.assert_equal(res, expected) a.imag = np.array([9, 8, 7]) res = self.executor.execute_tensor(a, concat=True)[0].toarray() expected = a_data.copy() expected.imag = np.array([9, 8, 7]) np.testing.assert_equal(res, expected)
def testTensordotExecution(self): size_executor = Executor( sync_provider_type=Executor.SyncProviderType.MOCK) a_data = np.arange(60).reshape(3, 4, 5) a = tensor(a_data, chunk_size=2) b_data = np.arange(24).reshape(4, 3, 2) b = tensor(b_data, chunk_size=2) axes = ([1, 0], [0, 1]) c = tensordot(a, b, axes=axes) size_res = size_executor.execute_tensor(c, mock=True) self.assertEqual(sum(s[0] for s in size_res), c.nbytes) self.assertEqual(sum(s[1] for s in size_res), c.nbytes) res = self.executor.execute_tensor(c) expected = np.tensordot(a_data, b_data, axes=axes) self.assertTrue(np.array_equal(res[0], expected[:2, :])) self.assertTrue(np.array_equal(res[1], expected[2:4, :])) self.assertTrue(np.array_equal(res[2], expected[4:, :])) a = ones((1000, 2000), chunk_size=500) b = ones((2000, 100), chunk_size=500) c = dot(a, b) res = self.executor.execute_tensor(c) expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((10, 8), chunk_size=2) b = ones((8, 10), chunk_size=2) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertEqual(len(res), 25) for r in res: self.assertTrue(np.array_equal(r, np.tile([8], [2, 2]))) a = ones((500, 500), chunk_size=500) b = ones((500, 100), chunk_size=500) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100]))) raw_a = np.random.random((100, 200, 50)) raw_b = np.random.random((200, 10, 100)) a = tensor(raw_a, chunk_size=50) b = tensor(raw_b, chunk_size=33) c = tensordot(a, b, axes=((0, 1), (2, 0))) res = self.executor.execute_tensor(c, concat=True) expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes)) self.assertTrue(np.allclose(res[0], expected)) a = ones((1000, 2000), chunk_size=500) b = ones((100, 2000), chunk_size=500) c = inner(a, b) res = self.executor.execute_tensor(c) expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((100, 100), chunk_size=30) b = ones((100, 100), chunk_size=30) c = a.dot(b) res = self.executor.execute_tensor(c, concat=True)[0] np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testReshapeExecution(self): x = ones((1, 2, 3), chunk_size=[4, 3, 5]) y = x.reshape(3, 2) res = self.executor.execute_tensor(y)[0] self.assertEqual(y.shape, (3, 2)) np.testing.assert_equal(res, np.ones((3, 2))) data = np.random.rand(6, 4) x2 = tensor(data, chunk_size=2) y2 = x2.reshape(3, 8, order='F') res = self.executor.execute_tensor(y2, concat=True)[0] expected = data.reshape((3, 8), order='F') np.testing.assert_array_equal(res, expected) self.assertTrue(res.flags['F_CONTIGUOUS']) self.assertFalse(res.flags['C_CONTIGUOUS']) data2 = np.asfortranarray(np.random.rand(6, 4)) x3 = tensor(data2, chunk_size=2) y3 = x3.reshape(3, 8) res = self.executor.execute_tensor(y3, concat=True)[0] expected = data2.reshape((3, 8)) np.testing.assert_array_equal(res, expected) self.assertTrue(res.flags['C_CONTIGUOUS']) self.assertFalse(res.flags['F_CONTIGUOUS']) data2 = np.asfortranarray(np.random.rand(6, 4)) x3 = tensor(data2, chunk_size=2) y3 = x3.reshape(3, 8, order='F') res = self.executor.execute_tensor(y3, concat=True)[0] expected = data2.reshape((3, 8), order='F') np.testing.assert_array_equal(res, expected) self.assertTrue(res.flags['F_CONTIGUOUS']) self.assertFalse(res.flags['C_CONTIGUOUS']) def testShuffleReshapeExecution(self): a = ones((31, 27), chunk_size=10) b = a.reshape(27, 31) b.op.extra_params['_reshape_with_shuffle'] = True res = self.executor.execute_tensor(b, concat=True)[0] np.testing.assert_array_equal(res, np.ones((27, 31))) b2 = a.reshape(27, 31, order='F') b.op.extra_params['_reshape_with_shuffle'] = True res = self.executor.execute_tensor(b2)[0] self.assertTrue(res.flags['F_CONTIGUOUS']) self.assertFalse(res.flags['C_CONTIGUOUS']) data = np.random.rand(6, 4) x2 = tensor(data, chunk_size=2) y2 = x2.reshape(4, 6, order='F') y2.op.extra_params['_reshape_with_shuffle'] = True res = self.executor.execute_tensor(y2, concat=True)[0] expected = data.reshape((4, 6), order='F') np.testing.assert_array_equal(res, expected) self.assertTrue(res.flags['F_CONTIGUOUS']) self.assertFalse(res.flags['C_CONTIGUOUS']) data2 = np.asfortranarray(np.random.rand(6, 4)) x3 = tensor(data2, chunk_size=2) y3 = x3.reshape(4, 6) y3.op.extra_params['_reshape_with_shuffle'] = True res = self.executor.execute_tensor(y3, concat=True)[0] expected = data2.reshape((4, 6)) np.testing.assert_array_equal(res, expected) self.assertTrue(res.flags['C_CONTIGUOUS']) self.assertFalse(res.flags['F_CONTIGUOUS'])
class Test(TestBase): def setUp(self): super(Test, self).setUp() self.executor = Executor() @unittest.skipIf(tiledb is None, 'tiledb not installed') def testStoreTileDBExecution(self): ctx = tiledb.Ctx() tempdir = tempfile.mkdtemp() try: # store TileDB dense array expected = np.random.rand(8, 4, 3) a = tensor(expected, chunk_size=(3, 3, 2)) save = totiledb(tempdir, a, ctx=ctx) self.executor.execute_tensor(save) with tiledb.DenseArray(uri=tempdir, ctx=ctx) as arr: np.testing.assert_allclose(expected, arr.read_direct()) finally: shutil.rmtree(tempdir) tempdir = tempfile.mkdtemp() try: # store tensor with 1 chunk to TileDB dense array a = arange(12) save = totiledb(tempdir, a, ctx=ctx) self.executor.execute_tensor(save) with tiledb.DenseArray(uri=tempdir, ctx=ctx) as arr: np.testing.assert_allclose(np.arange(12), arr.read_direct()) finally: shutil.rmtree(tempdir) tempdir = tempfile.mkdtemp() try: # store 2-d TileDB sparse array expected = sps.random(8, 7, density=0.1) a = tensor(expected, chunk_size=(3, 5)) save = totiledb(tempdir, a, ctx=ctx) self.executor.execute_tensor(save) with tiledb.SparseArray(uri=tempdir, ctx=ctx) as arr: data = arr[:, :] coords = data['coords'] value = data[arr.attr(0).name] ij = tuple(coords[arr.domain.dim(k).name] for k in range(arr.ndim)) result = sps.coo_matrix((value, ij), shape=arr.shape) np.testing.assert_allclose(expected.toarray(), result.toarray()) finally: shutil.rmtree(tempdir) tempdir = tempfile.mkdtemp() try: # store TileDB dense array expected = np.asfortranarray(np.random.rand(8, 4, 3)) a = tensor(expected, chunk_size=(3, 3, 2)) save = totiledb(tempdir, a, ctx=ctx) self.executor.execute_tensor(save) with tiledb.DenseArray(uri=tempdir, ctx=ctx) as arr: np.testing.assert_allclose(expected, arr.read_direct()) self.assertEqual(arr.schema.cell_order, 'col-major') finally: shutil.rmtree(tempdir)
def setup(self): self.executor = Executor()
class Test(TestBase): def setUp(self): super(Test, self).setUp() self.executor = Executor() def testCreateSparseExecution(self): mat = sps.csr_matrix([[0, 0, 2], [2, 0, 0]]) t = tensor(mat, dtype='f8', chunk_size=2) res = self.executor.execute_tensor(t) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.float64) np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray()) t2 = ones_like(t, dtype='f4') res = self.executor.execute_tensor(t2) expected = sps.csr_matrix([[0, 0, 1], [1, 0, 0]]) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.float32) np.testing.assert_array_equal(res[0].toarray(), expected[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), expected[..., 2:].toarray()) t3 = tensor(np.array([[0, 0, 2], [2, 0, 0]]), chunk_size=2).tosparse() res = self.executor.execute_tensor(t3) self.assertIsInstance(res[0], SparseNDArray) self.assertEqual(res[0].dtype, np.int_) np.testing.assert_array_equal(res[0].toarray(), mat[..., :2].toarray()) np.testing.assert_array_equal(res[1].toarray(), mat[..., 2:].toarray()) def testZerosExecution(self): t = zeros((20, 30), dtype='i8', chunk_size=5) res = self.executor.execute_tensor(t, concat=True) self.assertTrue(np.array_equal(res[0], np.zeros((20, 30), dtype='i8'))) self.assertEqual(res[0].dtype, np.int64) t2 = zeros_like(t) res = self.executor.execute_tensor(t2, concat=True) self.assertTrue(np.array_equal(res[0], np.zeros((20, 30), dtype='i8'))) self.assertEqual(res[0].dtype, np.int64) t = zeros((20, 30), dtype='i4', chunk_size=5, sparse=True) res = self.executor.execute_tensor(t, concat=True) self.assertEqual(res[0].nnz, 0) def testEmptyExecution(self): t = empty((20, 30), dtype='i8', chunk_size=5) res = self.executor.execute_tensor(t, concat=True) self.assertEqual(res[0].shape, (20, 30)) self.assertEqual(res[0].dtype, np.int64) t = empty((20, 30), chunk_size=5) res = self.executor.execute_tensor(t, concat=True) self.assertEqual(res[0].shape, (20, 30)) self.assertEqual(res[0].dtype, np.float64) t2 = empty_like(t) res = self.executor.execute_tensor(t2, concat=True) self.assertEqual(res[0].shape, (20, 30)) self.assertEqual(res[0].dtype, np.float64) def testFullExecution(self): t = full((2, 2), 1, dtype='f4', chunk_size=1) res = self.executor.execute_tensor(t, concat=True) self.assertTrue(np.array_equal(res[0], np.full((2, 2), 1, dtype='f4'))) t = full((2, 2), [1, 2], dtype='f8', chunk_size=1) res = self.executor.execute_tensor(t, concat=True) self.assertTrue(np.array_equal(res[0], np.full((2, 2), [1, 2], dtype='f8'))) def testArangeExecution(self): t = arange(1, 20, 3, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] self.assertTrue(np.array_equal(res, np.arange(1, 20, 3))) t = arange(1, 20, .3, chunk_size=4) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.arange(1, 20, .3) self.assertTrue(np.allclose(res, expected)) t = arange(1.0, 1.8, .3, chunk_size=4) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.arange(1.0, 1.8, .3) self.assertTrue(np.allclose(res, expected)) t = arange('1066-10-13', '1066-10-31', dtype=np.datetime64, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.arange('1066-10-13', '1066-10-31', dtype=np.datetime64) self.assertTrue(np.array_equal(res, expected)) def testDiagExecution(self): # 2-d 6 * 6 a = arange(36, chunk_size=2).reshape(6, 6) d = diag(a) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(6, 6)) np.testing.assert_equal(res, expected) d = diag(a, k=1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(6, 6), k=1) np.testing.assert_equal(res, expected) d = diag(a, k=3) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(6, 6), k=3) np.testing.assert_equal(res, expected) d = diag(a, k=-2) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(6, 6), k=-2) np.testing.assert_equal(res, expected) d = diag(a, k=-5) res = self.executor.execute_tensor(d)[0] expected = np.diag(np.arange(36).reshape(6, 6), k=-5) np.testing.assert_equal(res, expected) # 2-d 4 * 9 a = arange(36, chunk_size=2).reshape(4, 9) d = diag(a) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(4, 9)) np.testing.assert_equal(res, expected) d = diag(a, k=1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(4, 9), k=1) np.testing.assert_equal(res, expected) d = diag(a, k=3) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(4, 9), k=3) np.testing.assert_equal(res, expected) d = diag(a, k=-2) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(36).reshape(4, 9), k=-2) np.testing.assert_equal(res, expected) d = diag(a, k=-3) res = self.executor.execute_tensor(d)[0] expected = np.diag(np.arange(36).reshape(4, 9), k=-3) np.testing.assert_equal(res, expected) # 1-d a = arange(5, chunk_size=2) d = diag(a) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5)) np.testing.assert_equal(res, expected) d = diag(a, k=1) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=1) np.testing.assert_equal(res, expected) d = diag(a, k=3) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=3) np.testing.assert_equal(res, expected) d = diag(a, k=-2) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=-2) np.testing.assert_equal(res, expected) d = diag(a, k=-3) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=-3) np.testing.assert_equal(res, expected) d = diag(a, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5)) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) d = diag(a, k=1, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) d = diag(a, k=2, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) d = diag(a, k=-2, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=-2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) d = diag(a, k=-3, sparse=True) res = self.executor.execute_tensor(d, concat=True)[0] expected = np.diag(np.arange(5), k=-3) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) def testDiagflatExecution(self): a = diagflat([[1, 2], [3, 4]], chunk_size=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([[1, 2], [3, 4]]) np.testing.assert_equal(res, expected) d = tensor([[1, 2], [3, 4]], chunk_size=1) a = diagflat(d) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([[1, 2], [3, 4]]) np.testing.assert_equal(res, expected) a = diagflat([1, 2], 1, chunk_size=1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([1, 2], 1) np.testing.assert_equal(res, expected) d = tensor([[1, 2]], chunk_size=1) a = diagflat(d, 1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.diagflat([1, 2], 1) np.testing.assert_equal(res, expected) def testEyeExecution(self): t = eye(5, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5) np.testing.assert_equal(res, expected) t = eye(5, k=1, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=1) np.testing.assert_equal(res, expected) t = eye(5, k=2, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=2) np.testing.assert_equal(res, expected) t = eye(5, k=-1, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=-1) np.testing.assert_equal(res, expected) t = eye(5, k=-3, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=-3) np.testing.assert_equal(res, expected) t = eye(5, M=3, k=1, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=3, k=1) np.testing.assert_equal(res, expected) t = eye(5, M=3, k=-3, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=3, k=-3) np.testing.assert_equal(res, expected) t = eye(5, M=7, k=1, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=7, k=1) np.testing.assert_equal(res, expected) t = eye(5, M=8, k=-3, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=8, k=-3) np.testing.assert_equal(res, expected) t = eye(2, dtype=int) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.dtype, np.int_) # test sparse t = eye(5, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, k=1, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, k=2, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, k=-1, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=-1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, k=-3, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, k=-3) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, M=3, k=1, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=3, k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, M=3, k=-3, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=3, k=-3) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, M=7, k=1, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=7, k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) t = eye(5, M=8, k=-3, sparse=True, chunk_size=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.eye(5, M=8, k=-3) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res.toarray(), expected) def testLinspaceExecution(self): a = linspace(2.0, 9.0, num=11, chunk_size=3) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.linspace(2.0, 9.0, num=11) np.testing.assert_allclose(res, expected) a = linspace(2.0, 9.0, num=11, endpoint=False, chunk_size=3) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.linspace(2.0, 9.0, num=11, endpoint=False) np.testing.assert_allclose(res, expected) a = linspace(2.0, 9.0, num=11, chunk_size=3, dtype=int) res = self.executor.execute_tensor(a, concat=True)[0] self.assertEqual(res.dtype, np.int_) def testMeshgridExecution(self): a = arange(5, chunk_size=2) b = arange(6, 12, chunk_size=3) c = arange(12, 19, chunk_size=4) A, B, C = meshgrid(a, b, c) A_res = self.executor.execute_tensor(A, concat=True)[0] A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[0] np.testing.assert_equal(A_res, A_expected) B_res = self.executor.execute_tensor(B, concat=True)[0] B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[1] np.testing.assert_equal(B_res, B_expected) C_res = self.executor.execute_tensor(C, concat=True)[0] C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19))[2] np.testing.assert_equal(C_res, C_expected) A, B, C = meshgrid(a, b, c, indexing='ij') A_res = self.executor.execute_tensor(A, concat=True)[0] A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[0] np.testing.assert_equal(A_res, A_expected) B_res = self.executor.execute_tensor(B, concat=True)[0] B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[1] np.testing.assert_equal(B_res, B_expected) C_res = self.executor.execute_tensor(C, concat=True)[0] C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij')[2] np.testing.assert_equal(C_res, C_expected) A, B, C = meshgrid(a, b, c, sparse=True) A_res = self.executor.execute_tensor(A, concat=True)[0] A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[0] np.testing.assert_equal(A_res, A_expected) B_res = self.executor.execute_tensor(B, concat=True)[0] B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[1] np.testing.assert_equal(B_res, B_expected) C_res = self.executor.execute_tensor(C, concat=True)[0] C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), sparse=True)[2] np.testing.assert_equal(C_res, C_expected) A, B, C = meshgrid(a, b, c, indexing='ij', sparse=True) A_res = self.executor.execute_tensor(A, concat=True)[0] A_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij', sparse=True)[0] np.testing.assert_equal(A_res, A_expected) B_res = self.executor.execute_tensor(B, concat=True)[0] B_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij', sparse=True)[1] np.testing.assert_equal(B_res, B_expected) C_res = self.executor.execute_tensor(C, concat=True)[0] C_expected = np.meshgrid(np.arange(5), np.arange(6, 12), np.arange(12, 19), indexing='ij', sparse=True)[2] np.testing.assert_equal(C_res, C_expected) def testIndicesExecution(self): grid = indices((2, 3), chunk_size=1) res = self.executor.execute_tensor(grid, concat=True)[0] expected = np.indices((2, 3)) np.testing.assert_equal(res, expected) res = self.executor.execute_tensor(grid[0], concat=True)[0] np.testing.assert_equal(res, expected[0]) res = self.executor.execute_tensor(grid[1], concat=True)[0] np.testing.assert_equal(res, expected[1]) def testTriuExecution(self): a = arange(24, chunk_size=2).reshape(2, 3, 4) t = triu(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4)) np.testing.assert_equal(res, expected) t = triu(a, k=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4), k=1) np.testing.assert_equal(res, expected) t = triu(a, k=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4), k=2) np.testing.assert_equal(res, expected) t = triu(a, k=-1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4), k=-1) np.testing.assert_equal(res, expected) t = triu(a, k=-2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(24).reshape(2, 3, 4), k=-2) np.testing.assert_equal(res, expected) # test sparse a = arange(12, chunk_size=2).reshape(3, 4).tosparse() t = triu(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4)) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = triu(a, k=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4), k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = triu(a, k=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4), k=2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = triu(a, k=-1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4), k=-1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = triu(a, k=-2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.triu(np.arange(12).reshape(3, 4), k=-2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) def testTrilExecution(self): a = arange(24, chunk_size=2).reshape(2, 3, 4) t = tril(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4)) np.testing.assert_equal(res, expected) t = tril(a, k=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4), k=1) np.testing.assert_equal(res, expected) t = tril(a, k=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4), k=2) np.testing.assert_equal(res, expected) t = tril(a, k=-1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4), k=-1) np.testing.assert_equal(res, expected) t = tril(a, k=-2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(24).reshape(2, 3, 4), k=-2) np.testing.assert_equal(res, expected) a = arange(12, chunk_size=2).reshape(3, 4).tosparse() t = tril(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4)) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = tril(a, k=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4), k=1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = tril(a, k=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4), k=2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = tril(a, k=-1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4), k=-1) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) t = tril(a, k=-2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tril(np.arange(12).reshape(3, 4), k=-2) self.assertIsInstance(res, SparseNDArray) np.testing.assert_equal(res, expected) def testIndexTrickExecution(self): mgrid = nd_grid() t = mgrid[0:5, 0:5] res = self.executor.execute_tensor(t, concat=True)[0] expected = np.lib.index_tricks.nd_grid()[0:5, 0:5] np.testing.assert_equal(res, expected) t = mgrid[-1:1:5j] res = self.executor.execute_tensor(t, concat=True)[0] expected = np.lib.index_tricks.nd_grid()[-1:1:5j] np.testing.assert_equal(res, expected) ogrid = nd_grid(sparse=True) t = ogrid[0:5, 0:5] res = [self.executor.execute_tensor(o, concat=True)[0] for o in t] expected = np.lib.index_tricks.nd_grid(sparse=True)[0:5, 0:5] [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] @unittest.skipIf(tiledb is None, 'tiledb not installed') def testReadTileDBExecution(self): ctx = tiledb.Ctx() tempdir = tempfile.mkdtemp() try: # create TileDB dense array dom = tiledb.Domain(ctx, tiledb.Dim(ctx, domain=(1, 100), tile=30, dtype=np.int32), tiledb.Dim(ctx, domain=(0, 90), tile=22, dtype=np.int32), tiledb.Dim(ctx, domain=(0, 9), tile=8, dtype=np.int32), ) schema = tiledb.ArraySchema(ctx, domain=dom, sparse=False, attrs=[tiledb.Attr(ctx, dtype=np.float64)]) tiledb.DenseArray.create(tempdir, schema) expected = np.random.rand(100, 91, 10) with tiledb.DenseArray(ctx, tempdir, mode='w') as arr: arr.write_direct(expected) a = fromtiledb(tempdir, ctx=ctx) result = self.executor.execute_tensor(a, concat=True)[0] np.testing.assert_allclose(expected, result) finally: shutil.rmtree(tempdir) tempdir = tempfile.mkdtemp() try: # create 2-d TileDB sparse array dom = tiledb.Domain(ctx, tiledb.Dim(ctx, domain=(0, 99), tile=30, dtype=np.int32), tiledb.Dim(ctx, domain=(2, 11), tile=8, dtype=np.int32), ) schema = tiledb.ArraySchema(ctx, domain=dom, sparse=True, attrs=[tiledb.Attr(ctx, dtype=np.float64)]) tiledb.SparseArray.create(tempdir, schema) expected = sps.rand(100, 10, density=0.01) with tiledb.SparseArray(ctx, tempdir, mode='w') as arr: I, J = expected.row, expected.col + 2 arr[I, J] = {arr.attr(0).name: expected.data} a = fromtiledb(tempdir, ctx=ctx) result = self.executor.execute_tensor(a, concat=True)[0] np.testing.assert_allclose(expected.toarray(), result.toarray()) finally: shutil.rmtree(tempdir) tempdir = tempfile.mkdtemp() try: # create 1-d TileDB sparse array dom = tiledb.Domain(ctx, tiledb.Dim(ctx, domain=(1, 100), tile=30, dtype=np.int32), ) schema = tiledb.ArraySchema(ctx, domain=dom, sparse=True, attrs=[tiledb.Attr(ctx, dtype=np.float64)]) tiledb.SparseArray.create(tempdir, schema) expected = sps.rand(1, 100, density=0.05) with tiledb.SparseArray(ctx, tempdir, mode='w') as arr: I= expected.col + 1 arr[I] = expected.data a = fromtiledb(tempdir, ctx=ctx) result = self.executor.execute_tensor(a, concat=True)[0] np.testing.assert_allclose(expected.toarray()[0], result.toarray()) finally: shutil.rmtree(tempdir)
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testRechunkExecution(self): raw = np.random.random((11, 8)) arr = tensor(raw, chunk_size=3) arr2 = arr.rechunk(4) res = self.executor.execute_tensor(arr2) self.assertTrue(np.array_equal(res[0], raw[:4, :4])) self.assertTrue(np.array_equal(res[1], raw[:4, 4:])) self.assertTrue(np.array_equal(res[2], raw[4:8, :4])) self.assertTrue(np.array_equal(res[3], raw[4:8, 4:])) self.assertTrue(np.array_equal(res[4], raw[8:, :4])) self.assertTrue(np.array_equal(res[5], raw[8:, 4:])) def testCopytoExecution(self): a = ones((2, 3), chunk_size=1) b = tensor([3, -1, 3], chunk_size=2) copyto(a, b, where=b > 1) res = self.executor.execute_tensor(a, concat=True)[0] expected = np.array([[3, 1, 3], [3, 1, 3]]) np.testing.assert_equal(res, expected) def testAstypeExecution(self): raw = np.random.random((10, 5)) arr = tensor(raw, chunk_size=3) arr2 = arr.astype('i8') res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw.astype('i8'))) raw = sps.random(10, 5, density=.2) arr = tensor(raw, chunk_size=3) arr2 = arr.astype('i8') res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue( np.array_equal(res[0].toarray(), raw.astype('i8').toarray())) def testTransposeExecution(self): raw = np.random.random((11, 8, 5)) arr = tensor(raw, chunk_size=3) arr2 = transpose(arr) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw.T)) arr3 = transpose(arr, axes=(-2, -1, -3)) res = self.executor.execute_tensor(arr3, concat=True) self.assertTrue(np.array_equal(res[0], raw.transpose(1, 2, 0))) raw = sps.random(11, 8) arr = tensor(raw, chunk_size=3) arr2 = transpose(arr) self.assertTrue(arr2.issparse()) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0].toarray(), raw.T.toarray())) def testSwapaxesExecution(self): raw = np.random.random((11, 8, 5)) arr = tensor(raw, chunk_size=3) arr2 = arr.swapaxes(2, 0) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue(np.array_equal(res[0], raw.swapaxes(2, 0))) raw = sps.random(11, 8, density=.2) arr = tensor(raw, chunk_size=3) arr2 = arr.swapaxes(1, 0) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue( np.array_equal(res[0].toarray(), raw.toarray().swapaxes(1, 0))) def testMoveaxisExecution(self): x = zeros((3, 4, 5), chunk_size=2) t = moveaxis(x, 0, -1) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (4, 5, 3)) t = moveaxis(x, -1, 0) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 3, 4)) t = moveaxis(x, [0, 1], [-1, -2]) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 4, 3)) t = moveaxis(x, [0, 1, 2], [-1, -2, -3]) res = self.executor.execute_tensor(t, concat=True)[0] self.assertEqual(res.shape, (5, 4, 3)) def testBroadcastToExecution(self): raw = np.random.random((10, 5, 1)) arr = tensor(raw, chunk_size=2) arr2 = broadcast_to(arr, (5, 10, 5, 6)) res = self.executor.execute_tensor(arr2, concat=True) self.assertTrue( np.array_equal(res[0], np.broadcast_to(raw, (5, 10, 5, 6)))) def testBroadcastArraysExecutions(self): x_data = [[1, 2, 3]] x = tensor(x_data, chunk_size=1) y_data = [[1], [2], [3]] y = tensor(y_data, chunk_size=2) a = broadcast_arrays(x, y) res = [self.executor.execute_tensor(arr, concat=True)[0] for arr in a] expected = np.broadcast_arrays(x_data, y_data) for r, e in zip(res, expected): np.testing.assert_equal(r, e) def testWhereExecution(self): raw_cond = np.random.randint(0, 2, size=(4, 4), dtype='?') raw_x = np.random.rand(4, 1) raw_y = np.random.rand(4, 4) cond, x, y = tensor(raw_cond, chunk_size=2), tensor( raw_x, chunk_size=2), tensor(raw_y, chunk_size=2) arr = where(cond, x, y) res = self.executor.execute_tensor(arr, concat=True) self.assertTrue( np.array_equal(res[0], np.where(raw_cond, raw_x, raw_y))) raw_cond = sps.csr_matrix( np.random.randint(0, 2, size=(4, 4), dtype='?')) raw_x = sps.random(4, 1, density=.1) raw_y = sps.random(4, 4, density=.1) cond, x, y = tensor(raw_cond, chunk_size=2), tensor( raw_x, chunk_size=2), tensor(raw_y, chunk_size=2) arr = where(cond, x, y) res = self.executor.execute_tensor(arr, concat=True)[0] self.assertTrue( np.array_equal( res.toarray(), np.where(raw_cond.toarray(), raw_x.toarray(), raw_y.toarray()))) def testReshapeExecution(self): raw_data = np.random.rand(10, 20, 30) x = tensor(raw_data, chunk_size=6) y = x.reshape(-1, 30) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(-1, 30))) y2 = x.reshape(10, -1) res = self.executor.execute_tensor(y2, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(10, -1))) y3 = x.reshape(-1) res = self.executor.execute_tensor(y3, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(-1))) y4 = x.ravel() res = self.executor.execute_tensor(y4, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.ravel())) raw_data = np.random.rand(30, 100, 20) x = tensor(raw_data, chunk_size=6) y = x.reshape(-1, 20, 5, 5, 4) res = self.executor.execute_tensor(y, concat=True) self.assertTrue( np.array_equal(res[0], raw_data.reshape(-1, 20, 5, 5, 4))) y2 = x.reshape(3000, 10, 2) res = self.executor.execute_tensor(y2, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(3000, 10, 2))) y3 = x.reshape(60, 25, 40) res = self.executor.execute_tensor(y3, concat=True) self.assertTrue(np.array_equal(res[0], raw_data.reshape(60, 25, 40))) y4 = x.reshape(60, 25, 40) y4.op.extra_params['_reshape_with_shuffle'] = True size_res = self.executor.execute_tensor(y4, mock=True) res = self.executor.execute_tensor(y4, concat=True) self.assertEqual(res[0].nbytes, sum(v[0] for v in size_res)) self.assertTrue(np.array_equal(res[0], raw_data.reshape(60, 25, 40))) def testExpandDimsExecution(self): raw_data = np.random.rand(10, 20, 30) x = tensor(raw_data, chunk_size=6) y = expand_dims(x, 1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 1))) y = expand_dims(x, 0) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 0))) y = expand_dims(x, 3) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, 3))) y = expand_dims(x, -1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -1))) y = expand_dims(x, -4) res = self.executor.execute_tensor(y, concat=True) self.assertTrue(np.array_equal(res[0], np.expand_dims(raw_data, -4))) with self.assertRaises(np.AxisError): expand_dims(x, -5) with self.assertRaises(np.AxisError): expand_dims(x, 4) def testRollAxisExecution(self): x = ones((3, 4, 5, 6), chunk_size=1) y = rollaxis(x, 3, 1) res = self.executor.execute_tensor(y, concat=True) self.assertTrue( np.array_equal(res[0], np.rollaxis(np.ones((3, 4, 5, 6)), 3, 1))) def testAtleast1dExecution(self): x = 1 y = ones(3, chunk_size=2) z = ones((3, 4), chunk_size=2) t = atleast_1d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.array([1]))) self.assertTrue(np.array_equal(res[1], np.ones(3))) self.assertTrue(np.array_equal(res[2], np.ones((3, 4)))) def testAtleast2dExecution(self): x = 1 y = ones(3, chunk_size=2) z = ones((3, 4), chunk_size=2) t = atleast_2d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.array([[1]]))) self.assertTrue(np.array_equal(res[1], np.atleast_2d(np.ones(3)))) self.assertTrue(np.array_equal(res[2], np.ones((3, 4)))) def testAtleast3dExecution(self): x = 1 y = ones(3, chunk_size=2) z = ones((3, 4), chunk_size=2) t = atleast_3d(x, y, z) res = [self.executor.execute_tensor(i, concat=True)[0] for i in t] self.assertTrue(np.array_equal(res[0], np.atleast_3d(x))) self.assertTrue(np.array_equal(res[1], np.atleast_3d(np.ones(3)))) self.assertTrue(np.array_equal(res[2], np.atleast_3d(np.ones((3, 4))))) def testArgwhereExecution(self): x = arange(6, chunk_size=2).reshape(2, 3) t = argwhere(x > 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.argwhere(np.arange(6).reshape(2, 3) > 1) self.assertTrue(np.array_equal(res, expected)) def testArraySplitExecution(self): x = arange(48, chunk_size=3).reshape(2, 3, 8) ss = array_split(x, 3, axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.array_split(np.arange(48).reshape(2, 3, 8), 3, axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] ss = array_split(x, [3, 5, 6, 10], axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.array_split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10], axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] def testSplitExecution(self): x = arange(48, chunk_size=3).reshape(2, 3, 8) ss = split(x, 4, axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(np.arange(48).reshape(2, 3, 8), 4, axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] ss = split(x, [3, 5, 6, 10], axis=2) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(np.arange(48).reshape(2, 3, 8), [3, 5, 6, 10], axis=2) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # hsplit x = arange(120, chunk_size=3).reshape(2, 12, 5) ss = hsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.hsplit(np.arange(120).reshape(2, 12, 5), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # vsplit x = arange(48, chunk_size=3).reshape(8, 3, 2) ss = vsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.vsplit(np.arange(48).reshape(8, 3, 2), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] # dsplit x = arange(48, chunk_size=3).reshape(2, 3, 8) ss = dsplit(x, 4) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.dsplit(np.arange(48).reshape(2, 3, 8), 4) self.assertEqual(len(res), len(expected)) [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] x_data = sps.random(12, 8, density=.1) x = tensor(x_data, chunk_size=3) ss = split(x, 4, axis=0) res = [self.executor.execute_tensor(i, concat=True)[0] for i in ss] expected = np.split(x_data.toarray(), 4, axis=0) self.assertEqual(len(res), len(expected)) [ np.testing.assert_equal(r.toarray(), e) for r, e in zip(res, expected) ] def testRollExecution(self): x = arange(10, chunk_size=2) t = roll(x, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10), 2) np.testing.assert_equal(res, expected) x2 = x.reshape(2, 5) t = roll(x2, 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1) np.testing.assert_equal(res, expected) t = roll(x2, 1, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=0) np.testing.assert_equal(res, expected) t = roll(x2, 1, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.roll(np.arange(10).reshape(2, 5), 1, axis=1) np.testing.assert_equal(res, expected) def testSqueezeExecution(self): data = np.array([[[0], [1], [2]]]) x = tensor(data, chunk_size=1) t = squeeze(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.squeeze(data) np.testing.assert_equal(res, expected) t = squeeze(x, axis=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.squeeze(data, axis=2) np.testing.assert_equal(res, expected) def testPtpExecution(self): x = arange(4, chunk_size=1).reshape(2, 2) t = ptp(x, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ptp(np.arange(4).reshape(2, 2), axis=0) np.testing.assert_equal(res, expected) t = ptp(x, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ptp(np.arange(4).reshape(2, 2), axis=1) np.testing.assert_equal(res, expected) t = ptp(x) res = self.executor.execute_tensor(t)[0] expected = np.ptp(np.arange(4).reshape(2, 2)) np.testing.assert_equal(res, expected) def testDiffExecution(self): data = np.array([1, 2, 4, 7, 0]) x = tensor(data, chunk_size=2) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data) np.testing.assert_equal(res, expected) t = diff(x, n=2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data, n=2) np.testing.assert_equal(res, expected) data = np.array([[1, 3, 6, 10], [0, 5, 6, 8]]) x = tensor(data, chunk_size=2) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data) np.testing.assert_equal(res, expected) t = diff(x, axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff(data, axis=0) np.testing.assert_equal(res, expected) x = mt.arange('1066-10-13', '1066-10-16', dtype=mt.datetime64) t = diff(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.diff( np.arange('1066-10-13', '1066-10-16', dtype=np.datetime64)) np.testing.assert_equal(res, expected) def testEdiff1d(self): data = np.array([1, 2, 4, 7, 0]) x = tensor(data, chunk_size=2) t = ediff1d(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data) np.testing.assert_equal(res, expected) to_begin = tensor(-99, chunk_size=2) to_end = tensor([88, 99], chunk_size=2) t = ediff1d(x, to_begin=to_begin, to_end=to_end) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data, to_begin=-99, to_end=np.array([88, 99])) np.testing.assert_equal(res, expected) data = [[1, 2, 4], [1, 6, 24]] t = ediff1d(tensor(data, chunk_size=2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.ediff1d(data) np.testing.assert_equal(res, expected) def testDigitizeExecution(self): data = np.array([0.2, 6.4, 3.0, 1.6]) x = tensor(data, chunk_size=2) bins = np.array([0.0, 1.0, 2.5, 4.0, 10.0]) inds = digitize(x, bins) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins) np.testing.assert_equal(res, expected) b = tensor(bins, chunk_size=2) inds = digitize(x, b) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins) np.testing.assert_equal(res, expected) data = np.array([1.2, 10.0, 12.4, 15.5, 20.]) x = tensor(data, chunk_size=2) bins = np.array([0, 5, 10, 15, 20]) inds = digitize(x, bins, right=True) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins, right=True) np.testing.assert_equal(res, expected) inds = digitize(x, bins, right=False) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data, bins, right=False) np.testing.assert_equal(res, expected) data = sps.random(10, 1, density=.1) * 12 x = tensor(data, chunk_size=2) bins = np.array([1.0, 2.0, 2.5, 4.0, 10.0]) inds = digitize(x, bins) res = self.executor.execute_tensor(inds, concat=True)[0] expected = np.digitize(data.toarray(), bins, right=False) np.testing.assert_equal(res.toarray(), expected) def testAverageExecution(self): data = arange(1, 5, chunk_size=1) t = average(data) res = self.executor.execute_tensor(t)[0] expected = np.average(np.arange(1, 5)) self.assertEqual(res, expected) t = average(arange(1, 11, chunk_size=2), weights=arange(10, 0, -1, chunk_size=2)) res = self.executor.execute_tensor(t)[0] expected = np.average(range(1, 11), weights=range(10, 0, -1)) self.assertEqual(res, expected) data = arange(6, chunk_size=2).reshape((3, 2)) t = average(data, axis=1, weights=tensor([1. / 4, 3. / 4], chunk_size=2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.average(np.arange(6).reshape(3, 2), axis=1, weights=(1. / 4, 3. / 4)) np.testing.assert_equal(res, expected) with self.assertRaises(TypeError): average(data, weights=tensor([1. / 4, 3. / 4], chunk_size=2)) def testCovExecution(self): data = np.array([[0, 2], [1, 1], [2, 0]]).T x = tensor(data, chunk_size=1) t = cov(x) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.cov(data) np.testing.assert_equal(res, expected) data_x = [-2.1, -1, 4.3] data_y = [3, 1.1, 0.12] x = tensor(data_x, chunk_size=1) y = tensor(data_y, chunk_size=1) X = stack((x, y), axis=0) t = cov(x, y) r = tall(t == cov(X)) self.assertTrue(self.executor.execute_tensor(r)[0]) def testCorrcoefExecution(self): data_x = [-2.1, -1, 4.3] data_y = [3, 1.1, 0.12] x = tensor(data_x, chunk_size=1) y = tensor(data_y, chunk_size=1) t = corrcoef(x, y) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.corrcoef(data_x, data_y) np.testing.assert_equal(res, expected) def testFlipExecution(self): a = arange(8, chunk_size=2).reshape((2, 2, 2)) t = flip(a, 0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flip(np.arange(8).reshape(2, 2, 2), 0) np.testing.assert_equal(res, expected) t = flip(a, 1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flip(np.arange(8).reshape(2, 2, 2), 1) np.testing.assert_equal(res, expected) t = flipud(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.flipud(np.arange(8).reshape(2, 2, 2)) np.testing.assert_equal(res, expected) t = fliplr(a) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.fliplr(np.arange(8).reshape(2, 2, 2)) np.testing.assert_equal(res, expected) def testRepeatExecution(self): a = repeat(3, 4) res = self.executor.execute_tensor(a)[0] expected = np.repeat(3, 4) np.testing.assert_equal(res, expected) x_data = np.random.randn(20, 30) x = tensor(x_data, chunk_size=(3, 4)) t = repeat(x, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, 2) np.testing.assert_equal(res, expected) t = repeat(x, 3, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, 3, axis=1) np.testing.assert_equal(res, expected) t = repeat(x, np.arange(20), axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, np.arange(20), axis=0) np.testing.assert_equal(res, expected) t = repeat(x, arange(20, chunk_size=5), axis=0) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data, np.arange(20), axis=0) np.testing.assert_equal(res, expected) x_data = sps.random(20, 30, density=.1) x = tensor(x_data, chunk_size=(3, 4)) t = repeat(x, 2, axis=1) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.repeat(x_data.toarray(), 2, axis=1) np.testing.assert_equal(res.toarray(), expected) def testTileExecution(self): a_data = np.array([0, 1, 2]) a = tensor(a_data, chunk_size=2) t = tile(a, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, 2) np.testing.assert_equal(res, expected) t = tile(a, (2, 2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, (2, 2)) np.testing.assert_equal(res, expected) t = tile(a, (2, 1, 2)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(a_data, (2, 1, 2)) np.testing.assert_equal(res, expected) b_data = np.array([[1, 2], [3, 4]]) b = tensor(b_data, chunk_size=1) t = tile(b, 2) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(b_data, 2) np.testing.assert_equal(res, expected) t = tile(b, (2, 1)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(b_data, (2, 1)) np.testing.assert_equal(res, expected) c_data = np.array([1, 2, 3, 4]) c = tensor(c_data, chunk_size=3) t = tile(c, (4, 1)) res = self.executor.execute_tensor(t, concat=True)[0] expected = np.tile(c_data, (4, 1)) np.testing.assert_equal(res, expected) def testIsInExecution(self): element = 2 * arange(4, chunk_size=1).reshape((2, 2)) test_elements = [1, 2, 4, 8] mask = isin(element, test_elements) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements) np.testing.assert_equal(res, expected) res = self.executor.execute_tensor(element[mask], concat=True)[0] expected = np.array([2, 4]) np.testing.assert_equal(res, expected) mask = isin(element, test_elements, invert=True) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_elements, invert=True) np.testing.assert_equal(res, expected) res = self.executor.execute_tensor(element[mask], concat=True)[0] expected = np.array([0, 6]) np.testing.assert_equal(res, expected) test_set = {1, 2, 4, 8} mask = isin(element, test_set) res = self.executor.execute_tensor(mask, concat=True)[0] expected = np.isin(2 * np.arange(4).reshape((2, 2)), test_set) np.testing.assert_equal(res, expected)
def setUp(self): self.executor = Executor('cupy')
def setUp(self): self.executor = Executor('numpy')
class Test(unittest.TestCase): def setUp(self): self.executor = Executor('numpy') def testFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = fft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw) np.testing.assert_allclose(res, expected) r = fft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = fft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw, n=11) np.testing.assert_allclose(res, expected) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 4)) r = fft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw) np.testing.assert_allclose(res, expected) r = fft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = fft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft(raw, n=11) np.testing.assert_allclose(res, expected) def testIFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = ifft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw) np.testing.assert_allclose(res, expected) r = ifft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = ifft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw, n=11) np.testing.assert_allclose(res, expected) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 5)) r = ifft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw) np.testing.assert_allclose(res, expected) r = ifft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = ifft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft(raw, n=11) np.testing.assert_allclose(res, expected) def testFFT2Execution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 20, 30)) r = fft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw) np.testing.assert_allclose(res, expected) r = fft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = fft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, s=(11, 12)) np.testing.assert_allclose(res, expected) r = fft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, s=(11, 12), axes=(-1, -2)) np.testing.assert_allclose(res, expected) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 5, 6)) r = fft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw) np.testing.assert_allclose(res, expected) r = fft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = fft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, s=(11, 12)) np.testing.assert_allclose(res, expected) r = fft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fft2(raw, s=(11, 12), axes=(-1, -2)) np.testing.assert_allclose(res, expected) def testIFFT2Execution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 20, 30)) r = ifft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw) np.testing.assert_allclose(res, expected) r = ifft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = ifft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, s=(11, 12)) np.testing.assert_allclose(res, expected) r = ifft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, s=(11, 12), axes=(-1, -2)) np.testing.assert_allclose(res, expected) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 3, 5)) r = ifft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw) np.testing.assert_allclose(res, expected) r = ifft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = ifft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, s=(11, 12)) np.testing.assert_allclose(res, expected) r = ifft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifft2(raw, s=(11, 12), axes=(-1, -2)) np.testing.assert_allclose(res, expected) def testFFTNExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(10, 20, 30)) r = fftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw) np.testing.assert_allclose(res, expected) r = fftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = fftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, s=(11, 12, 5)) np.testing.assert_allclose(res, expected) r = fftn(t, s=(11, 12, 5), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, s=(11, 12, 5), axes=(-1, -2, -3)) np.testing.assert_allclose(res, expected) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(3, 3, 4)) r = fftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw) np.testing.assert_allclose(res, expected) r = fftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = fftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, s=(11, 12, 5)) np.testing.assert_allclose(res, expected) r = fftn(t, s=(11, 12, 5), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftn(raw, s=(11, 12, 5), axes=(-1, -2, -3)) np.testing.assert_allclose(res, expected) def testIFFTNExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(10, 20, 30)) r = ifftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw) np.testing.assert_allclose(res, expected) r = ifftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = ifftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, s=(11, 12, 5)) np.testing.assert_allclose(res, expected) r = ifftn(t, s=(11, 12, 5), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, s=(11, 12, 5), axes=(-1, -2, -3)) np.testing.assert_allclose(res, expected) raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(3, 4, 7)) r = ifftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw) np.testing.assert_allclose(res, expected) r = ifftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = ifftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, s=(11, 12, 5)) np.testing.assert_allclose(res, expected) r = ifftn(t, s=(11, 12, 5), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftn(raw, s=(11, 12, 5), axes=(-1, -2, -3)) np.testing.assert_allclose(res, expected) def testRFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = rfft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft(raw) np.testing.assert_allclose(res, expected) r = rfft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = rfft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft(raw, n=11) np.testing.assert_allclose(res, expected) def testIRFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = irfft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft(raw) np.testing.assert_allclose(res, expected) r = irfft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = irfft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft(raw, n=11) np.testing.assert_allclose(res, expected) def testRFFT2Execution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 20, 30)) r = rfft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft2(raw) np.testing.assert_allclose(res, expected) r = rfft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft2(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = rfft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft2(raw, s=(11, 12)) np.testing.assert_allclose(res, expected) r = rfft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfft2(raw, s=(11, 12), axes=(-1, -2)) np.testing.assert_allclose(res, expected) def testIRFFT2Execution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 20, 30)) r = irfft2(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft2(raw) np.testing.assert_allclose(res, expected) r = irfft2(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft2(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = irfft2(t, s=(11, 12)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft2(raw, s=(11, 12)) np.testing.assert_allclose(res, expected) r = irfft2(t, s=(11, 12), axes=(-1, -2)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfft2(raw, s=(11, 12), axes=(-1, -2)) np.testing.assert_allclose(res, expected) def testRFFTNExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(10, 20, 30)) r = rfftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfftn(raw) np.testing.assert_allclose(res, expected) r = rfftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfftn(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = rfftn(t, s=(11, 12, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfftn(raw, s=(11, 12, 5)) np.testing.assert_allclose(res, expected) r = rfftn(t, s=(11, 12, 11), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.rfftn(raw, s=(11, 12, 11), axes=(-1, -2, -3)) np.testing.assert_allclose(res, expected) def testIRFFTNExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(10, 20, 30)) r = irfftn(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfftn(raw) np.testing.assert_allclose(res, expected) r = irfftn(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfftn(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = irfftn(t, s=(11, 21, 5)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfftn(raw, s=(11, 21, 5)) np.testing.assert_allclose(res, expected) r = irfftn(t, s=(11, 21, 30), axes=(-1, -2, -3)) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.irfftn(raw, s=(11, 21, 30), axes=(-1, -2, -3)) np.testing.assert_allclose(res, expected) def testHFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = hfft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.hfft(raw) np.testing.assert_allclose(res, expected) r = hfft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.hfft(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = hfft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.hfft(raw, n=11) np.testing.assert_allclose(res, expected) def testIHFFTExecution(self): raw = np.random.rand(10, 20, 30) t = tensor(raw, chunk_size=(4, 4, 30)) r = ihfft(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ihfft(raw) np.testing.assert_allclose(res, expected) r = ihfft(t, norm='ortho') res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ihfft(raw, norm='ortho') np.testing.assert_allclose(res, expected) r = ihfft(t, n=11) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ihfft(raw, n=11) np.testing.assert_allclose(res, expected) r = ihfft(t, n=12) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ihfft(raw, n=12) np.testing.assert_allclose(res, expected) def testFFTFreqExecution(self): t = fftfreq(10, .1, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, np.fft.fftfreq(10, .1)) t = fftfreq(11, .01, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, np.fft.fftfreq(11, .01)) def testRFFTFreqExecution(self): t = rfftfreq(20, .1, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, np.fft.rfftfreq(20, .1)) t = rfftfreq(21, .01, chunk_size=3) res = self.executor.execute_tensor(t, concat=True)[0] np.testing.assert_allclose(res, np.fft.rfftfreq(21, .01)) def testFFTShiftExecution(self): t = fftfreq(10, .1, chunk_size=3) r = fftshift(t) res = self.executor.execute_tensor(r, concat=True)[0] np.testing.assert_allclose(res, np.fft.fftshift(np.fft.fftfreq(10, .1))) freqs = fftfreq(9, d=1. / 9, chunk_size=2).reshape(3, 3) r = fftshift(freqs, axes=(1, )) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.fftshift(np.fft.fftfreq(9, d=1. / 9).reshape(3, 3), axes=(1, )) np.testing.assert_allclose(res, expected) def testIFFTShiftExecution(self): t = fftfreq(9, d=1. / 9, chunk_size=2).reshape(3, 3) r = ifftshift(t) res = self.executor.execute_tensor(r, concat=True)[0] expected = np.fft.ifftshift(np.fft.fftfreq(9, d=1. / 9).reshape(3, 3)) np.testing.assert_allclose(res, expected)
class Test(TestBase): def setUp(self): super(Test, self).setUp() self.executor = Executor() def testAddWithoutShuffleExecution(self): # all the axes are monotonic # data1 with index split into [0...4], [5...9], # columns [3...7], [8...12] data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=np.arange(3, 13)) df1 = from_pandas(data1, chunk_size=5) # data2 with index split into [6...11], [2, 5], # columns [4...9], [10, 13] data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=np.arange(4, 14)) df2 = from_pandas(data2, chunk_size=6) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testAddWithOneShuffleExecution(self): # only 1 axis is monotonic # data1 with index split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=5) # data2 with index split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=6) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # only 1 axis is monotonic # data1 with columns split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7], columns=np.arange(10)) df1 = from_pandas(data1, chunk_size=5) # data2 with columns split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2], columns=np.arange(11, 1, -1)) df2 = from_pandas(data2, chunk_size=6) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testAddWithAllShuffleExecution(self): # no axis is monotonic data1 = pd.DataFrame(np.random.rand(10, 10), index=[0, 10, 2, 3, 4, 5, 6, 7, 8, 9], columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame(np.random.rand(10, 10), index=[11, 1, 2, 5, 7, 6, 8, 9, 10, 3], columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=6) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testAddBothWithOneChunk(self): # only 1 axis is monotonic # data1 with index split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=10) # data2 with index split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=10) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # only 1 axis is monotonic # data1 with columns split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7], columns=np.arange(10)) df1 = from_pandas(data1, chunk_size=10) # data2 with columns split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2], columns=np.arange(11, 1, -1)) df2 = from_pandas(data2, chunk_size=10) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testAddWithoutShuffleAndWithOneChunk(self): # only 1 axis is monotonic # data1 with index split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=(5, 10)) # data2 with index split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=(6, 10)) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # only 1 axis is monotonic # data1 with columns split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7], columns=np.arange(10)) df1 = from_pandas(data1, chunk_size=(10, 5)) # data2 with columns split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2], columns=np.arange(11, 1, -1)) df2 = from_pandas(data2, chunk_size=(10, 6)) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testAddWithShuffleAndWithOneChunk(self): # only 1 axis is monotonic # data1 with index split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(10), columns=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7]) df1 = from_pandas(data1, chunk_size=(10, 5)) # data2 with index split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=np.arange(11, 1, -1), columns=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2]) df2 = from_pandas(data2, chunk_size=(10, 6)) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) # only 1 axis is monotonic # data1 with columns split into [0...4], [5...9], data1 = pd.DataFrame(np.random.rand(10, 10), index=[4, 1, 3, 2, 10, 5, 9, 8, 6, 7], columns=np.arange(10)) df1 = from_pandas(data1, chunk_size=(5, 10)) # data2 with columns split into [6...11], [2, 5], data2 = pd.DataFrame(np.random.rand(10, 10), index=[5, 9, 12, 3, 11, 10, 6, 4, 1, 2], columns=np.arange(11, 1, -1)) df2 = from_pandas(data2, chunk_size=(6, 10)) df3 = add(df1, df2) expected = data1 + data2 result = self.executor.execute_dataframe(df3, concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testAddWithAdded(self): data1 = pd.DataFrame(np.random.rand(10, 10)) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame(np.random.rand(10, 10)) df2 = from_pandas(data2, chunk_size=6) df3 = add(df1, df2) data4 = pd.DataFrame(np.random.rand(10, 10)) df4 = from_pandas(data4, chunk_size=6) df5 = add(df3, df4) result = self.executor.execute_dataframe(df5, concat=True)[0] expected = data1 + data2 + data4 pd.testing.assert_frame_equal(expected, result) def testRadd(self): data1 = pd.DataFrame(np.random.rand(10, 10)) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame(np.random.rand(10, 10)) df2 = from_pandas(data2, chunk_size=6) radd = getattr(df2, '__radd__') df3 = radd(df1, df2) result = self.executor.execute_dataframe(df3, concat=True)[0] expected = data1 + data2 pd.testing.assert_frame_equal(expected, result) def testAddWithMultiForms(self): # test multiple forms of add # such as self+other, self.add(other), add(self,other) data1 = pd.DataFrame(np.random.rand(10, 10)) df1 = from_pandas(data1, chunk_size=5) data2 = pd.DataFrame(np.random.rand(10, 10)) df2 = from_pandas(data2, chunk_size=6) expected = data1 + data2 result = self.executor.execute_dataframe(df1 + df2, concat=True)[0] pd.testing.assert_frame_equal(expected, result) result = self.executor.execute_dataframe(add(df1, df2), concat=True)[0] pd.testing.assert_frame_equal(expected, result) result = self.executor.execute_dataframe(df1.add(df2), concat=True)[0] pd.testing.assert_frame_equal(expected, result) result = self.executor.execute_dataframe(df1.radd(df2), concat=True)[0] pd.testing.assert_frame_equal(expected, result) def testAbs(self): data1 = pd.DataFrame(np.random.uniform(low=-1, high=1, size=(10, 10))) df1 = from_pandas(data1, chunk_size=5) result = self.executor.execute_dataframe(abs(df1), concat=True)[0] expected = data1.abs() pd.testing.assert_frame_equal(expected, result)
def setUp(self): self.executor = Executor('numpy', storage=MockStorage(), prefetch=True)