def test_single_shard_matrix(self): X = np.random.randn(128, 128) X_sharded = local_numpy_init(X, X.shape) local_numpy_init(X, shard_sizes=X.shape) X_sharded_local = X_sharded.numpy() X_sharded.free() assert (np.all(X_sharded_local == X))
def test_multiple_shard_matrix_uneven(self): X = np.random.randn(200, 200) shard_sizes = (101, 101) X_sharded = local_numpy_init(X, shard_sizes=shard_sizes) X_sharded_local = X_sharded.numpy() X_sharded.free() assert (np.all(X == X_sharded_local))
def test_multiple_shard_matrix(self): X = np.random.randn(128, 128) shard_sizes = tuple(map(int, np.array(X.shape) / 2)) X_sharded = local_numpy_init(X, shard_sizes=shard_sizes) X_sharded_local = X_sharded.numpy() X_sharded.free() assert (np.all(X == X_sharded_local))
def test_sharded_matrix_row_get(self): X = np.random.randn(8, 8) X_sharded = local_numpy_init(X, shard_sizes=[1, 1]) row_0 = matrix_utils.get_row(X_sharded, 0) X_sharded.free() print(X[0].shape) print(row_0.shape) os.system("rm -rf /dev/shm/*") assert (np.all(X[0] == row_0))
def test_multiple_shard_matrix(self): pwex = pywren.default_executor() X = np.random.randn(128, 128) X_sharded = local_numpy_init(X, (64, 64)) X_sharded_down = matrix_init.reshard_down(X_sharded, (4, 4), pwex) X_sharded_local = X_sharded.numpy() X_sharded_local_2 = X_sharded_down.numpy() X_sharded.free() X_sharded_down.free() assert (np.all(X_sharded_local == X)) assert (np.all(X_sharded_local_2 == X))
def test_matrix_header(self): np.random.seed(0) X = np.random.randn(128, 128) shard_sizes = tuple(map(int, np.array(X.shape) / 2)) X_sharded = local_numpy_init(X, shard_sizes=shard_sizes, write_header=True) X_sharded_local = X_sharded.numpy() assert (np.all(X == X_sharded_local)) X_sharded_2 = BigMatrix(X_sharded.key) X_sharded_local_2 = X_sharded_2.numpy() assert (np.all(X == X_sharded_local_2))
def test_single_shard_matrix(self): # pwex = pywren.default_executor() pwex = lithops.FunctionExecutor() X = np.random.randn(128,128) X_sharded = local_numpy_init(X, (128,128)) X_sharded_down = matrix_init.reshard_down(X_sharded, (4,4), pwex) X_sharded_local = X_sharded.numpy() X_sharded_local_2 = X_sharded_down.numpy() X_sharded.free() X_sharded_down.free() assert(np.all(X_sharded_local == X)) assert(np.all(X_sharded_local_2 == X))
def test_sharded_matrix_row_get_big(self): s = 2 X = np.arange(0, 2048 * 2048 * s).reshape(2048, 2048 * s) X_sharded = local_numpy_init(X, shard_sizes=[2048, 2048]) t = time.time() row_0 = matrix_utils.get_row(X_sharded, 0) e = time.time() print(row_0.shape) print("Effective GB/s", (2048 * 2048 * s * 8) / (1e9 * (e - t))) print("Download Time", e - t) X_sharded.free() os.system("rm -rf /dev/shm/*") assert (np.all(X == row_0))
def test_multiple_shard_tensor(self): pwex = pywren.default_executor() X = np.random.randn(128,128, 4) X_sharded = local_numpy_init(X, (64,64, 4)) X_sharded.autosqueeze = False X_sharded_down = matrix_init.reshard_down(X_sharded, (4,4,2), pwex=None) X_sharded_local = X_sharded.numpy() X_sharded_local_2 = X_sharded_down.numpy() X_sharded.free() X_sharded_down.free() X_sharded_down.get_block(0,0,0) assert(np.all(X_sharded_local == X)) assert(np.all(X_sharded_local_2 == X))