def test_single_shard_matrix(self):
X = np.random.randn(128, 128)
X_sharded = local_numpy_init(X, X.shape)
local_numpy_init(X, shard_sizes=X.shape)
X_sharded_local = X_sharded.numpy()
X_sharded.free()
assert (np.all(X_sharded_local == X))
def test_multiple_shard_matrix_uneven(self):
X = np.random.randn(200, 200)
shard_sizes = (101, 101)
X_sharded = local_numpy_init(X, shard_sizes=shard_sizes)
X_sharded_local = X_sharded.numpy()
X_sharded.free()
assert (np.all(X == X_sharded_local))
def test_multiple_shard_matrix(self):
X = np.random.randn(128, 128)
shard_sizes = tuple(map(int, np.array(X.shape) / 2))
X_sharded = local_numpy_init(X, shard_sizes=shard_sizes)
X_sharded_local = X_sharded.numpy()
X_sharded.free()
assert (np.all(X == X_sharded_local))
def test_sharded_matrix_row_get(self):
X = np.random.randn(8, 8)
X_sharded = local_numpy_init(X, shard_sizes=[1, 1])
row_0 = matrix_utils.get_row(X_sharded, 0)
X_sharded.free()
print(X[0].shape)
print(row_0.shape)
os.system("rm -rf /dev/shm/*")
assert (np.all(X[0] == row_0))
def test_multiple_shard_matrix(self):
pwex = pywren.default_executor()
X = np.random.randn(128, 128)
X_sharded = local_numpy_init(X, (64, 64))
X_sharded_down = matrix_init.reshard_down(X_sharded, (4, 4), pwex)
X_sharded_local = X_sharded.numpy()
X_sharded_local_2 = X_sharded_down.numpy()
X_sharded.free()
X_sharded_down.free()
assert (np.all(X_sharded_local == X))
assert (np.all(X_sharded_local_2 == X))
def test_matrix_header(self):
np.random.seed(0)
X = np.random.randn(128, 128)
shard_sizes = tuple(map(int, np.array(X.shape) / 2))
X_sharded = local_numpy_init(X,
shard_sizes=shard_sizes,
write_header=True)
X_sharded_local = X_sharded.numpy()
assert (np.all(X == X_sharded_local))
X_sharded_2 = BigMatrix(X_sharded.key)
X_sharded_local_2 = X_sharded_2.numpy()
assert (np.all(X == X_sharded_local_2))
def test_single_shard_matrix(self):
# pwex = pywren.default_executor()
pwex = lithops.FunctionExecutor()
X = np.random.randn(128,128)
X_sharded = local_numpy_init(X, (128,128))
X_sharded_down = matrix_init.reshard_down(X_sharded, (4,4), pwex)
X_sharded_local = X_sharded.numpy()
X_sharded_local_2 = X_sharded_down.numpy()
X_sharded.free()
X_sharded_down.free()
assert(np.all(X_sharded_local == X))
assert(np.all(X_sharded_local_2 == X))
def test_sharded_matrix_row_get_big(self):
s = 2
X = np.arange(0, 2048 * 2048 * s).reshape(2048, 2048 * s)
X_sharded = local_numpy_init(X, shard_sizes=[2048, 2048])
t = time.time()
row_0 = matrix_utils.get_row(X_sharded, 0)
e = time.time()
print(row_0.shape)
print("Effective GB/s", (2048 * 2048 * s * 8) / (1e9 * (e - t)))
print("Download Time", e - t)
X_sharded.free()
os.system("rm -rf /dev/shm/*")
assert (np.all(X == row_0))
def test_multiple_shard_tensor(self):
pwex = pywren.default_executor()
X = np.random.randn(128,128, 4)
X_sharded = local_numpy_init(X, (64,64, 4))
X_sharded.autosqueeze = False
X_sharded_down = matrix_init.reshard_down(X_sharded, (4,4,2), pwex=None)
X_sharded_local = X_sharded.numpy()
X_sharded_local_2 = X_sharded_down.numpy()
X_sharded.free()
X_sharded_down.free()
X_sharded_down.get_block(0,0,0)
assert(np.all(X_sharded_local == X))
assert(np.all(X_sharded_local_2 == X))