def test_if_run(self):
X = np.random.randn(64)
shard_sizes = (int(X.shape[0]/8),)
X_sharded = BigMatrix("if_test", shape=X.shape,
shard_sizes=shard_sizes, write_header=True)
O_sharded = BigMatrix("if_test_output", shape=X.shape,
shard_sizes=shard_sizes, write_header=True)
X_sharded.free()
shard_matrix(X_sharded, X)
f = frontend.lpcompile(f1_if)
p = f(X_sharded, O_sharded, X_sharded.num_blocks(0))
num_cores = 1
executor = fs.ProcessPoolExecutor(num_cores)
config = npw.config.default()
p_ex = lp.LambdaPackProgram(p, config=config)
p_ex.start()
all_futures = []
for i in range(num_cores):
all_futures.append(executor.submit(
job_runner.lambdapack_run, p_ex, pipeline_width=1, idle_timeout=5, timeout=60))
p_ex.wait()
time.sleep(5)
p_ex.free()
for i in range(X_sharded.num_blocks(0)):
Ob = O_sharded.get_block(i)
Xb = X_sharded.get_block(i)
if ((i % 2) == 0):
assert(np.allclose(Ob, 1*Xb))
else:
assert(np.allclose(Ob, 2*Xb))
def test_bdfac(self):
N = 8
shard_size = 2
shard_sizes = (shard_size, shard_size)
X = np.random.randn(8, 8)
X_sharded = BigMatrix("BDFAC_input_X", shape=X.shape,
shard_sizes=shard_sizes, write_header=True)
shard_matrix(X_sharded, X)
N_blocks = X_sharded.num_blocks(0)
b_fac = 2
num_tree_levels = max(
int(np.ceil(np.log2(X_sharded.num_blocks(0))/np.log2(b_fac))), 1)
async def parent_fn(self, loop, *block_idxs):
if (block_idxs[-1] == 0 and block_idxs[-2] == 0):
return await X_sharded.get_block_async(None, *block_idxs[:-2])
VLs = BigMatrix("VL", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False)
TLs = BigMatrix("TL", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False)
VRs = BigMatrix("VR", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False)
TRs = BigMatrix("TR", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False)
Rs = BigMatrix("R", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False)
S0 = BigMatrix("S0", shape=(N, N, N, num_tree_levels*shard_size), shard_sizes=(shard_size,
shard_size, shard_size, shard_size), write_header=True, parent_fn=parent_fn, safe=False)
S1 = BigMatrix("S1", shape=(N, N, N, num_tree_levels*shard_size), shard_sizes=(shard_size,
shard_size, shard_size, shard_size), write_header=True, parent_fn=parent_fn, safe=False)
Sigma = BigMatrix("Sigma", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False, parent_fn=parent_fn)
pc = frontend.lpcompile(BDFAC)(
VLs, TLs, Rs, Sigma, VRs, TRs, S0, S1, N_blocks, 0)
def test_matmul(self):
size = 4
shard_size = 2
np.random.seed(0)
A = np.random.randn(size, size)
B = np.random.randn(size, size)
C = np.dot(A, B)
shard_sizes = (shard_size, shard_size)
A_sharded = BigMatrix("matmul_test_A",
shape=A.shape,
shard_sizes=shard_sizes,
write_header=True)
A_sharded.free()
shard_matrix(A_sharded, A)
B_sharded = BigMatrix("matmul_test_B",
shape=B.shape,
shard_sizes=shard_sizes,
write_header=True)
B_sharded.free()
shard_matrix(B_sharded, B)
Temp = BigMatrix("matmul_test_Temp",
shape=[A.shape[0], B.shape[1], B.shape[0], 100],
shard_sizes=[
A_sharded.shard_sizes[0],
B_sharded.shard_sizes[1], 1, 1
],
write_header=True)
C_sharded = BigMatrix("matmul_test_C",
shape=C.shape,
shard_sizes=shard_sizes,
write_header=True)
b_fac = 2
config = npw.config.default()
compiled_matmul = frontend.lpcompile(matmul)
program = compiled_matmul(A_sharded, B_sharded,
A_sharded.num_blocks(0),
A_sharded.num_blocks(1),
B_sharded.num_blocks(1), b_fac, Temp,
C_sharded)
program_executable = lp.LambdaPackProgram(program, config=config)
program_executable.start()
job_runner.lambdapack_run(program_executable,
pipeline_width=1,
idle_timeout=5,
timeout=60)
executor = fs.ThreadPoolExecutor(1)
all_futures = [
executor.submit(job_runner.lambdapack_run,
program_executable,
pipeline_width=1,
idle_timeout=5,
timeout=60)
]
program_executable.wait()
program_executable.free()
C_remote = C_sharded.numpy()
assert (np.allclose(C, C_remote))
def test_qr_lambda():
N = 16
shard_size = 8
shard_sizes = (shard_size, shard_size)
X = np.random.randn(N, N)
X_sharded = BigMatrix("QR_input_X", shape=X.shape,
shard_sizes=shard_sizes, write_header=True)
N_blocks = X_sharded.num_blocks(0)
shard_matrix(X_sharded, X)
program, meta = qr(X_sharded)
print(program.hash)
program.start()
print("starting program...")
futures = run_program_in_pywren(program, num_workers=1)
# futures[0].result()
program.wait()
program.free()
Rs = meta["outputs"][0]
R_remote = Rs.get_block(N_blocks - 1, N_blocks - 1, 0)
R_local = np.linalg.qr(X)[1][-shard_size:, -shard_size:]
sign_matrix_local = np.eye(R_local.shape[0])
sign_matrix_remote = np.eye(R_local.shape[0])
sign_matrix_local[np.where(np.diag(R_local) <= 0)] *= -1
sign_matrix_remote[np.where(np.diag(R_remote) <= 0)] *= -1
# make the signs match
R_remote *= np.diag(sign_matrix_remote)[:, np.newaxis]
R_local *= np.diag(sign_matrix_local)[:, np.newaxis]
assert(np.allclose(R_local, R_remote))
def test_qr():
N = 28
shard_size = 7
shard_sizes = (shard_size, shard_size)
X = np.random.randn(N, N)
X_sharded = BigMatrix("QR_input_X", shape=X.shape,
shard_sizes=shard_sizes, write_header=True)
N_blocks = X_sharded.num_blocks(0)
shard_matrix(X_sharded, X)
program, meta = qr(X_sharded)
executor = fs.ProcessPoolExecutor(2)
program.start()
print("starting program...")
future = executor.submit(job_runner.lambdapack_run,
program, timeout=60, idle_timeout=6, pipeline_width=1)
future = executor.submit(job_runner.lambdapack_run,
program, timeout=60, idle_timeout=6, pipeline_width=1)
future = executor.submit(job_runner.lambdapack_run,
program, timeout=60, idle_timeout=6, pipeline_width=1)
program.wait()
program.free()
Rs = meta["outputs"][0]
R_remote = Rs.get_block(N_blocks - 1, N_blocks - 1, 0)
R_local = np.linalg.qr(X)[1][-shard_size:, -shard_size:]
sign_matrix_local = np.eye(R_local.shape[0])
sign_matrix_remote = np.eye(R_local.shape[0])
sign_matrix_local[np.where(np.diag(R_local) <= 0)] *= -1
sign_matrix_remote[np.where(np.diag(R_remote) <= 0)] *= -1
# make the signs match
R_remote *= np.diag(sign_matrix_remote)[:, np.newaxis]
R_local *= np.diag(sign_matrix_local)[:, np.newaxis]
assert(np.allclose(R_local, R_remote))
def test_bdfac_truncated():
N = 16
shard_size = 4
shard_sizes = (shard_size, shard_size)
np.random.seed(0)
X = np.random.randn(N, N)
U, S, V = bdfac_python(X, block_size=shard_size)
svd_bdfac = np.linalg.svd(S, compute_uv=False)
svd_local = np.linalg.svd(X, compute_uv=False)
print(svd_bdfac)
print(svd_local)
assert (np.allclose(svd_bdfac, svd_local))
X_sharded = BigMatrix("BDFAC_input_X",
shape=X.shape,
shard_sizes=shard_sizes,
write_header=True)
N_blocks = X_sharded.num_blocks(0)
shard_matrix(X_sharded, X)
program, meta = bdfac(X_sharded, truncate=2)
executor = fs.ProcessPoolExecutor(1)
program.start()
executor.submit(job_runner.lambdapack_run,
program,
timeout=200,
idle_timeout=200,
pipeline_width=1)
program.wait()
print("returned..")
def test_bdfac():
N = 16
shard_size = 4
shard_sizes = (shard_size, shard_size)
np.random.seed(0)
X = np.random.randn(N, N)
U, S, V = bdfac_python(X, block_size=shard_size)
svd_bdfac = np.linalg.svd(S, compute_uv=False)
svd_local = np.linalg.svd(X, compute_uv=False)
print(svd_bdfac)
print(svd_local)
assert(np.allclose(svd_bdfac, svd_local))
X_sharded = BigMatrix("BDFAC_input_X", shape=X.shape,
shard_sizes=shard_sizes, write_header=True)
N_blocks = X_sharded.num_blocks(0)
shard_matrix(X_sharded, X)
program, meta = bdfac(X_sharded)
executor = fs.ProcessPoolExecutor(1)
program.start()
job_runner.lambdapack_run(program, timeout=200,
idle_timeout=200, pipeline_width=1)
program.wait()
print("returned..")
program.free()
R = meta["outputs"][1]
L = meta["outputs"][0]
print("====="*10)
R_remote = R.get_block(N_blocks - 1, 0, N_blocks - 1)
R_local = S[-shard_size:, -shard_size:]
print("original", R_local)
print("remote", R_remote)
print('==='*10)
sign_matrix_local = np.eye(R_local.shape[0])
sign_matrix_remote = np.eye(R_local.shape[0])
sign_matrix_local[np.where(np.diag(R_local) <= 0)] *= -1
sign_matrix_remote[np.where(np.diag(R_remote) <= 0)] *= -1
# make the signs match
R_remote *= np.diag(sign_matrix_remote)[:, np.newaxis]
R_local *= np.diag(sign_matrix_local)[:, np.newaxis]
print(R_local)
print(R_remote)
assert(np.allclose(np.abs(R_local), np.abs(R_remote)))
fac = np.block([[R.get_block(0, 2, 0), L.get_block(0, 2, 1), np.zeros(shard_sizes), np.zeros(shard_sizes)],
[np.zeros(shard_sizes), R.get_block(1, 2, 1),
L.get_block(1, 1, 2), np.zeros(shard_sizes)],
[np.zeros(shard_sizes), np.zeros(shard_sizes),
R.get_block(2, 1, 2), L.get_block(2, 0, 3)],
[np.zeros(shard_sizes), np.zeros(shard_sizes), np.zeros(shard_sizes), R.get_block(3, 0, 3)]])
svd_remote = np.linalg.svd(fac, compute_uv=False)
svd_local = np.linalg.svd(X, compute_uv=False)
assert(np.allclose(svd_remote, svd_local))
return 0
def test_if_static(self):
X = np.random.randn(64, 64)
shard_sizes = (int(X.shape[0]/8), X.shape[1])
X_sharded = BigMatrix("if_test", shape=X.shape,
shard_sizes=shard_sizes, write_header=True)
O_sharded = BigMatrix("if_test_output", shape=X.shape,
shard_sizes=shard_sizes, write_header=True)
X_sharded.free()
shard_matrix(X_sharded, X)
f = frontend.lpcompile(f1_if)
p = f(X_sharded, O_sharded, X_sharded.num_blocks(0))
assert(p.starters == p.find_terminators())
for s, var_values in p.starters:
if(var_values['i'] % 2 == 0):
assert s == 0
else:
assert s == 1
size = 64
shard_size = 16
N = 64
shard_size = 16
shard_sizes = (shard_size, shard_size)
X = np.random.randn(size, size)
X_sharded = BigMatrix("tsqr_test_X", shape=X.shape,
shard_sizes=shard_sizes, write_header=False)
b_fac = 2
async def parent_fn(self, loop, *block_idxs):
if (block_idxs[-1] == 0 and block_idxs[-2] == 0):
return await X_sharded.get_block_async(None, *block_idxs[:-2])
num_tree_levels = max(
int(np.ceil(np.log2(X_sharded.num_blocks(0))/np.log2(b_fac))), 1)
R_sharded = BigMatrix("tsqr_test_R", shape=(num_tree_levels*shard_size,
X_sharded.shape[0]), shard_sizes=shard_sizes, write_header=False, safe=False)
V_sharded = BigMatrix("tsqr_test_V", shape=(num_tree_levels*shard_size*b_fac,
X_sharded.shape[0]), shard_sizes=(shard_size*b_fac, shard_size), write_header=False, safe=False)
T_sharded = BigMatrix("tsqr_test_T", shape=(num_tree_levels*shard_size*b_fac,
X_sharded.shape[0]), shard_sizes=(shard_size*b_fac, shard_size), write_header=False, safe=False)
I = BigMatrix("I", shape=(N, N), shard_sizes=(
shard_size, shard_size), write_header=True, safe=False)
Vs = BigMatrix("Vs", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False)
Ts = BigMatrix("Ts", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False)
Rs = BigMatrix("Rs", shape=(num_tree_levels, N, N), shard_sizes=(
1, shard_size, shard_size), write_header=True, safe=False)
Ss = BigMatrix("Ss", shape=(N, N, N, num_tree_levels*shard_size), shard_sizes=(shard_size,