def test_version_mismatch(self):
my_version_str = pywren.runtime.version_str(sys.version_info)
for supported_version in pywren.wrenconfig.default_runtime.keys():
if my_version_str != supported_version:
wrong_version = supported_version
config = pywren.wrenconfig.default()
config['runtime']['s3_key'] = pywren.wrenconfig.default_runtime[wrong_version]
with pytest.raises(Exception) as excinfo:
pywren.lambda_executor(config)
assert 'python version' in str(excinfo.value)
def benchmark(loopcount, workers, matn, verbose=False):
t1 = time.time()
N = workers
iters = np.arange(N)
def f(x):
return {'flops': compute_flops(loopcount, matn)}
pwex = pywren.lambda_executor(shard_runtime=True)
futures = pwex.map(f, iters)
print("invocation done, dur=", time.time() - t1)
print("callset id: ", futures[0].callset_id)
local_jobs_done_timeline = []
result_count = 0
while result_count < N:
fs_dones, fs_notdones = pywren.wait(futures, pywren.wren.ALWAYS)
result_count = len(fs_dones)
local_jobs_done_timeline.append((time.time(), result_count))
est_flop = 2 * result_count * loopcount * matn**3
est_gflops = est_flop / 1e9 / (time.time() - t1)
if verbose:
print("jobs done: {:5d} runtime: {:5.1f}s {:8.1f} GFLOPS ".
format(result_count,
time.time() - t1, est_gflops))
if result_count == N:
break
time.sleep(1)
if verbose:
print("getting results")
results = [f.result(throw_except=False) for f in futures]
if verbose:
print("getting status")
run_statuses = [f.run_status for f in futures]
invoke_statuses = [f.invoke_status for f in futures]
all_done = time.time()
total_time = all_done - t1
print("total time", total_time)
est_flop = result_count * 2 * loopcount * matn**3
print est_flop / 1e9 / total_time, "GFLOPS"
res = {
'total_time': total_time,
'est_flop': est_flop,
'run_statuses': run_statuses,
'invoke_statuses': invoke_statuses,
'callset_id': futures[0].callset_id,
'local_jobs_done_timeline': local_jobs_done_timeline,
'results': results
}
return res
def gemm_recompute(A, B, thresh, s3_key):
"""
Compute A * B.T via speculative execution (i.e., recompute straggling workers).
Params
======
A : numpywren.matrix.BigMatrix
First input matrix.
B : numpywren.matrix.BigMatrix
Second input matrix.
thresh : float (in [0, 1])
Fraction of workers that should finish before recomputing.
s3_key : str
Storage key for output matrix.
Returns
=======
C : matrix.BigMatrix
Resultant matrix product.
t_comp : float
Time for thresh percentage of the workers to finish.
t_straggle : float
Time for the remaining 1 - thresh percentage of the workers to finish after
we begin recomputing.
"""
if not (0 <= thresh <= 1):
raise ValueError("thresh must be in the interval [0, 1]")
"""Initialize output matrix"""
num_col_blocks = A.shape[1] // A.shard_sizes[1]
shard_sizes = (A.shard_sizes[0], B.shard_sizes[0])
C = matrix.BigMatrix(s3_key, shape=(A.shape[0], B.shape[0]), shard_sizes=shard_sizes, autosqueeze=False, write_header=True)
C.delete() # Only needed if you reuse the same s3_key (if the blocks already exist, no work will be done here)
"""Stage 1: Compute "thresh" percentage of the results"""
t_comp_start = time.time()
pwex = pywren.lambda_executor()
futures, num_done = pwex.map(lambda x: pywren_gemm(x, A, B, C, num_col_blocks), C.block_idxs), 0
while num_done < thresh * len(futures):
fs_dones, _ = pywren.wait(futures, return_when=ANY_COMPLETED)
num_done = len(fs_dones)
t_comp = time.time() - t_comp_start # Total stage 1 time
"""Stage 2: Recompute straggling workers (the last 1-thresh percent of jobs)"""
t_straggle_start = time.time()
futures_stragglers = pwex.map(lambda x: pywren_gemm(x, A, B, C, num_col_blocks), C.block_idxs_not_exist)
while len(C.block_idxs_not_exist) > 0:
pywren.wait(futures, return_when=ALWAYS)
pywren.wait(futures_stragglers, return_when=ALWAYS)
t_straggle = time.time() - t_straggle_start # Total stage 2 time
return C, t_comp, t_straggle
def test_single_shard_matrix_multiply(self):
X = np.random.randn(16, 16)
X_sharded = BigMatrix("gemm_test_0",
shape=X.shape,
shard_sizes=X.shape)
shard_matrix(X_sharded, X)
pwex = pywren.lambda_executor()
XXT_sharded = binops.gemm(pwex, X_sharded, X_sharded.T,
X_sharded.bucket, 1)
XXT_sharded_local = XXT_sharded.numpy()
XXT = X.dot(X.T)
X_sharded.free()
XXT_sharded.free()
assert (np.all(np.isclose(XXT, XXT_sharded_local)))
os.system("rm -rf /dev/shm/*")
def coded_mat_vec_mul(A_coded_2D, x, num_parity_blocks, coding_length):
def coded_mat_vec_mul(id):
shard_size = A_coded_2D.shard_sizes[1]
reduce_idxs = A_coded_2D._block_idxs(axis=1)
x_loc = x.get_block(0, 0)
Ax_block = None
for r in reduce_idxs:
block1 = A_coded_2D.get_block(id, r)
sidx = r * shard_size
eidx = (r + 1) * shard_size
x_block = x_loc[sidx:eidx]
if (Ax_block is None):
Ax_block = block1.dot(x_block)
else:
Ax_block = Ax_block + block1.dot(x_block)
return Ax_block
shard_size = A_coded_2D.shard_sizes[0]
n_coded_procs = len(A_coded_2D._block_idxs(0))
len_A_coded = n_coded_procs - coding_length - 1
pwex = pywren.lambda_executor()
futures = pwex.map(coded_mat_vec_mul, range(n_coded_procs))
Ax = np.zeros((A_coded_2D.shape[0], 1))
bitmask = np.ones((coding_length + 1, num_parity_blocks + 1))
not_done = list(range(n_coded_procs))
while decode2D.cant_be_decoded_with_bitmask(deepcopy(bitmask)):
fs_dones, fs_not_dones = pywren.wait(futures, 2)
for (id, f) in enumerate(futures):
if f in fs_dones and id in not_done:
# print("Worker done", id)
try:
Ax[id * shard_size:(id + 1) * shard_size] = f.result()
i, j = decode2D.ind1Dto2D(id, len_A_coded,
num_parity_blocks)
bitmask[i, j] = 0
not_done.remove(id)
except Exception as e:
print(e)
pass
# print("1: Decoding not dones", not_done)
Ax = decode2D.decode_vector_with_bitmask(Ax, bitmask, num_parity_blocks,
len_A_coded, shard_size,
coding_length)
return Ax
def test_multiple_shard_matrix_multiply_symmetric_2(self):
X = np.random.randn(16, 16)
shard_sizes = [8, 16]
X_sharded = BigMatrix("gemm_test_1",
shape=X.shape,
shard_sizes=shard_sizes)
shard_matrix(X_sharded, X)
pwex = pywren.lambda_executor()
XTX_sharded = binops.gemm(pwex,
X_sharded.T,
X_sharded,
X_sharded.bucket,
1,
local=True)
XTX_sharded_local = XTX_sharded.numpy()
XTX = X.T.dot(X)
X_sharded.free()
XTX_sharded.free()
assert (np.all(np.isclose(XTX, XTX_sharded_local)))
os.system("rm -rf /dev/shm/*")
def test_multiple_shard_matrix_multiply(self):
X = np.random.randn(16, 16)
Y = np.random.randn(16, 16)
shard_sizes = tuple(map(int, np.array(X.shape) / 2))
X_sharded = BigMatrix("gemm_test_1",
shape=X.shape,
shard_sizes=shard_sizes)
Y_sharded = BigMatrix("gemm_test_2",
shape=X.shape,
shard_sizes=shard_sizes)
shard_matrix(X_sharded, X)
shard_matrix(Y_sharded, Y)
pwex = pywren.lambda_executor()
XY_sharded = binops.gemm(pwex, X_sharded, Y_sharded, X_sharded.bucket,
1)
XY_sharded_local = XY_sharded.numpy()
XY = X.dot(Y)
X_sharded.free()
Y_sharded.free()
XY_sharded.free()
assert (np.all(np.isclose(XY, XY_sharded_local)))
os.system("rm -rf /dev/shm/*")
def recompute_mat_vec_mul(A_coded_2D, x, thres=0.95):
def shard_mat_vec_mul(id):
shard_size = A_coded_2D.shard_sizes[1]
reduce_idxs = A_coded_2D._block_idxs(axis=1)
x_loc = x.get_block(0, 0)
Ax_block = None
for r in reduce_idxs:
block1 = A_coded_2D.get_block(id, r)
sidx = r * shard_size
eidx = (r + 1) * shard_size
x_block = x_loc[sidx:eidx]
if (Ax_block is None):
Ax_block = block1.dot(x_block)
else:
Ax_block = Ax_block + block1.dot(x_block)
return Ax_block, id
shard_size = A_coded_2D.shard_sizes[0]
n_coded_procs = len(A_coded_2D._block_idxs(0))
pwex = pywren.lambda_executor()
futures = pwex.map(shard_mat_vec_mul, range(n_coded_procs))
Ax = np.zeros((A_coded_2D.shape[0], 1))
not_done = list(range(n_coded_procs))
fs_dones = []
f_result_dones = []
while len(fs_dones) < thres * n_coded_procs:
fs_dones, fs_not_dones = pywren.wait(futures, 2)
for f in list(set(fs_dones) - set(f_result_dones)):
# print("Worker done", id)
f_result_dones.append(f)
try:
result = f.result()
id = result[1]
Ax[id * shard_size:(id + 1) * shard_size] = result[0]
not_done.remove(id)
except Exception as e:
#print(e)
pass
time.sleep(2)
print("Recomputing not dones", not_done)
futures2 = pwex.map(shard_mat_vec_mul, not_done)
f_result_dones2 = []
while not_done != []:
fs_dones2, fs_not_dones2 = pywren.wait(futures2, 3)
for f in list(set(fs_dones2) - set(f_result_dones2)):
f_result_dones2.append(f)
try:
result = f.result()
id = result[1]
if id in not_done:
print("Recomputed", id)
Ax[id * shard_size:(id + 1) * shard_size] = result[0]
not_done.remove(id)
except Exception as e:
#print(e)
pass
time.sleep(2)
fs_dones, fs_not_dones = pywren.wait(futures, 3)
for f in list(set(fs_dones) - set(f_result_dones)):
f_result_dones.append(f)
try:
result = f.result()
id = result[1]
if id in not_done:
print("Straggler computed", id)
Ax[id * shard_size:(id + 1) * shard_size] = result[0]
not_done.remove(id)
except Exception as e:
#print(e)
pass
time.sleep(2)
if fs_not_dones2 == [] and fs_not_dones == []:
print("NOT DONE", not_done)
break
print("Recomputing done")
return Ax
def start_encode_mtx(M, blocks_per_parity, s3_key):
"""
Apply a (blocks_per_parity + 1, blocks_per_parity) MDS code to the matrix M every
blocks_per_parity rows by summing up the previous blocks_per_parity rows.
Params
======
M : numpywren.matrix.BigMatrix
The matrix to encode.
blocks_per_parity : int
The number of input blocks sum up in creating each parity block. Note that as
this number increases, less redundancy is provided.
s3_key : str
The storage key for Amazon S3.
Returns
=======
M_coded : numpywren.matrix.BigMatrix
The encoded matrix.
futures : list
List of the pywren futures.
num_workers : int
The number of workers.
"""
# Useful definitions
num_row_blocks = M.shape[0] // M.shard_sizes[0]
num_col_blocks = M.shape[1] // M.shard_sizes[1]
num_parity = num_row_blocks // blocks_per_parity # total number of parity blocks that will be added
coded_shape = (M.shape[0] + num_parity * M.shard_sizes[0], M.shape[1])
# Ensure no blocks will go uncoded
if not num_row_blocks % blocks_per_parity == 0:
raise ValueError("Number of row blocks ({0}) is not divisible \
by number of blocks per parity ({1})".format(
num_row_blocks, blocks_per_parity))
# Create the coded matrix object
coding_fn = make_coding_function(M, blocks_per_parity)
M_coded = matrix.BigMatrix(s3_key,
shape=coded_shape,
shard_sizes=M.shard_sizes,
write_header=True,
parent_fn=coding_fn)
M_coded.delete(
) # Only needed if you reuse the same s3_key (if the blocks already exist, no work will be done here)
# Generate encoding indices
encode_idx = []
for j in range(num_col_blocks):
for i in range(1, num_parity + 1):
encode_idx.append((i * (blocks_per_parity + 1) - 1, j))
num_workers = len(encode_idx)
# Encode the matrix
pwex = pywren.lambda_executor()
futures = pwex.map(lambda x: get_block_wrapper(M_coded, x), encode_idx)
return M_coded, futures, num_workers
def setUp(self):
self.lambda_exec = pywren.lambda_executor()
self.remote_exec = pywren.remote_executor()
import pywren
if __name__ == "__main__":
lambda_exec = pywren.lambda_executor()
remote_exec = pywren.remote_executor()
def increment(x):
return x + 1
x = [1, 2, 3, 4]
futures = lambda_exec.map(increment, x)
def reduce_func(x):
return sum(x)
reduce_future = remote_exec.reduce(reduce_func, futures)
print reduce_future.result()
if sys.version_info.major == python_ver_major \
and sys.version_info.minor == python_ver_minor:
print("running runtime config {}".format(runtime_name))
# create an executor
config = pywren.wrenconfig.default()
staged_runtime_url, staged_meta_url = runtimes.get_staged_runtime_url(
runtime_name, python_ver)
assert staged_runtime_url[:5] == "s3://"
splits = staged_runtime_url[5:].split("/")
bucket = splits[0]
key = "/".join(splits[1:])
config['runtime']['bucket'] = bucket
config['runtime']['s3_key'] = key
print("running with bucket={} key={}".format(bucket, key))
wrenexec = pywren.lambda_executor(config)
def import_check(x):
results = {}
conda_results = {}
for pkg in packages['conda_install']:
if pkg in runtimes.CONDA_TEST_STRS:
test_str = runtimes.CONDA_TEST_STRS[pkg]
try:
eval(test_str)
conda_results[pkg] = True
except ImportError:
conda_results[pkg] = False
results['conda'] = conda_results
def OverSketchFunc(A, B, d, thres = 0.95):
m = A.shape[0]
n = A.shape[1]
l = B.shape[1]
b = A.shard_sizes[0]
assert (d % b == 0)
assert (m % b == 0)
assert (l % b == 0)
assert (b == B.shard_sizes[1])
N = int(d/b)
sketch_A = matrix.BigMatrix("sketch_A_{0}_{1}".format(m, d), shape=(m, d), shard_sizes=(b, b))
sketch_BT = matrix.BigMatrix("sketch_B_{0}_{1}".format(l, d), shape=(l, d), shard_sizes=(b, b))
hashes = np.random.randint(0, b, size=(N, n))
flips = np.random.choice([-1,1], size=(N, n))
def OverSketchMatrix(id, X, hashes, flips, b, sketch):
"""
Calculates OverSketch AS for a row-block of a fat matrix A with block-size b
"""
x = id[0]
y = id[1]
A = X.get_block(x,0)
m,n = A.shape
hash_local = hashes[y,:]
flip_local = flips[y,:]
sketch_block = np.zeros((m, b))
for i in range(n):
sketch_block[:, hash_local[i]] += flip_local[i]*A[:,i]
sketch.put_block(sketch_block, x, y)
return 0
pwex = pywren.lambda_executor()
t1 = time.time()
futuresA = pwex.map(lambda x: OverSketchMatrix(x, A, hashes, flips, b, sketch_A), sketch_A.block_idxs)
futuresB = pwex.map(lambda x: OverSketchMatrix(x, B.T, hashes, flips, b, sketch_BT), sketch_BT.block_idxs)
fs_donesA = pywren.wait(futuresA, 2)[0]
fs_donesB = pywren.wait(futuresB, 2)[0]
while len(fs_donesA)<thres*len(futuresA) and len(fs_donesB)<thres*len(futuresB):
fs_donesA = pywren.wait(futuresA, 2)[0]
fs_donesB = pywren.wait(futuresB, 2)[0]
print("Sketching time", time.time() - t1)
## Computation phase
def blockMatMul(A, B, tensorAB, id):
"""
Multiplies A and B.T in a blocked fashion
"""
i = id[0]
j = id[1]
k = id[2]
X = A.get_block(i,k)
Y = B.get_block(j,k)
tensorAB[k].put_block(X.dot(Y.T), i, j)
return 0
tensorAB = []
for x in range(N):
tensorAB.append(matrix.BigMatrix("AxB_outer_{0}_{1}_{2}".format(m, l, x), shape=(m, l), shard_sizes=(b, b)))
computeArr = [(i,j,k) for (i,k) in sketch_A.block_idxs for j in sketch_BT._block_idxs(0)]
t1 = time.time()
futures = pwex.map(lambda x: blockMatMul(sketch_A, sketch_BT, tensorAB, x), computeArr)
fs_dones = pywren.wait(futures, 2)[0]
while len(fs_dones)<thres*len(futures):
fs_dones = pywren.wait(futures, 2)[0]
print("Computation time", time.time() - t1)
## Reduction phase
def blockMatMulReduction(tensorAB, AB, id):
"""
Reduces the output from computation phase to get A*B
Variable 'count' keeps track of number of blocks that have returned
"""
i = id[0]
j = id[1]
X = None
count = 1
for k in range(N):
if X is None:
try:
X = tensorAB[k].get_block(i,j)
except Exception as e:
print(e)
pass
else:
try:
X = X + tensorAB[k].get_block(i,j)
count = count+1
except Exception as e:
print(e)
pass
AB.put_block(X/count, i, j)
return 0
AB = matrix.BigMatrix("AxB_{0}_{1}".format(m, l), shape=(m, l), shard_sizes=(b, b))
reduceArr = [(i,j) for i in sketch_A._block_idxs(0) for j in sketch_BT._block_idxs(0)]
t1 = time.time()
futures_red = pwex.map(lambda x: blockMatMulReduction(tensorAB, AB, x), reduceArr)
fs_dones = pywren.wait(futures_red)[0]
print("Reduction time", time.time() - t1)
return AB
def gemm_coded(A, B, blocks_per_parity, s3_key, completion_pct=.7, encode_A=True, encode_B=True, np_A=-1, np_B=-1):
"""
Compute A * B.T using a locally recoverable product code for redundancy.
Params
======
A : numpywren.matrix.BigMatrix
First input matrix.
B : numpywren.matrix.BigMatrix
Second input matrix.
blocks_per_parity : int
Number of blocks to sum up when creating each parity block.
s3_key: str
Storage key for output matrix.
completion_pct: int
The fraction of multiplication workers that must finish before moving on to decoding.
encode_A : bool
Whether or not A needs to be encoded.
Allows for the user to pre-encode A if it will be used multiple times.
encode_B : bool
Whether or not B needs to be encoded.
Allows for the user to pre-encode B if it will be used multiple times.
np_A : int
Number of parity blocks in the matrix A. Should be provided if and only if
encode_A is set to false.
np_B : int
Number of parity blocks in the matrix B. Should be provided if and only if
encode_B is set to false.
Returns
=======
C : numpywren.matrix.BigMatrix
Resultant matrix product.
t_enc : float
Encoding time.
t_comp : float
Computation time.
t_dec : float
Decoding time.
"""
if (not encode_A) and np_A == -1:
raise ValueError("You must provide the number of parity blocks in A if you pre-encoded it.")
if (not encode_B) and np_B == -1:
raise ValueError("You must provide the number of parity blocks in B if you pre-encoded it.")
"""Stage 1: Encoding"""
start = time.time()
if encode_A or encode_B:
# Spin up encoding workers
num_workers = 0
if encode_A:
A_coded, futures_encode_A, num_workers_A = start_encode_mtx(A, blocks_per_parity, "A_coded")
num_workers += num_workers_A
if encode_B:
B_coded, futures_encode_B, num_workers_B = start_encode_mtx(B, blocks_per_parity, "B_coded")
num_workers += num_workers_B
# Wait until enough encoding workers are done to move on
num_done = 0
while num_done < MIN_ENCODING_COMPLETION_PCT * num_workers:
fs_A, fs_B = [], []
if encode_A:
fs_A, _ = pywren.wait(futures_encode_A, return_when=ANY_COMPLETED)
if encode_B:
fs_B, _ = pywren.wait(futures_encode_B, return_when=ANY_COMPLETED)
num_done = len(fs_A) + len(fs_B)
if not encode_A:
A_coded = A
if not encode_B:
B_coded = B
t_enc = time.time() - start # Total encoding time
"""Intermediate step: Initialize output matrix (untimed for consistency with gemm_recompute)."""
# Determine coded dimensions of A, B
if encode_A:
num_parity_A = (A.shape[0] // A.shard_sizes[0]) // blocks_per_parity
coded_shape_A = (A.shape[0] + num_parity_A * A.shard_sizes[0], A.shape[1])
else:
num_parity_A = np_A
coded_shape_A = A_coded.shape
if encode_B:
num_parity_B = (B.shape[0] // B.shard_sizes[0]) // blocks_per_parity
coded_shape_B = (B.shape[0] + num_parity_B * B.shard_sizes[0], B.shape[1])
else:
num_parity_B = np_B
coded_shape_B = B_coded.shape
# Create (encoded) output matrix
shard_sizes_C = (A.shard_sizes[0], B.shard_sizes[0])
C_coded = matrix.BigMatrix(s3_key + "coded", shape=(A_coded.shape[0], B_coded.shape[0]), \
shard_sizes=shard_sizes_C, \
autosqueeze=False, \
write_header=True)
C_coded.delete() # Only needed if you reuse the same s3_key (if the blocks already exist, no work will be done here)
# Generate indices for the output matrix
num_row_blocks_C = C_coded.shape[0] // C_coded.shard_sizes[0]
num_col_blocks_C = C_coded.shape[1] // C_coded.shard_sizes[1]
num_cols_coded = A_coded.shape[1] // A_coded.shard_sizes[1] # Inner dimension of the coded multiplication
block_idx_C = C_coded.block_idxs
num_workers = len(block_idx_C)
np.random.shuffle(block_idx_C) # Randomize jobs to avoid bad straggler locality
"""Stage 2: Multiply"""
t_comp_start = time.time()
pwex = pywren.lambda_executor()
futures_matmul = pwex.map(lambda x: pywren_gemm(x, A_coded, B_coded, C_coded, num_cols_coded), block_idx_C)
fs_done_matmul, num_done = [], 0
while num_done < completion_pct * num_workers:
fs_done_matmul, _ = pywren.wait(futures_matmul, return_when=ANY_COMPLETED)
num_done = len(fs_done_matmul)
t_comp = time.time() - t_comp_start # Total stage 2 time
"""Stage 3: Decoding"""
t_dec_start = time.time()
decode_idx = [(i, j) for i in range(num_parity_A) for j in range(num_parity_B)]
num_workers = len(decode_idx)
futures_decode = pwex.map(lambda x: decode_gemm(num_row_blocks_C, num_parity_A, C_coded, x), decode_idx)
fs_done_decode, num_done = [], 0
while num_done < num_workers and len(C_coded.block_idxs_not_exist) > 0:
fs_done_decode, _ = pywren.wait(futures_decode, return_when=ANY_COMPLETED)
num_done = len(fs_done_decode)
t_dec = time.time() - t_dec_start # Total stage 3 time
"""Final step: Specify the systematic part (i.e., all non-parity blocks) of the result"""
# Determine output dimensions
if encode_A:
C_num_rows = A.shape[0]
else:
C_num_rows = A.shape[0] - np_A * A.shard_sizes[0]
if encode_B:
C_num_cols = B.shape[0]
else:
C_num_cols = B.shape[0] - np_B * B.shard_sizes[0]
# Create the output matrix containing only the systematic part of the result
get_systematic_part = systematicize(C_coded, blocks_per_parity)
C_shard_sizes = (A.shard_sizes[0], B.shard_sizes[0])
C = matrix.BigMatrix(s3_key, shape=(C_num_rows, C_num_cols), shard_sizes=C_shard_sizes, parent_fn=get_systematic_part)
C.delete() # Only needed if you reuse the same s3_key (if the blocks already exist, no work will be done here)
return C, t_enc, t_comp, t_dec
def setUp(self):
self.wrenexec = pywren.lambda_executor(job_max_runtime=40)
def code_2D(A, num_parity_blocks, thres=1):
assert (len(A._block_idxs(0)) % num_parity_blocks == 0)
shard_size = A.shard_sizes[0]
coded_shape = (A.shape[0] + num_parity_blocks * A.shard_sizes[0],
A.shape[1])
coding_length = int(np.ceil(len(A._block_idxs(0)) / num_parity_blocks))
coding_fn2D = make_coding_function2D(A, coding_length)
coded_2D_shape = (
A.shape[0] +
(coding_length + 1 + num_parity_blocks) * A.shard_sizes[0], A.shape[1])
A_coded_2D = matrix.BigMatrix(A.key + "CODED2D_{0}_{1}_{2}".format(
A.shape[0], shard_size, num_parity_blocks),
shape=coded_2D_shape,
shard_sizes=A.shard_sizes,
write_header=True,
parent_fn=coding_fn2D)
# if list(set(A_coded_2D.block_idxs_not_exist) - set(A.block_idxs_exist)) == []:
# return A_coded_2D
last_block = max(A._block_idxs(0))
columns = A_coded_2D._block_idxs(1)
rows = A_coded_2D._block_idxs(0)
to_read = []
blocks_exist = A_coded_2D.block_idxs_exist
for row in rows:
if (row <= last_block): continue
for column in columns:
if (row, column) in blocks_exist:
continue
else:
to_read.append((row, column))
print("Number of parity blocks", len(to_read))
num_parities_1D = coding_length * len(A._block_idxs(1))
to_read_phase1 = to_read[0:num_parities_1D]
to_read_phase2 = to_read[num_parities_1D:]
def get_block_wrapper(x):
A_coded_2D.get_block(*x)
return 0
#### For 2D ENCODING of A, uncomment
pwex = pywren.lambda_executor()
t_enc1 = time.time()
futures2 = pwex.map(get_block_wrapper, to_read_phase1)
result_count = 0
fs_dones = []
while (result_count < thres * len(to_read_phase1)):
fs_dones, fs_notdones = pywren.wait(futures2, 2)
result_count = len(fs_dones)
print(result_count)
time.sleep(3)
for f in fs_dones:
try:
f.result()
except Exception as e:
print(e)
pass
t_enc1 = time.time() - t_enc1
print("Encoding phase 1 time", t_enc1)
t_enc2 = time.time()
futures2 = pwex.map(get_block_wrapper, to_read_phase2)
result_count = 0
while (result_count < thres * len(to_read_phase2)):
fs_dones, fs_notdones = pywren.wait(futures2, 2)
result_count = len(fs_dones)
print(result_count)
time.sleep(3)
for f in fs_dones:
try:
f.result()
except Exception as e:
print(e)
pass
t_enc2 = time.time() - t_enc2
print("Encoding phase 2 time", t_enc2)
print("Total ENCODING time", t_enc1 + t_enc2)
# a = list(set(A_coded_2D.block_idxs_not_exist) - set(A.block_idxs_exist))
# print("Still to encode", a)
return A_coded_2D
# Get numpywren results for matrix multiplication:
import numpy as np
import time
from numpywren.matrix import BigMatrix
from numpywren.matrix_init import shard_matrix
from numpywren.binops import gemm
import pywren
pwex = pywren.lambda_executor()
Ns = [5000, 10000, 15000, 20000, 25000, 30000]
shard_size = (5000, 5000)
np.random.seed(42)
# Only run this if matrices not already in the bucket.
# This takes a very long time (for 30000x30000xf64 - 8GB of data)
# Big_X = BigMatrix("multiply_test2", shape=(max(Ns),max(Ns), shard_sizes=shard_size)
# for i in range():
# for j in range():
# X = np.random.randn(5000,5000)
# Big_X.put_block(X, i, j)
# start = time.time()
# gemm(pwex, Big_X, Big_X, Big_X.bucket, 1)
# end = time.time()
# print(end - start)
for N in Ns:
X_sharded = BigMatrix("multiply_test2",
shape=(N, N),
shard_sizes=shard_size)
