def test_version_mismatch(self): my_version_str = pywren.runtime.version_str(sys.version_info) for supported_version in pywren.wrenconfig.default_runtime.keys(): if my_version_str != supported_version: wrong_version = supported_version config = pywren.wrenconfig.default() config['runtime']['s3_key'] = pywren.wrenconfig.default_runtime[wrong_version] with pytest.raises(Exception) as excinfo: pywren.lambda_executor(config) assert 'python version' in str(excinfo.value)
def benchmark(loopcount, workers, matn, verbose=False): t1 = time.time() N = workers iters = np.arange(N) def f(x): return {'flops': compute_flops(loopcount, matn)} pwex = pywren.lambda_executor(shard_runtime=True) futures = pwex.map(f, iters) print("invocation done, dur=", time.time() - t1) print("callset id: ", futures[0].callset_id) local_jobs_done_timeline = [] result_count = 0 while result_count < N: fs_dones, fs_notdones = pywren.wait(futures, pywren.wren.ALWAYS) result_count = len(fs_dones) local_jobs_done_timeline.append((time.time(), result_count)) est_flop = 2 * result_count * loopcount * matn**3 est_gflops = est_flop / 1e9 / (time.time() - t1) if verbose: print("jobs done: {:5d} runtime: {:5.1f}s {:8.1f} GFLOPS ". format(result_count, time.time() - t1, est_gflops)) if result_count == N: break time.sleep(1) if verbose: print("getting results") results = [f.result(throw_except=False) for f in futures] if verbose: print("getting status") run_statuses = [f.run_status for f in futures] invoke_statuses = [f.invoke_status for f in futures] all_done = time.time() total_time = all_done - t1 print("total time", total_time) est_flop = result_count * 2 * loopcount * matn**3 print est_flop / 1e9 / total_time, "GFLOPS" res = { 'total_time': total_time, 'est_flop': est_flop, 'run_statuses': run_statuses, 'invoke_statuses': invoke_statuses, 'callset_id': futures[0].callset_id, 'local_jobs_done_timeline': local_jobs_done_timeline, 'results': results } return res
def gemm_recompute(A, B, thresh, s3_key): """ Compute A * B.T via speculative execution (i.e., recompute straggling workers). Params ====== A : numpywren.matrix.BigMatrix First input matrix. B : numpywren.matrix.BigMatrix Second input matrix. thresh : float (in [0, 1]) Fraction of workers that should finish before recomputing. s3_key : str Storage key for output matrix. Returns ======= C : matrix.BigMatrix Resultant matrix product. t_comp : float Time for thresh percentage of the workers to finish. t_straggle : float Time for the remaining 1 - thresh percentage of the workers to finish after we begin recomputing. """ if not (0 <= thresh <= 1): raise ValueError("thresh must be in the interval [0, 1]") """Initialize output matrix""" num_col_blocks = A.shape[1] // A.shard_sizes[1] shard_sizes = (A.shard_sizes[0], B.shard_sizes[0]) C = matrix.BigMatrix(s3_key, shape=(A.shape[0], B.shape[0]), shard_sizes=shard_sizes, autosqueeze=False, write_header=True) C.delete() # Only needed if you reuse the same s3_key (if the blocks already exist, no work will be done here) """Stage 1: Compute "thresh" percentage of the results""" t_comp_start = time.time() pwex = pywren.lambda_executor() futures, num_done = pwex.map(lambda x: pywren_gemm(x, A, B, C, num_col_blocks), C.block_idxs), 0 while num_done < thresh * len(futures): fs_dones, _ = pywren.wait(futures, return_when=ANY_COMPLETED) num_done = len(fs_dones) t_comp = time.time() - t_comp_start # Total stage 1 time """Stage 2: Recompute straggling workers (the last 1-thresh percent of jobs)""" t_straggle_start = time.time() futures_stragglers = pwex.map(lambda x: pywren_gemm(x, A, B, C, num_col_blocks), C.block_idxs_not_exist) while len(C.block_idxs_not_exist) > 0: pywren.wait(futures, return_when=ALWAYS) pywren.wait(futures_stragglers, return_when=ALWAYS) t_straggle = time.time() - t_straggle_start # Total stage 2 time return C, t_comp, t_straggle
def test_single_shard_matrix_multiply(self): X = np.random.randn(16, 16) X_sharded = BigMatrix("gemm_test_0", shape=X.shape, shard_sizes=X.shape) shard_matrix(X_sharded, X) pwex = pywren.lambda_executor() XXT_sharded = binops.gemm(pwex, X_sharded, X_sharded.T, X_sharded.bucket, 1) XXT_sharded_local = XXT_sharded.numpy() XXT = X.dot(X.T) X_sharded.free() XXT_sharded.free() assert (np.all(np.isclose(XXT, XXT_sharded_local))) os.system("rm -rf /dev/shm/*")
def coded_mat_vec_mul(A_coded_2D, x, num_parity_blocks, coding_length): def coded_mat_vec_mul(id): shard_size = A_coded_2D.shard_sizes[1] reduce_idxs = A_coded_2D._block_idxs(axis=1) x_loc = x.get_block(0, 0) Ax_block = None for r in reduce_idxs: block1 = A_coded_2D.get_block(id, r) sidx = r * shard_size eidx = (r + 1) * shard_size x_block = x_loc[sidx:eidx] if (Ax_block is None): Ax_block = block1.dot(x_block) else: Ax_block = Ax_block + block1.dot(x_block) return Ax_block shard_size = A_coded_2D.shard_sizes[0] n_coded_procs = len(A_coded_2D._block_idxs(0)) len_A_coded = n_coded_procs - coding_length - 1 pwex = pywren.lambda_executor() futures = pwex.map(coded_mat_vec_mul, range(n_coded_procs)) Ax = np.zeros((A_coded_2D.shape[0], 1)) bitmask = np.ones((coding_length + 1, num_parity_blocks + 1)) not_done = list(range(n_coded_procs)) while decode2D.cant_be_decoded_with_bitmask(deepcopy(bitmask)): fs_dones, fs_not_dones = pywren.wait(futures, 2) for (id, f) in enumerate(futures): if f in fs_dones and id in not_done: # print("Worker done", id) try: Ax[id * shard_size:(id + 1) * shard_size] = f.result() i, j = decode2D.ind1Dto2D(id, len_A_coded, num_parity_blocks) bitmask[i, j] = 0 not_done.remove(id) except Exception as e: print(e) pass # print("1: Decoding not dones", not_done) Ax = decode2D.decode_vector_with_bitmask(Ax, bitmask, num_parity_blocks, len_A_coded, shard_size, coding_length) return Ax
def test_multiple_shard_matrix_multiply_symmetric_2(self): X = np.random.randn(16, 16) shard_sizes = [8, 16] X_sharded = BigMatrix("gemm_test_1", shape=X.shape, shard_sizes=shard_sizes) shard_matrix(X_sharded, X) pwex = pywren.lambda_executor() XTX_sharded = binops.gemm(pwex, X_sharded.T, X_sharded, X_sharded.bucket, 1, local=True) XTX_sharded_local = XTX_sharded.numpy() XTX = X.T.dot(X) X_sharded.free() XTX_sharded.free() assert (np.all(np.isclose(XTX, XTX_sharded_local))) os.system("rm -rf /dev/shm/*")
def test_multiple_shard_matrix_multiply(self): X = np.random.randn(16, 16) Y = np.random.randn(16, 16) shard_sizes = tuple(map(int, np.array(X.shape) / 2)) X_sharded = BigMatrix("gemm_test_1", shape=X.shape, shard_sizes=shard_sizes) Y_sharded = BigMatrix("gemm_test_2", shape=X.shape, shard_sizes=shard_sizes) shard_matrix(X_sharded, X) shard_matrix(Y_sharded, Y) pwex = pywren.lambda_executor() XY_sharded = binops.gemm(pwex, X_sharded, Y_sharded, X_sharded.bucket, 1) XY_sharded_local = XY_sharded.numpy() XY = X.dot(Y) X_sharded.free() Y_sharded.free() XY_sharded.free() assert (np.all(np.isclose(XY, XY_sharded_local))) os.system("rm -rf /dev/shm/*")
def recompute_mat_vec_mul(A_coded_2D, x, thres=0.95): def shard_mat_vec_mul(id): shard_size = A_coded_2D.shard_sizes[1] reduce_idxs = A_coded_2D._block_idxs(axis=1) x_loc = x.get_block(0, 0) Ax_block = None for r in reduce_idxs: block1 = A_coded_2D.get_block(id, r) sidx = r * shard_size eidx = (r + 1) * shard_size x_block = x_loc[sidx:eidx] if (Ax_block is None): Ax_block = block1.dot(x_block) else: Ax_block = Ax_block + block1.dot(x_block) return Ax_block, id shard_size = A_coded_2D.shard_sizes[0] n_coded_procs = len(A_coded_2D._block_idxs(0)) pwex = pywren.lambda_executor() futures = pwex.map(shard_mat_vec_mul, range(n_coded_procs)) Ax = np.zeros((A_coded_2D.shape[0], 1)) not_done = list(range(n_coded_procs)) fs_dones = [] f_result_dones = [] while len(fs_dones) < thres * n_coded_procs: fs_dones, fs_not_dones = pywren.wait(futures, 2) for f in list(set(fs_dones) - set(f_result_dones)): # print("Worker done", id) f_result_dones.append(f) try: result = f.result() id = result[1] Ax[id * shard_size:(id + 1) * shard_size] = result[0] not_done.remove(id) except Exception as e: #print(e) pass time.sleep(2) print("Recomputing not dones", not_done) futures2 = pwex.map(shard_mat_vec_mul, not_done) f_result_dones2 = [] while not_done != []: fs_dones2, fs_not_dones2 = pywren.wait(futures2, 3) for f in list(set(fs_dones2) - set(f_result_dones2)): f_result_dones2.append(f) try: result = f.result() id = result[1] if id in not_done: print("Recomputed", id) Ax[id * shard_size:(id + 1) * shard_size] = result[0] not_done.remove(id) except Exception as e: #print(e) pass time.sleep(2) fs_dones, fs_not_dones = pywren.wait(futures, 3) for f in list(set(fs_dones) - set(f_result_dones)): f_result_dones.append(f) try: result = f.result() id = result[1] if id in not_done: print("Straggler computed", id) Ax[id * shard_size:(id + 1) * shard_size] = result[0] not_done.remove(id) except Exception as e: #print(e) pass time.sleep(2) if fs_not_dones2 == [] and fs_not_dones == []: print("NOT DONE", not_done) break print("Recomputing done") return Ax
def start_encode_mtx(M, blocks_per_parity, s3_key): """ Apply a (blocks_per_parity + 1, blocks_per_parity) MDS code to the matrix M every blocks_per_parity rows by summing up the previous blocks_per_parity rows. Params ====== M : numpywren.matrix.BigMatrix The matrix to encode. blocks_per_parity : int The number of input blocks sum up in creating each parity block. Note that as this number increases, less redundancy is provided. s3_key : str The storage key for Amazon S3. Returns ======= M_coded : numpywren.matrix.BigMatrix The encoded matrix. futures : list List of the pywren futures. num_workers : int The number of workers. """ # Useful definitions num_row_blocks = M.shape[0] // M.shard_sizes[0] num_col_blocks = M.shape[1] // M.shard_sizes[1] num_parity = num_row_blocks // blocks_per_parity # total number of parity blocks that will be added coded_shape = (M.shape[0] + num_parity * M.shard_sizes[0], M.shape[1]) # Ensure no blocks will go uncoded if not num_row_blocks % blocks_per_parity == 0: raise ValueError("Number of row blocks ({0}) is not divisible \ by number of blocks per parity ({1})".format( num_row_blocks, blocks_per_parity)) # Create the coded matrix object coding_fn = make_coding_function(M, blocks_per_parity) M_coded = matrix.BigMatrix(s3_key, shape=coded_shape, shard_sizes=M.shard_sizes, write_header=True, parent_fn=coding_fn) M_coded.delete( ) # Only needed if you reuse the same s3_key (if the blocks already exist, no work will be done here) # Generate encoding indices encode_idx = [] for j in range(num_col_blocks): for i in range(1, num_parity + 1): encode_idx.append((i * (blocks_per_parity + 1) - 1, j)) num_workers = len(encode_idx) # Encode the matrix pwex = pywren.lambda_executor() futures = pwex.map(lambda x: get_block_wrapper(M_coded, x), encode_idx) return M_coded, futures, num_workers
def setUp(self): self.lambda_exec = pywren.lambda_executor() self.remote_exec = pywren.remote_executor()
import pywren if __name__ == "__main__": lambda_exec = pywren.lambda_executor() remote_exec = pywren.remote_executor() def increment(x): return x + 1 x = [1, 2, 3, 4] futures = lambda_exec.map(increment, x) def reduce_func(x): return sum(x) reduce_future = remote_exec.reduce(reduce_func, futures) print reduce_future.result()
if sys.version_info.major == python_ver_major \ and sys.version_info.minor == python_ver_minor: print("running runtime config {}".format(runtime_name)) # create an executor config = pywren.wrenconfig.default() staged_runtime_url, staged_meta_url = runtimes.get_staged_runtime_url( runtime_name, python_ver) assert staged_runtime_url[:5] == "s3://" splits = staged_runtime_url[5:].split("/") bucket = splits[0] key = "/".join(splits[1:]) config['runtime']['bucket'] = bucket config['runtime']['s3_key'] = key print("running with bucket={} key={}".format(bucket, key)) wrenexec = pywren.lambda_executor(config) def import_check(x): results = {} conda_results = {} for pkg in packages['conda_install']: if pkg in runtimes.CONDA_TEST_STRS: test_str = runtimes.CONDA_TEST_STRS[pkg] try: eval(test_str) conda_results[pkg] = True except ImportError: conda_results[pkg] = False results['conda'] = conda_results
def OverSketchFunc(A, B, d, thres = 0.95): m = A.shape[0] n = A.shape[1] l = B.shape[1] b = A.shard_sizes[0] assert (d % b == 0) assert (m % b == 0) assert (l % b == 0) assert (b == B.shard_sizes[1]) N = int(d/b) sketch_A = matrix.BigMatrix("sketch_A_{0}_{1}".format(m, d), shape=(m, d), shard_sizes=(b, b)) sketch_BT = matrix.BigMatrix("sketch_B_{0}_{1}".format(l, d), shape=(l, d), shard_sizes=(b, b)) hashes = np.random.randint(0, b, size=(N, n)) flips = np.random.choice([-1,1], size=(N, n)) def OverSketchMatrix(id, X, hashes, flips, b, sketch): """ Calculates OverSketch AS for a row-block of a fat matrix A with block-size b """ x = id[0] y = id[1] A = X.get_block(x,0) m,n = A.shape hash_local = hashes[y,:] flip_local = flips[y,:] sketch_block = np.zeros((m, b)) for i in range(n): sketch_block[:, hash_local[i]] += flip_local[i]*A[:,i] sketch.put_block(sketch_block, x, y) return 0 pwex = pywren.lambda_executor() t1 = time.time() futuresA = pwex.map(lambda x: OverSketchMatrix(x, A, hashes, flips, b, sketch_A), sketch_A.block_idxs) futuresB = pwex.map(lambda x: OverSketchMatrix(x, B.T, hashes, flips, b, sketch_BT), sketch_BT.block_idxs) fs_donesA = pywren.wait(futuresA, 2)[0] fs_donesB = pywren.wait(futuresB, 2)[0] while len(fs_donesA)<thres*len(futuresA) and len(fs_donesB)<thres*len(futuresB): fs_donesA = pywren.wait(futuresA, 2)[0] fs_donesB = pywren.wait(futuresB, 2)[0] print("Sketching time", time.time() - t1) ## Computation phase def blockMatMul(A, B, tensorAB, id): """ Multiplies A and B.T in a blocked fashion """ i = id[0] j = id[1] k = id[2] X = A.get_block(i,k) Y = B.get_block(j,k) tensorAB[k].put_block(X.dot(Y.T), i, j) return 0 tensorAB = [] for x in range(N): tensorAB.append(matrix.BigMatrix("AxB_outer_{0}_{1}_{2}".format(m, l, x), shape=(m, l), shard_sizes=(b, b))) computeArr = [(i,j,k) for (i,k) in sketch_A.block_idxs for j in sketch_BT._block_idxs(0)] t1 = time.time() futures = pwex.map(lambda x: blockMatMul(sketch_A, sketch_BT, tensorAB, x), computeArr) fs_dones = pywren.wait(futures, 2)[0] while len(fs_dones)<thres*len(futures): fs_dones = pywren.wait(futures, 2)[0] print("Computation time", time.time() - t1) ## Reduction phase def blockMatMulReduction(tensorAB, AB, id): """ Reduces the output from computation phase to get A*B Variable 'count' keeps track of number of blocks that have returned """ i = id[0] j = id[1] X = None count = 1 for k in range(N): if X is None: try: X = tensorAB[k].get_block(i,j) except Exception as e: print(e) pass else: try: X = X + tensorAB[k].get_block(i,j) count = count+1 except Exception as e: print(e) pass AB.put_block(X/count, i, j) return 0 AB = matrix.BigMatrix("AxB_{0}_{1}".format(m, l), shape=(m, l), shard_sizes=(b, b)) reduceArr = [(i,j) for i in sketch_A._block_idxs(0) for j in sketch_BT._block_idxs(0)] t1 = time.time() futures_red = pwex.map(lambda x: blockMatMulReduction(tensorAB, AB, x), reduceArr) fs_dones = pywren.wait(futures_red)[0] print("Reduction time", time.time() - t1) return AB
def gemm_coded(A, B, blocks_per_parity, s3_key, completion_pct=.7, encode_A=True, encode_B=True, np_A=-1, np_B=-1): """ Compute A * B.T using a locally recoverable product code for redundancy. Params ====== A : numpywren.matrix.BigMatrix First input matrix. B : numpywren.matrix.BigMatrix Second input matrix. blocks_per_parity : int Number of blocks to sum up when creating each parity block. s3_key: str Storage key for output matrix. completion_pct: int The fraction of multiplication workers that must finish before moving on to decoding. encode_A : bool Whether or not A needs to be encoded. Allows for the user to pre-encode A if it will be used multiple times. encode_B : bool Whether or not B needs to be encoded. Allows for the user to pre-encode B if it will be used multiple times. np_A : int Number of parity blocks in the matrix A. Should be provided if and only if encode_A is set to false. np_B : int Number of parity blocks in the matrix B. Should be provided if and only if encode_B is set to false. Returns ======= C : numpywren.matrix.BigMatrix Resultant matrix product. t_enc : float Encoding time. t_comp : float Computation time. t_dec : float Decoding time. """ if (not encode_A) and np_A == -1: raise ValueError("You must provide the number of parity blocks in A if you pre-encoded it.") if (not encode_B) and np_B == -1: raise ValueError("You must provide the number of parity blocks in B if you pre-encoded it.") """Stage 1: Encoding""" start = time.time() if encode_A or encode_B: # Spin up encoding workers num_workers = 0 if encode_A: A_coded, futures_encode_A, num_workers_A = start_encode_mtx(A, blocks_per_parity, "A_coded") num_workers += num_workers_A if encode_B: B_coded, futures_encode_B, num_workers_B = start_encode_mtx(B, blocks_per_parity, "B_coded") num_workers += num_workers_B # Wait until enough encoding workers are done to move on num_done = 0 while num_done < MIN_ENCODING_COMPLETION_PCT * num_workers: fs_A, fs_B = [], [] if encode_A: fs_A, _ = pywren.wait(futures_encode_A, return_when=ANY_COMPLETED) if encode_B: fs_B, _ = pywren.wait(futures_encode_B, return_when=ANY_COMPLETED) num_done = len(fs_A) + len(fs_B) if not encode_A: A_coded = A if not encode_B: B_coded = B t_enc = time.time() - start # Total encoding time """Intermediate step: Initialize output matrix (untimed for consistency with gemm_recompute).""" # Determine coded dimensions of A, B if encode_A: num_parity_A = (A.shape[0] // A.shard_sizes[0]) // blocks_per_parity coded_shape_A = (A.shape[0] + num_parity_A * A.shard_sizes[0], A.shape[1]) else: num_parity_A = np_A coded_shape_A = A_coded.shape if encode_B: num_parity_B = (B.shape[0] // B.shard_sizes[0]) // blocks_per_parity coded_shape_B = (B.shape[0] + num_parity_B * B.shard_sizes[0], B.shape[1]) else: num_parity_B = np_B coded_shape_B = B_coded.shape # Create (encoded) output matrix shard_sizes_C = (A.shard_sizes[0], B.shard_sizes[0]) C_coded = matrix.BigMatrix(s3_key + "coded", shape=(A_coded.shape[0], B_coded.shape[0]), \ shard_sizes=shard_sizes_C, \ autosqueeze=False, \ write_header=True) C_coded.delete() # Only needed if you reuse the same s3_key (if the blocks already exist, no work will be done here) # Generate indices for the output matrix num_row_blocks_C = C_coded.shape[0] // C_coded.shard_sizes[0] num_col_blocks_C = C_coded.shape[1] // C_coded.shard_sizes[1] num_cols_coded = A_coded.shape[1] // A_coded.shard_sizes[1] # Inner dimension of the coded multiplication block_idx_C = C_coded.block_idxs num_workers = len(block_idx_C) np.random.shuffle(block_idx_C) # Randomize jobs to avoid bad straggler locality """Stage 2: Multiply""" t_comp_start = time.time() pwex = pywren.lambda_executor() futures_matmul = pwex.map(lambda x: pywren_gemm(x, A_coded, B_coded, C_coded, num_cols_coded), block_idx_C) fs_done_matmul, num_done = [], 0 while num_done < completion_pct * num_workers: fs_done_matmul, _ = pywren.wait(futures_matmul, return_when=ANY_COMPLETED) num_done = len(fs_done_matmul) t_comp = time.time() - t_comp_start # Total stage 2 time """Stage 3: Decoding""" t_dec_start = time.time() decode_idx = [(i, j) for i in range(num_parity_A) for j in range(num_parity_B)] num_workers = len(decode_idx) futures_decode = pwex.map(lambda x: decode_gemm(num_row_blocks_C, num_parity_A, C_coded, x), decode_idx) fs_done_decode, num_done = [], 0 while num_done < num_workers and len(C_coded.block_idxs_not_exist) > 0: fs_done_decode, _ = pywren.wait(futures_decode, return_when=ANY_COMPLETED) num_done = len(fs_done_decode) t_dec = time.time() - t_dec_start # Total stage 3 time """Final step: Specify the systematic part (i.e., all non-parity blocks) of the result""" # Determine output dimensions if encode_A: C_num_rows = A.shape[0] else: C_num_rows = A.shape[0] - np_A * A.shard_sizes[0] if encode_B: C_num_cols = B.shape[0] else: C_num_cols = B.shape[0] - np_B * B.shard_sizes[0] # Create the output matrix containing only the systematic part of the result get_systematic_part = systematicize(C_coded, blocks_per_parity) C_shard_sizes = (A.shard_sizes[0], B.shard_sizes[0]) C = matrix.BigMatrix(s3_key, shape=(C_num_rows, C_num_cols), shard_sizes=C_shard_sizes, parent_fn=get_systematic_part) C.delete() # Only needed if you reuse the same s3_key (if the blocks already exist, no work will be done here) return C, t_enc, t_comp, t_dec
def setUp(self): self.wrenexec = pywren.lambda_executor(job_max_runtime=40)
def code_2D(A, num_parity_blocks, thres=1): assert (len(A._block_idxs(0)) % num_parity_blocks == 0) shard_size = A.shard_sizes[0] coded_shape = (A.shape[0] + num_parity_blocks * A.shard_sizes[0], A.shape[1]) coding_length = int(np.ceil(len(A._block_idxs(0)) / num_parity_blocks)) coding_fn2D = make_coding_function2D(A, coding_length) coded_2D_shape = ( A.shape[0] + (coding_length + 1 + num_parity_blocks) * A.shard_sizes[0], A.shape[1]) A_coded_2D = matrix.BigMatrix(A.key + "CODED2D_{0}_{1}_{2}".format( A.shape[0], shard_size, num_parity_blocks), shape=coded_2D_shape, shard_sizes=A.shard_sizes, write_header=True, parent_fn=coding_fn2D) # if list(set(A_coded_2D.block_idxs_not_exist) - set(A.block_idxs_exist)) == []: # return A_coded_2D last_block = max(A._block_idxs(0)) columns = A_coded_2D._block_idxs(1) rows = A_coded_2D._block_idxs(0) to_read = [] blocks_exist = A_coded_2D.block_idxs_exist for row in rows: if (row <= last_block): continue for column in columns: if (row, column) in blocks_exist: continue else: to_read.append((row, column)) print("Number of parity blocks", len(to_read)) num_parities_1D = coding_length * len(A._block_idxs(1)) to_read_phase1 = to_read[0:num_parities_1D] to_read_phase2 = to_read[num_parities_1D:] def get_block_wrapper(x): A_coded_2D.get_block(*x) return 0 #### For 2D ENCODING of A, uncomment pwex = pywren.lambda_executor() t_enc1 = time.time() futures2 = pwex.map(get_block_wrapper, to_read_phase1) result_count = 0 fs_dones = [] while (result_count < thres * len(to_read_phase1)): fs_dones, fs_notdones = pywren.wait(futures2, 2) result_count = len(fs_dones) print(result_count) time.sleep(3) for f in fs_dones: try: f.result() except Exception as e: print(e) pass t_enc1 = time.time() - t_enc1 print("Encoding phase 1 time", t_enc1) t_enc2 = time.time() futures2 = pwex.map(get_block_wrapper, to_read_phase2) result_count = 0 while (result_count < thres * len(to_read_phase2)): fs_dones, fs_notdones = pywren.wait(futures2, 2) result_count = len(fs_dones) print(result_count) time.sleep(3) for f in fs_dones: try: f.result() except Exception as e: print(e) pass t_enc2 = time.time() - t_enc2 print("Encoding phase 2 time", t_enc2) print("Total ENCODING time", t_enc1 + t_enc2) # a = list(set(A_coded_2D.block_idxs_not_exist) - set(A.block_idxs_exist)) # print("Still to encode", a) return A_coded_2D
# Get numpywren results for matrix multiplication: import numpy as np import time from numpywren.matrix import BigMatrix from numpywren.matrix_init import shard_matrix from numpywren.binops import gemm import pywren pwex = pywren.lambda_executor() Ns = [5000, 10000, 15000, 20000, 25000, 30000] shard_size = (5000, 5000) np.random.seed(42) # Only run this if matrices not already in the bucket. # This takes a very long time (for 30000x30000xf64 - 8GB of data) # Big_X = BigMatrix("multiply_test2", shape=(max(Ns),max(Ns), shard_sizes=shard_size) # for i in range(): # for j in range(): # X = np.random.randn(5000,5000) # Big_X.put_block(X, i, j) # start = time.time() # gemm(pwex, Big_X, Big_X, Big_X.bucket, 1) # end = time.time() # print(end - start) for N in Ns: X_sharded = BigMatrix("multiply_test2", shape=(N, N), shard_sizes=shard_size)