def doPCA_eig(self, data, blocks): def center(X): meanX = X.mean(axis=0)[np.newaxis, :] centeredX = X - meanX return (meanX, centeredX) (means, dataNew) = center(data) size = len(data) / blocks covs = [] psis = np.array([]) for k in range(blocks): dataBlock = dataNew[(k * blocks):((k + 1) * blocks), :] covs.append(1. / size * matrix_multiply(dataBlock.T, dataBlock)) w, v = np.linalg.eig(covs[k]) psi = matrix_multiply(v, (np.diag((size * w)**0.5))) if len(psis) == 0: psis = psi else: psis = np.hstack((psis, psi)) R = 1. / size * matrix_multiply(psis.T, psis) wR, vR = np.linalg.eig(R) wR = np.real(wR) vR = np.real(vR) inv_sqrt = np.diag((size * wR)**(-0.5)) vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt) idx = (-wR).argsort() print wR[idx], vT[:, idx].T return wR[idx], vT[:, idx].T
def doPCA_eig(self, data, blocks): def center(X): meanX = X.mean(axis = 0)[np.newaxis,:] centeredX = X - meanX return (meanX, centeredX) (means, dataNew) = center(data) size = len(data) / blocks covs = [] psis = np.array([]) for k in range(blocks): dataBlock = dataNew[(k * blocks):((k + 1) * blocks),:] covs.append(1./size * matrix_multiply(dataBlock.T, dataBlock)) w, v = np.linalg.eig(covs[k]) psi = matrix_multiply(v, (np.diag((size * w)**0.5))) if len(psis) == 0: psis = psi else: psis = np.hstack((psis, psi)) R = 1./size * matrix_multiply(psis.T, psis) wR, vR = np.linalg.eig(R) wR = np.real(wR) vR = np.real(vR) inv_sqrt = np.diag((size * wR)**(-0.5)) vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt) idx = (-wR).argsort() print wR[idx], vT[:,idx].T return wR[idx], vT[:,idx].T
def test_order_6(self): # given A = M6(1, 2, -1, 3, -5, 3, 3, 0, 2, 7, 1, 2, 5, 3, 3, 1, 4, 4, 2, 1, 2, 2, 3, 3, -2, 2, 6, 6, 9, 7, -2, 2, -4, 2, 6, 5) B = M6(1, 2, 7, 7, -1, 3, 2, 1, 2, 1, -1, 2, 3, 0, 5, 5, 6, 1, 3, 1, 4, 1, 6, 4, 2, 4, 3, 3, 3, 9, 1, 4, 3, 2, 1, 9) # when C = matrix_multiply(A, B) # then expected = M6(4, -1, 12, -2, -3, 0, 34, 25, 68, 45, 56, 66, 35, 46, 84, 74, 32, 100, 25, 31, 52, 42, 33, 72, 63, 68, 92, 65, 106,172, 13, 44, 11, -2, 11, 101) assert_that(C, is_(expected))
def mapper(self, k, line): print "mapper", k dataBlockRow = np.array([float(x) for x in line.split()]) size = len(dataBlockRow) / dimension dataBlock = dataBlockRow.reshape((size, dimension)) print "doing multiplication" cov = 1./size * matrix_multiply(dataBlock.T, dataBlock) print "done with mutiplication" yield None, cov
def mapper(self, k, line): print "mapper", k dataBlockRow = np.array([float(x) for x in line.split()]) size = len(dataBlockRow) / dimension dataBlock = dataBlockRow.reshape((size, dimension)) print "doing multiplication" cov = 1. / size * matrix_multiply(dataBlock.T, dataBlock) print "done with mutiplication" yield None, cov
def test_order_1(self): # given A = Cannon.Matrix(1, [3]) B = Cannon.Matrix(1, [4]) # when C = matrix_multiply(A, B) # then assert_that(C, is_(Cannon.Matrix(1, [12])))
def test_1_order_2(self): # given A = M2(1, 2, 3, 4) B = M2(3, 5, 1, 0) # when C = matrix_multiply(A, B) # then assert_that(C, is_(M2(5, 5, 13, 15)))
def test_2_order_2(self): # given A = M2(1, 2, 3, 4) B = M2(5, 6, 7, 8) # when C = matrix_multiply(A, B) # then assert_that(C, is_(M2(19, 22, 43, 50)))
def test_order_2_floats(self): # given A = M2(1.1, 2.3, 3.5, 4.6) B = M2(3.7, 5.1, 1.5, 1.0 / 3) # when C = matrix_multiply(A, B) # then assert_that(C.ncols, is_(2)) assert_array_almost_equal(C.data, [7.52, 6.37, 19.85, 19.38], decimal=2)
def mapper(self, _, line): start_time_y = time.time() dataBlockRow = np.array([float(x) for x in line.split()]) size = len(dataBlockRow) / dimension dataBlock = dataBlockRow.reshape((size, dimension)) if SMART: cov = 1. / size * matrix_multiply(dataBlock, dataBlock.T) w, v = np.linalg.eig(cov) v = matrix_multiply(dataBlock.T, v) else: cov = 1. / size * matrix_multiply(dataBlock.T, dataBlock) w, v = np.linalg.eig(cov) num_eigens = 10 idx = (-w).argsort() w = w[idx[:num_eigens]] v = v[:, idx[:num_eigens]] psi = matrix_multiply(v, (np.diag((per_block * w)**0.5))) end_time_y = time.time() print "MAP TIME ", end_time_y - start_time_y yield None, psi
def mapper(self, _, line): start_time_y = time.time() dataBlockRow = np.array([float(x) for x in line.split()]) size = len(dataBlockRow) / dimension dataBlock = dataBlockRow.reshape((size, dimension)) if SMART: cov = 1./size * matrix_multiply(dataBlock, dataBlock.T) w, v = np.linalg.eig(cov) v = matrix_multiply(dataBlock.T, v) else: cov = 1./size * matrix_multiply(dataBlock.T, dataBlock) w, v = np.linalg.eig(cov) num_eigens = 10 idx = (-w).argsort() w = w[idx[:num_eigens]] v = v[:,idx[:num_eigens]] psi = matrix_multiply(v, (np.diag((per_block * w)**0.5))) end_time_y = time.time() print "MAP TIME ", end_time_y - start_time_y yield None, psi
def Sum(self, A, B, step): self.C = matrix_add(self.C, matrix_multiply(A, B)) self.g_step = self.g_step + 1 if self.g_step == self.g_order: if step != (self.g_order - 1): self.g_left.begin_injectFirst(A, (step + 1)) self.g_above.begin_injectSecond(B, (step + 1)) self.collector.injectSubmatrix(self.C, self.g_row, self.g_col) else: if step != (self.g_order - 1): self.g_left.begin_injectFirst(A, (step + 1)) self.g_above.begin_injectSecond(B, (step + 1))
def Sum(self, A, B, step): self.C = matrix_add(self.C, matrix_multiply(A, B)) self.g_step = self.g_step + 1 if self.g_step == self.g_order: if step != (self.g_order - 1): self.g_left.injectFirst(A, (step + 1)) self.g_above.injectSecond(B, (step + 1)) self.collector.injectSubmatrix(self.C, self.g_row, self.g_col) else: if step != (self.g_order - 1): self.g_left.injectFirst(A, (step + 1)) self.g_above.injectSecond(B, (step + 1))
def reducer(self, _, values): start_time_x = time.time() psis = np.array([]) for psi in values: if len(psis) == 0: psis = psi else: psis = np.hstack((psis, psi)) R = 1. / per_block * matrix_multiply(psis.T, psis) wR, vR = np.linalg.eig(R) wR = np.real(wR) vR = np.real(vR) wR[wR <= 0] = 1e-20 inv_sqrt = np.diag((per_block * wR)**(-0.5)) vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt) num_final_eigens = 50 idx = (-wR).argsort() end_time_x = time.time() print vT, idx print "reduce time ", end_time_x - start_time_x yield None, 0
def reducer(self, _, values): start_time_x = time.time() psis = np.array([]) for psi in values: if len(psis) == 0: psis = psi else: psis = np.hstack((psis, psi)) R = 1. / per_block * matrix_multiply(psis.T, psis) wR, vR = np.linalg.eig(R) wR = np.real(wR) vR = np.real(vR) wR[wR <= 0] = 1e-20 inv_sqrt = np.diag((per_block * wR)**(-0.5)) vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt) num_final_eigens = 50 idx = (-wR).argsort() end_time_x = time.time() print vT, idx print "reduce time ",end_time_x - start_time_x yield None, 0
def test_order_2_floats(self): # given A = M2(1.1, 2.3, 3.5, 4.6) B = M2(3.7, 5.1, 1.5, 1.0 / 3) # when C = matrix_multiply(A, B) # then assert_that(C.ncols, is_(2)) assert_array_almost_equal(C.data, [ 7.52, 6.37, 19.85, 19.38], decimal=2)
def test_5x5_processors_200x200_operands(self): nprocs = 25 # given P = [self.broker.add_servant(ProcessorI(), Cannon.ProcessorPrx) for i in range(nprocs)] loader = self.broker.add_servant(OperationsI(P), Cannon.OperationsPrx) A_last = 200 * 200 A = Cannon.Matrix(200, range(1, 1 + A_last)) B = Cannon.Matrix(200, range(1 + A_last, 1 + A_last * 2)) # when C = loader.matrixMultiply(A, B) # then expected = matrix_multiply(A, B) assert_that(C, is_(expected))
class Client(Ice.Application): def run(self, argv): t_dist = 0; t_secu = 0; loader = self.string_to_proxy(argv[1], Cannon.OperationsPrx) example = argv[2] A = load_matrix_from_file('m/{}A'.format(example)) B = load_matrix_from_file('m/{}B'.format(example)) t_dist = time.time() C = loader.matrixMultiply(A, B) t_dist = time.time() - t_dist t_secu = time.time() c = matrix_multiply(A,B) t_secu = time.time() - t_secu
def test_order_6(self): # given A = M6(1, 2, -1, 3, -5, 3, 3, 0, 2, 7, 1, 2, 5, 3, 3, 1, 4, 4, 2, 1, 2, 2, 3, 3, -2, 2, 6, 6, 9, 7, -2, 2, -4, 2, 6, 5) B = M6(1, 2, 7, 7, -1, 3, 2, 1, 2, 1, -1, 2, 3, 0, 5, 5, 6, 1, 3, 1, 4, 1, 6, 4, 2, 4, 3, 3, 3, 9, 1, 4, 3, 2, 1, 9) # when C = matrix_multiply(A, B) # then expected = M6(4, -1, 12, -2, -3, 0, 34, 25, 68, 45, 56, 66, 35, 46, 84, 74, 32, 100, 25, 31, 52, 42, 33, 72, 63, 68, 92, 65, 106, 172, 13, 44, 11, -2, 11, 101) assert_that(C, is_(expected))
def doPCA_cov(self, data, blocks): def center(X): meanX = X.mean(axis = 0)[np.newaxis,:] centeredX = X - meanX return (meanX, centeredX) (means, dataNew) = center(data) size = len(data) / blocks print size covs = [] for k in range(blocks): dataBlock = dataNew[(k * blocks):((k + 1) * blocks),:] covs.append(1./size * matrix_multiply(dataBlock.T, dataBlock)) print covs[0] acc = np.zeros(covs[0].shape) for cov in covs: acc += (1. / blocks) * cov w, v = np.linalg.eig(cov) return (w, np.transpose(v))
def doPCA_cov(self, data, blocks): def center(X): meanX = X.mean(axis=0)[np.newaxis, :] centeredX = X - meanX return (meanX, centeredX) (means, dataNew) = center(data) size = len(data) / blocks print size covs = [] for k in range(blocks): dataBlock = dataNew[(k * blocks):((k + 1) * blocks), :] covs.append(1. / size * matrix_multiply(dataBlock.T, dataBlock)) print covs[0] acc = np.zeros(covs[0].shape) for cov in covs: acc += (1. / blocks) * cov w, v = np.linalg.eig(cov) return (w, np.transpose(v))
def matrix_check(self,step, current=None): current_A=self.matrixA[step] current_B=self.matrixB[step] order_auxiliar=self.order multiplicacion=matrix_multiply(current_A, current_B) self.step_aux += 1 if self.step_aux == 1: self.resultado=multiplicacion if self.step_aux != 1: self.resultado=matrix_add(self.resultado,multiplicacion) if step < order_auxiliar-1: step += 1 self.left.injectA(current_A, step) self.above.injectB(current_B, step) if self.step_aux == order_auxiliar: self.target.inject(self.index,self.resultado)
def do_pca(self, orig_data): data = copy.copy(orig_data) # Step 1: Make all data mean 0 data = np.transpose(data) for row in data: mean = float(sum(row)) / len(row) for i in range(0, len(row)): row[i] = row[i] - mean x = data print "getting covariance" print data.shape covariance_matrix = matrix_multiply(data, np.transpose(data)) print "got covariance" print covariance_matrix.shape [eigenvalues,eigenvectors] = la.eig(covariance_matrix) eigenvectors = np.transpose(np.real(eigenvectors)) eigenvalues = np.real(eigenvalues) print "got eigenvectors" # Sort by eigenvalue combined = [] for i in range(0, len(eigenvalues)): combined.append((eigenvectors[i], eigenvalues[i])) combined.sort(key=lambda c: c[1], reverse=True) eigenvectors = [] eigenvalues = [] numFinalEigens = 100 for i in range(0, numFinalEigens): eigenvectors.append(combined[i][0]) eigenvalues.append(combined[i][1]) eigenvectors = np.array(eigenvectors) # utils.reconstruct_images(eigenvectors, orig_data) # utils.calc_error(eigenvectors, orig_data) utils.save_eigenvectors(eigenvectors)
def do_pca(self, orig_data): data = copy.copy(orig_data) # Step 1: Make all data mean 0 data = np.transpose(data) for row in data: mean = float(sum(row)) / len(row) for i in range(0, len(row)): row[i] = row[i] - mean x = data print "getting covariance" print data.shape covariance_matrix = matrix_multiply(data, np.transpose(data)) print "got covariance" print covariance_matrix.shape [eigenvalues, eigenvectors] = la.eig(covariance_matrix) eigenvectors = np.transpose(np.real(eigenvectors)) eigenvalues = np.real(eigenvalues) print "got eigenvectors" # Sort by eigenvalue combined = [] for i in range(0, len(eigenvalues)): combined.append((eigenvectors[i], eigenvalues[i])) combined.sort(key=lambda c: c[1], reverse=True) eigenvectors = [] eigenvalues = [] numFinalEigens = 100 for i in range(0, numFinalEigens): eigenvectors.append(combined[i][0]) eigenvalues.append(combined[i][1]) eigenvectors = np.array(eigenvectors) # utils.reconstruct_images(eigenvectors, orig_data) # utils.calc_error(eigenvectors, orig_data) utils.save_eigenvectors(eigenvectors)
class cliente(Ice.Application): def run(self, argv): prxFrontend = self.communicator().stringToProxy(argv[1]) frontend = Cannon.FrontendPrx.checkedCast(prxFrontend) if not frontend: raise RuntimeError("Invalid proxy frontend") A_last = 400 * 400 A400 = Cannon.Matrix(400, range(1, 1 + A_last)) B400 = Cannon.Matrix(400, range(1 + A_last, 1 + A_last * 2)) res400=frontend.multiply(A400,B400) res400dir=matrix_multiply(A400,B400) if(res400 == res400dir): print("Ok en matrix 400x400") print_matrix(res400) else: print("Error en matrix 400x400") A_last = 600 * 600
def try_multiply(self): current_A = self.A_blocks[self.current_step] current_B = self.B_blocks[self.current_step] if current_A is None or current_B is None: return if self.current_step == 0: self.result_parcial = Cannon.Matrix(current_A.ncols, [0] * (current_A.ncols**2)) product = matrix_multiply(current_A, current_B) self.result_parcial = matrix_add(self.result_parcial, product) self.current_step += 1 if self.current_step == self.order: self.target.injectSubmatrix(self.result_parcial, self.row, self.col) return self.left.injectFirst(current_A, self.current_step) self.above.injectSecond(current_B, self.current_step) self.try_multiply()
def doPCA_eig_smart(self, data, blocks): def center(X): meanX = X.mean(axis = 0)[np.newaxis,:] centeredX = X - meanX return (meanX, centeredX) (means, dataNew) = center(data) size = len(data) / blocks covs = [] psis = np.array([]) for k in range(blocks): dataBlock = dataNew[(k * size):((k + 1) * size),:] # do the inner product instead of outer product covs.append(1./size * matrix_multiply(dataBlock, dataBlock.T)) w, v = np.linalg.eig(covs[k]) v = matrix_multiply(dataBlock.T, v) idx = (-w).argsort() num_eigens = 10 w = w[idx[:num_eigens]] v = v[:,idx[:num_eigens]] psi = matrix_multiply(v, (np.diag((size * w)**0.5))) if len(psis) == 0: psis = psi else: psis = np.hstack((psis, psi)) R = 1./size * matrix_multiply(psis.T, psis) wR, vR = np.linalg.eig(R) inv_sqrt = np.diag((size * wR)**(-0.5)) vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt) idx = (-wR).argsort() print wR[idx], vT[:,idx[:20]].T utils.calc_error(vT[:,idx[:20]].T, data) return wR[idx], vT[:,idx[:20]].T
def doPCA_eig_smart(self, data, blocks): def center(X): meanX = X.mean(axis=0)[np.newaxis, :] centeredX = X - meanX return (meanX, centeredX) (means, dataNew) = center(data) size = len(data) / blocks covs = [] psis = np.array([]) for k in range(blocks): dataBlock = dataNew[(k * size):((k + 1) * size), :] # do the inner product instead of outer product covs.append(1. / size * matrix_multiply(dataBlock, dataBlock.T)) w, v = np.linalg.eig(covs[k]) v = matrix_multiply(dataBlock.T, v) idx = (-w).argsort() num_eigens = 10 w = w[idx[:num_eigens]] v = v[:, idx[:num_eigens]] psi = matrix_multiply(v, (np.diag((size * w)**0.5))) if len(psis) == 0: psis = psi else: psis = np.hstack((psis, psi)) R = 1. / size * matrix_multiply(psis.T, psis) wR, vR = np.linalg.eig(R) inv_sqrt = np.diag((size * wR)**(-0.5)) vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt) idx = (-wR).argsort() print wR[idx], vT[:, idx[:20]].T utils.calc_error(vT[:, idx[:20]].T, data) return wR[idx], vT[:, idx[:20]].T
res400=frontend.multiply(A400,B400) res400dir=matrix_multiply(A400,B400) if(res400 == res400dir): print("Ok en matrix 400x400") print_matrix(res400) else: print("Error en matrix 400x400") A_last = 600 * 600 A600 = Cannon.Matrix(600, range(1, 1 + A_last)) B600 = Cannon.Matrix(600, range(1 + A_last, 1 + A_last * 2)) res600=frontend.multiply(A600,B600) res600dir=matrix_multiply(A600,B600) if(res600 == res600dir): print("Ok en matrix 600x600") print_matrix(res600) else: print("Error en matrix 600x600") A_last = 1000 * 1000 A1000 = Cannon.Matrix(600, range(1, 1 + A_last)) B1000 = Cannon.Matrix(600, range(1 + A_last, 1 + A_last * 2)) res1000=frontend.multiply(A1000,B1000) res1000dir=matrix_multiply(A1000,B1000) if(res1000 == res1000dir): print("Ok en matrix 1000x1000")