def doPCA_eig(self, data, blocks):
def center(X):
meanX = X.mean(axis=0)[np.newaxis, :]
centeredX = X - meanX
return (meanX, centeredX)
(means, dataNew) = center(data)
size = len(data) / blocks
covs = []
psis = np.array([])
for k in range(blocks):
dataBlock = dataNew[(k * blocks):((k + 1) * blocks), :]
covs.append(1. / size * matrix_multiply(dataBlock.T, dataBlock))
w, v = np.linalg.eig(covs[k])
psi = matrix_multiply(v, (np.diag((size * w)**0.5)))
if len(psis) == 0:
psis = psi
else:
psis = np.hstack((psis, psi))
R = 1. / size * matrix_multiply(psis.T, psis)
wR, vR = np.linalg.eig(R)
wR = np.real(wR)
vR = np.real(vR)
inv_sqrt = np.diag((size * wR)**(-0.5))
vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt)
idx = (-wR).argsort()
print wR[idx], vT[:, idx].T
return wR[idx], vT[:, idx].T
def doPCA_eig(self, data, blocks):
def center(X):
meanX = X.mean(axis = 0)[np.newaxis,:]
centeredX = X - meanX
return (meanX, centeredX)
(means, dataNew) = center(data)
size = len(data) / blocks
covs = []
psis = np.array([])
for k in range(blocks):
dataBlock = dataNew[(k * blocks):((k + 1) * blocks),:]
covs.append(1./size * matrix_multiply(dataBlock.T, dataBlock))
w, v = np.linalg.eig(covs[k])
psi = matrix_multiply(v, (np.diag((size * w)**0.5)))
if len(psis) == 0:
psis = psi
else:
psis = np.hstack((psis, psi))
R = 1./size * matrix_multiply(psis.T, psis)
wR, vR = np.linalg.eig(R)
wR = np.real(wR)
vR = np.real(vR)
inv_sqrt = np.diag((size * wR)**(-0.5))
vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt)
idx = (-wR).argsort()
print wR[idx], vT[:,idx].T
return wR[idx], vT[:,idx].T
def test_order_6(self):
# given
A = M6(1, 2, -1, 3, -5, 3,
3, 0, 2, 7, 1, 2,
5, 3, 3, 1, 4, 4,
2, 1, 2, 2, 3, 3,
-2, 2, 6, 6, 9, 7,
-2, 2, -4, 2, 6, 5)
B = M6(1, 2, 7, 7, -1, 3,
2, 1, 2, 1, -1, 2,
3, 0, 5, 5, 6, 1,
3, 1, 4, 1, 6, 4,
2, 4, 3, 3, 3, 9,
1, 4, 3, 2, 1, 9)
# when
C = matrix_multiply(A, B)
# then
expected = M6(4, -1, 12, -2, -3, 0,
34, 25, 68, 45, 56, 66,
35, 46, 84, 74, 32, 100,
25, 31, 52, 42, 33, 72,
63, 68, 92, 65, 106,172,
13, 44, 11, -2, 11, 101)
assert_that(C, is_(expected))
def mapper(self, k, line): print "mapper", k dataBlockRow = np.array([float(x) for x in line.split()]) size = len(dataBlockRow) / dimension dataBlock = dataBlockRow.reshape((size, dimension)) print "doing multiplication" cov = 1./size * matrix_multiply(dataBlock.T, dataBlock) print "done with mutiplication" yield None, cov
def mapper(self, k, line):
print "mapper", k
dataBlockRow = np.array([float(x) for x in line.split()])
size = len(dataBlockRow) / dimension
dataBlock = dataBlockRow.reshape((size, dimension))
print "doing multiplication"
cov = 1. / size * matrix_multiply(dataBlock.T, dataBlock)
print "done with mutiplication"
yield None, cov
def test_order_1(self):
# given
A = Cannon.Matrix(1, [3])
B = Cannon.Matrix(1, [4])
# when
C = matrix_multiply(A, B)
# then
assert_that(C, is_(Cannon.Matrix(1, [12])))
def test_order_1(self):
# given
A = Cannon.Matrix(1, [3])
B = Cannon.Matrix(1, [4])
# when
C = matrix_multiply(A, B)
# then
assert_that(C, is_(Cannon.Matrix(1, [12])))
def test_1_order_2(self):
# given
A = M2(1, 2, 3, 4)
B = M2(3, 5, 1, 0)
# when
C = matrix_multiply(A, B)
# then
assert_that(C, is_(M2(5, 5, 13, 15)))
def test_2_order_2(self):
# given
A = M2(1, 2, 3, 4)
B = M2(5, 6, 7, 8)
# when
C = matrix_multiply(A, B)
# then
assert_that(C, is_(M2(19, 22, 43, 50)))
def test_order_2_floats(self):
# given
A = M2(1.1, 2.3, 3.5, 4.6)
B = M2(3.7, 5.1, 1.5, 1.0 / 3)
# when
C = matrix_multiply(A, B)
# then
assert_that(C.ncols, is_(2))
assert_array_almost_equal(C.data, [7.52, 6.37, 19.85, 19.38],
decimal=2)
def mapper(self, _, line):
start_time_y = time.time()
dataBlockRow = np.array([float(x) for x in line.split()])
size = len(dataBlockRow) / dimension
dataBlock = dataBlockRow.reshape((size, dimension))
if SMART:
cov = 1. / size * matrix_multiply(dataBlock, dataBlock.T)
w, v = np.linalg.eig(cov)
v = matrix_multiply(dataBlock.T, v)
else:
cov = 1. / size * matrix_multiply(dataBlock.T, dataBlock)
w, v = np.linalg.eig(cov)
num_eigens = 10
idx = (-w).argsort()
w = w[idx[:num_eigens]]
v = v[:, idx[:num_eigens]]
psi = matrix_multiply(v, (np.diag((per_block * w)**0.5)))
end_time_y = time.time()
print "MAP TIME ", end_time_y - start_time_y
yield None, psi
def mapper(self, _, line):
start_time_y = time.time()
dataBlockRow = np.array([float(x) for x in line.split()])
size = len(dataBlockRow) / dimension
dataBlock = dataBlockRow.reshape((size, dimension))
if SMART:
cov = 1./size * matrix_multiply(dataBlock, dataBlock.T)
w, v = np.linalg.eig(cov)
v = matrix_multiply(dataBlock.T, v)
else:
cov = 1./size * matrix_multiply(dataBlock.T, dataBlock)
w, v = np.linalg.eig(cov)
num_eigens = 10
idx = (-w).argsort()
w = w[idx[:num_eigens]]
v = v[:,idx[:num_eigens]]
psi = matrix_multiply(v, (np.diag((per_block * w)**0.5)))
end_time_y = time.time()
print "MAP TIME ", end_time_y - start_time_y
yield None, psi
def Sum(self, A, B, step):
self.C = matrix_add(self.C, matrix_multiply(A, B))
self.g_step = self.g_step + 1
if self.g_step == self.g_order:
if step != (self.g_order - 1):
self.g_left.begin_injectFirst(A, (step + 1))
self.g_above.begin_injectSecond(B, (step + 1))
self.collector.injectSubmatrix(self.C, self.g_row, self.g_col)
else:
if step != (self.g_order - 1):
self.g_left.begin_injectFirst(A, (step + 1))
self.g_above.begin_injectSecond(B, (step + 1))
def test_2_order_2(self):
# given
A = M2(1, 2,
3, 4)
B = M2(5, 6,
7, 8)
# when
C = matrix_multiply(A, B)
# then
assert_that(C, is_(M2(19, 22,
43, 50)))
def test_1_order_2(self):
# given
A = M2(1, 2,
3, 4)
B = M2(3, 5,
1, 0)
# when
C = matrix_multiply(A, B)
# then
assert_that(C, is_(M2(5, 5,
13, 15)))
def Sum(self, A, B, step):
self.C = matrix_add(self.C, matrix_multiply(A, B))
self.g_step = self.g_step + 1
if self.g_step == self.g_order:
if step != (self.g_order - 1):
self.g_left.injectFirst(A, (step + 1))
self.g_above.injectSecond(B, (step + 1))
self.collector.injectSubmatrix(self.C, self.g_row, self.g_col)
else:
if step != (self.g_order - 1):
self.g_left.injectFirst(A, (step + 1))
self.g_above.injectSecond(B, (step + 1))
def reducer(self, _, values):
start_time_x = time.time()
psis = np.array([])
for psi in values:
if len(psis) == 0:
psis = psi
else:
psis = np.hstack((psis, psi))
R = 1. / per_block * matrix_multiply(psis.T, psis)
wR, vR = np.linalg.eig(R)
wR = np.real(wR)
vR = np.real(vR)
wR[wR <= 0] = 1e-20
inv_sqrt = np.diag((per_block * wR)**(-0.5))
vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt)
num_final_eigens = 50
idx = (-wR).argsort()
end_time_x = time.time()
print vT, idx
print "reduce time ", end_time_x - start_time_x
yield None, 0
def reducer(self, _, values):
start_time_x = time.time()
psis = np.array([])
for psi in values:
if len(psis) == 0:
psis = psi
else:
psis = np.hstack((psis, psi))
R = 1. / per_block * matrix_multiply(psis.T, psis)
wR, vR = np.linalg.eig(R)
wR = np.real(wR)
vR = np.real(vR)
wR[wR <= 0] = 1e-20
inv_sqrt = np.diag((per_block * wR)**(-0.5))
vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt)
num_final_eigens = 50
idx = (-wR).argsort()
end_time_x = time.time()
print vT, idx
print "reduce time ",end_time_x - start_time_x
yield None, 0
def test_order_2_floats(self):
# given
A = M2(1.1, 2.3,
3.5, 4.6)
B = M2(3.7, 5.1,
1.5, 1.0 / 3)
# when
C = matrix_multiply(A, B)
# then
assert_that(C.ncols, is_(2))
assert_array_almost_equal(C.data,
[ 7.52, 6.37,
19.85, 19.38],
decimal=2)
def test_5x5_processors_200x200_operands(self):
nprocs = 25
# given
P = [self.broker.add_servant(ProcessorI(), Cannon.ProcessorPrx) for i in range(nprocs)]
loader = self.broker.add_servant(OperationsI(P), Cannon.OperationsPrx)
A_last = 200 * 200
A = Cannon.Matrix(200, range(1, 1 + A_last))
B = Cannon.Matrix(200, range(1 + A_last, 1 + A_last * 2))
# when
C = loader.matrixMultiply(A, B)
# then
expected = matrix_multiply(A, B)
assert_that(C, is_(expected))
class Client(Ice.Application):
def run(self, argv):
t_dist = 0;
t_secu = 0;
loader = self.string_to_proxy(argv[1], Cannon.OperationsPrx)
example = argv[2]
A = load_matrix_from_file('m/{}A'.format(example))
B = load_matrix_from_file('m/{}B'.format(example))
t_dist = time.time()
C = loader.matrixMultiply(A, B)
t_dist = time.time() - t_dist
t_secu = time.time()
c = matrix_multiply(A,B)
t_secu = time.time() - t_secu
def test_order_6(self):
# given
A = M6(1, 2, -1, 3, -5, 3, 3, 0, 2, 7, 1, 2, 5, 3, 3, 1, 4, 4, 2, 1, 2,
2, 3, 3, -2, 2, 6, 6, 9, 7, -2, 2, -4, 2, 6, 5)
B = M6(1, 2, 7, 7, -1, 3, 2, 1, 2, 1, -1, 2, 3, 0, 5, 5, 6, 1, 3, 1, 4,
1, 6, 4, 2, 4, 3, 3, 3, 9, 1, 4, 3, 2, 1, 9)
# when
C = matrix_multiply(A, B)
# then
expected = M6(4, -1, 12, -2, -3, 0, 34, 25, 68, 45, 56, 66, 35, 46, 84,
74, 32, 100, 25, 31, 52, 42, 33, 72, 63, 68, 92, 65, 106,
172, 13, 44, 11, -2, 11, 101)
assert_that(C, is_(expected))
def test_5x5_processors_200x200_operands(self):
nprocs = 25
# given
P = [self.broker.add_servant(ProcessorI(), Cannon.ProcessorPrx) for i in range(nprocs)]
loader = self.broker.add_servant(OperationsI(P), Cannon.OperationsPrx)
A_last = 200 * 200
A = Cannon.Matrix(200, range(1, 1 + A_last))
B = Cannon.Matrix(200, range(1 + A_last, 1 + A_last * 2))
# when
C = loader.matrixMultiply(A, B)
# then
expected = matrix_multiply(A, B)
assert_that(C, is_(expected))
def doPCA_cov(self, data, blocks):
def center(X):
meanX = X.mean(axis = 0)[np.newaxis,:]
centeredX = X - meanX
return (meanX, centeredX)
(means, dataNew) = center(data)
size = len(data) / blocks
print size
covs = []
for k in range(blocks):
dataBlock = dataNew[(k * blocks):((k + 1) * blocks),:]
covs.append(1./size * matrix_multiply(dataBlock.T, dataBlock))
print covs[0]
acc = np.zeros(covs[0].shape)
for cov in covs:
acc += (1. / blocks) * cov
w, v = np.linalg.eig(cov)
return (w, np.transpose(v))
def doPCA_cov(self, data, blocks):
def center(X):
meanX = X.mean(axis=0)[np.newaxis, :]
centeredX = X - meanX
return (meanX, centeredX)
(means, dataNew) = center(data)
size = len(data) / blocks
print size
covs = []
for k in range(blocks):
dataBlock = dataNew[(k * blocks):((k + 1) * blocks), :]
covs.append(1. / size * matrix_multiply(dataBlock.T, dataBlock))
print covs[0]
acc = np.zeros(covs[0].shape)
for cov in covs:
acc += (1. / blocks) * cov
w, v = np.linalg.eig(cov)
return (w, np.transpose(v))
def matrix_check(self,step, current=None):
current_A=self.matrixA[step]
current_B=self.matrixB[step]
order_auxiliar=self.order
multiplicacion=matrix_multiply(current_A, current_B)
self.step_aux += 1
if self.step_aux == 1:
self.resultado=multiplicacion
if self.step_aux != 1:
self.resultado=matrix_add(self.resultado,multiplicacion)
if step < order_auxiliar-1:
step += 1
self.left.injectA(current_A, step)
self.above.injectB(current_B, step)
if self.step_aux == order_auxiliar:
self.target.inject(self.index,self.resultado)
def do_pca(self, orig_data):
data = copy.copy(orig_data)
# Step 1: Make all data mean 0
data = np.transpose(data)
for row in data:
mean = float(sum(row)) / len(row)
for i in range(0, len(row)):
row[i] = row[i] - mean
x = data
print "getting covariance"
print data.shape
covariance_matrix = matrix_multiply(data, np.transpose(data))
print "got covariance"
print covariance_matrix.shape
[eigenvalues,eigenvectors] = la.eig(covariance_matrix)
eigenvectors = np.transpose(np.real(eigenvectors))
eigenvalues = np.real(eigenvalues)
print "got eigenvectors"
# Sort by eigenvalue
combined = []
for i in range(0, len(eigenvalues)):
combined.append((eigenvectors[i], eigenvalues[i]))
combined.sort(key=lambda c: c[1], reverse=True)
eigenvectors = []
eigenvalues = []
numFinalEigens = 100
for i in range(0, numFinalEigens):
eigenvectors.append(combined[i][0])
eigenvalues.append(combined[i][1])
eigenvectors = np.array(eigenvectors)
# utils.reconstruct_images(eigenvectors, orig_data)
# utils.calc_error(eigenvectors, orig_data)
utils.save_eigenvectors(eigenvectors)
def do_pca(self, orig_data):
data = copy.copy(orig_data)
# Step 1: Make all data mean 0
data = np.transpose(data)
for row in data:
mean = float(sum(row)) / len(row)
for i in range(0, len(row)):
row[i] = row[i] - mean
x = data
print "getting covariance"
print data.shape
covariance_matrix = matrix_multiply(data, np.transpose(data))
print "got covariance"
print covariance_matrix.shape
[eigenvalues, eigenvectors] = la.eig(covariance_matrix)
eigenvectors = np.transpose(np.real(eigenvectors))
eigenvalues = np.real(eigenvalues)
print "got eigenvectors"
# Sort by eigenvalue
combined = []
for i in range(0, len(eigenvalues)):
combined.append((eigenvectors[i], eigenvalues[i]))
combined.sort(key=lambda c: c[1], reverse=True)
eigenvectors = []
eigenvalues = []
numFinalEigens = 100
for i in range(0, numFinalEigens):
eigenvectors.append(combined[i][0])
eigenvalues.append(combined[i][1])
eigenvectors = np.array(eigenvectors)
# utils.reconstruct_images(eigenvectors, orig_data)
# utils.calc_error(eigenvectors, orig_data)
utils.save_eigenvectors(eigenvectors)
class cliente(Ice.Application):
def run(self, argv):
prxFrontend = self.communicator().stringToProxy(argv[1])
frontend = Cannon.FrontendPrx.checkedCast(prxFrontend)
if not frontend:
raise RuntimeError("Invalid proxy frontend")
A_last = 400 * 400
A400 = Cannon.Matrix(400, range(1, 1 + A_last))
B400 = Cannon.Matrix(400, range(1 + A_last, 1 + A_last * 2))
res400=frontend.multiply(A400,B400)
res400dir=matrix_multiply(A400,B400)
if(res400 == res400dir):
print("Ok en matrix 400x400")
print_matrix(res400)
else:
print("Error en matrix 400x400")
A_last = 600 * 600
def try_multiply(self):
current_A = self.A_blocks[self.current_step]
current_B = self.B_blocks[self.current_step]
if current_A is None or current_B is None:
return
if self.current_step == 0:
self.result_parcial = Cannon.Matrix(current_A.ncols,
[0] * (current_A.ncols**2))
product = matrix_multiply(current_A, current_B)
self.result_parcial = matrix_add(self.result_parcial, product)
self.current_step += 1
if self.current_step == self.order:
self.target.injectSubmatrix(self.result_parcial, self.row,
self.col)
return
self.left.injectFirst(current_A, self.current_step)
self.above.injectSecond(current_B, self.current_step)
self.try_multiply()
def doPCA_eig_smart(self, data, blocks):
def center(X):
meanX = X.mean(axis = 0)[np.newaxis,:]
centeredX = X - meanX
return (meanX, centeredX)
(means, dataNew) = center(data)
size = len(data) / blocks
covs = []
psis = np.array([])
for k in range(blocks):
dataBlock = dataNew[(k * size):((k + 1) * size),:]
# do the inner product instead of outer product
covs.append(1./size * matrix_multiply(dataBlock, dataBlock.T))
w, v = np.linalg.eig(covs[k])
v = matrix_multiply(dataBlock.T, v)
idx = (-w).argsort()
num_eigens = 10
w = w[idx[:num_eigens]]
v = v[:,idx[:num_eigens]]
psi = matrix_multiply(v, (np.diag((size * w)**0.5)))
if len(psis) == 0:
psis = psi
else:
psis = np.hstack((psis, psi))
R = 1./size * matrix_multiply(psis.T, psis)
wR, vR = np.linalg.eig(R)
inv_sqrt = np.diag((size * wR)**(-0.5))
vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt)
idx = (-wR).argsort()
print wR[idx], vT[:,idx[:20]].T
utils.calc_error(vT[:,idx[:20]].T, data)
return wR[idx], vT[:,idx[:20]].T
def doPCA_eig_smart(self, data, blocks):
def center(X):
meanX = X.mean(axis=0)[np.newaxis, :]
centeredX = X - meanX
return (meanX, centeredX)
(means, dataNew) = center(data)
size = len(data) / blocks
covs = []
psis = np.array([])
for k in range(blocks):
dataBlock = dataNew[(k * size):((k + 1) * size), :]
# do the inner product instead of outer product
covs.append(1. / size * matrix_multiply(dataBlock, dataBlock.T))
w, v = np.linalg.eig(covs[k])
v = matrix_multiply(dataBlock.T, v)
idx = (-w).argsort()
num_eigens = 10
w = w[idx[:num_eigens]]
v = v[:, idx[:num_eigens]]
psi = matrix_multiply(v, (np.diag((size * w)**0.5)))
if len(psis) == 0:
psis = psi
else:
psis = np.hstack((psis, psi))
R = 1. / size * matrix_multiply(psis.T, psis)
wR, vR = np.linalg.eig(R)
inv_sqrt = np.diag((size * wR)**(-0.5))
vT = matrix_multiply(matrix_multiply(psis, vR), inv_sqrt)
idx = (-wR).argsort()
print wR[idx], vT[:, idx[:20]].T
utils.calc_error(vT[:, idx[:20]].T, data)
return wR[idx], vT[:, idx[:20]].T
res400=frontend.multiply(A400,B400)
res400dir=matrix_multiply(A400,B400)
if(res400 == res400dir):
print("Ok en matrix 400x400")
print_matrix(res400)
else:
print("Error en matrix 400x400")
A_last = 600 * 600
A600 = Cannon.Matrix(600, range(1, 1 + A_last))
B600 = Cannon.Matrix(600, range(1 + A_last, 1 + A_last * 2))
res600=frontend.multiply(A600,B600)
res600dir=matrix_multiply(A600,B600)
if(res600 == res600dir):
print("Ok en matrix 600x600")
print_matrix(res600)
else:
print("Error en matrix 600x600")
A_last = 1000 * 1000
A1000 = Cannon.Matrix(600, range(1, 1 + A_last))
B1000 = Cannon.Matrix(600, range(1 + A_last, 1 + A_last * 2))
res1000=frontend.multiply(A1000,B1000)
res1000dir=matrix_multiply(A1000,B1000)
if(res1000 == res1000dir):
print("Ok en matrix 1000x1000")