def test5(self): ''' Test transpose. ''' random.seed(13) m, k, n = 31, 11, 31 transA, transB = True, True alpha, beta = -3.1415, 0.5 sparsityA, sparsityB, sparsityC = 0.2, 0.3, 0.4 protocolA, protocolB, protocolC = MatrixWrapper.RAW, MatrixWrapper.RAW, MatrixWrapper.RAW dfsDirA, dfsDirB, dfsDirC = None, None, None maxCoresA, maxCoresB, maxCoresC = 13, 11, 7 maxTotalBlocks = 10 # instantiate A A = randomSparseMatrix(k, m, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # instantiate B B = randomSparseMatrix(n, k, sparsityB) Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB) # instantiate C C = randomSparseMatrix(m, n, sparsityC) Cwrap = MatrixWrapper.wrapMatrix(C, protocolC, dfsDirC, maxCoresC) # multiply Zwrap = dgemm(disco, transA, transB, m, n, k, alpha, Awrap, Bwrap, beta, Cwrap, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(alpha, beta, transA, transB, A, B, C, Z)
def test1(self):
"""
Test normal usage.
"""
random.seed(13)
m, n = 101, 51
minRowId, minColId, maxRowId, maxColId = 10, 20, 30, 40
sparsityA = 0.5
protocolA = MatrixWrapper.RAW
dfsDirA = None
maxCoresA = 13
maxCores = 3
# instantiate A
A = randomSparseMatrix(m, n, sparsityA)
Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
# replace elements in matrix
args = {}
args['minRowId'] = minRowId
args['maxRowId'] = maxRowId
args['minColId'] = minColId
args['maxColId'] = maxColId
f = lambda args: args['v'] if args['i']>=args['minRowId'] and args['i']<=args['maxRowId'] and args['j']>=args['minColId'] and args['j']<=args['maxColId'] else 0
Zwrap = replace(disco, m, n, Awrap, f, args, maxCores)
Z = Zwrap.unwrapMatrix(m, n).todense()
# clean up
disco.purge(jobname(Zwrap.urls[0]))
# validate
self.validate(minRowId, minColId, maxRowId, maxColId, A, Z)
def test2(self): ''' Test transpose. ''' random.seed(13) m, n = 33, 3 transA, transB = True, True alpha, beta = 0.2, 0.4 sparsityA, sparsityB = 0.1, 0.1 protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW dfsDirA, dfsDirB = None, None maxCoresA, maxCoresB = 5, 5 maxTotalBlocks = 3 # instantiate A A = randomSparseMatrix(n, m, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # instantiate B B = randomSparseMatrix(n, m, sparsityB) Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB) # add Zwrap = dgema(disco, transA, transB, m, n, alpha, Awrap, Bwrap, beta, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(alpha, beta, transA, transB, A, B, Z)
def test3(self): ''' Test the case where alpha=0. ''' random.seed(13) m, n = 33, 77 transA, transB = False, False alpha, beta = 0.0, 0.5 sparsityA, sparsityB = 0.0, 0.0 protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW dfsDirA, dfsDirB = None, None maxCoresA, maxCoresB = 7, 5 maxTotalBlocks = 7 # instantiate A A = randomSparseMatrix(m, n, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # instantiate B B = randomSparseMatrix(m, n, sparsityB) Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB) # add Zwrap = dgema(disco, transA, transB, m, n, alpha, Awrap, Bwrap, beta, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(alpha, beta, transA, transB, A, B, Z)
def test0(self): ''' Test normal basic usage. ''' random.seed(13) m, n = 30, 30 transA, transB = False, False alpha, beta = 1.0, 1.0 sparsityA, sparsityB = 0.3, 0.0 protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW dfsDirA, dfsDirB = None, None maxCoresA, maxCoresB = 13, 11 maxTotalBlocks = 10 # instantiate A A = randomSparseMatrix(m, n, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # instantiate B B = randomSparseMatrix(m, n, sparsityB) Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB) # add Zwrap = dgema(disco, transA, transB, m, n, alpha, Awrap, Bwrap, beta, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(alpha, beta, transA, transB, A, B, Z)
def test1(self): ''' Test the case where size of matrix is smaller than the number of inputs for A and B. ''' random.seed(13) m, n = 3, 3 transA, transB = False, False alpha, beta = -1.2, 0.8 sparsityA, sparsityB = 0.0, 0.0 protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW dfsDirA, dfsDirB = None, None maxCoresA, maxCoresB = 13, 11 maxTotalBlocks = 10 # instantiate A A = randomSparseMatrix(m, n, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # instantiate B B = randomSparseMatrix(m, n, sparsityB) Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB) # add Zwrap = dgema(disco, transA, transB, m, n, alpha, Awrap, Bwrap, beta, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(alpha, beta, transA, transB, A, B, Z)
def purge(self, disco): """ Delete matrix from cluster. """ if len(self.urls) > 0: from disco.util import jobname disco.purge(jobname(self.urls[0]))
def test2(self): ''' Test the case where C is an empty wrapper. ''' random.seed(13) m, k, n = 22, 11, 21 transA, transB = False, False alpha, beta = 1.0, 1.0 sparsityA, sparsityB = 0.0, 0.0 protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW dfsDirA, dfsDirB = None, None maxCoresA, maxCoresB = 13, 11 maxTotalBlocks = 10 # instantiate A A = randomSparseMatrix(m, k, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # instantiate B B = randomSparseMatrix(k, n, sparsityB) Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB) # instantiate C Cwrap = MatrixWrapper() # multiply Zwrap = dgemm(disco, transA, transB, m, n, k, alpha, Awrap, Bwrap, beta, Cwrap, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(alpha, beta, transA, transB, A, B, C, Z)
def test1(self): ''' Test the case where size of matrix is smaller than the number of inputs for A, B and C. ''' random.seed(13) m, k, n = 2, 1, 2 transA, transB = False, False alpha, beta = 1.0, 1.0 sparsityA, sparsityB, sparsityC = 0.0, 0.0, 0.0 protocolA, protocolB, protocolC = MatrixWrapper.RAW, MatrixWrapper.RAW, MatrixWrapper.RAW dfsDirA, dfsDirB, dfsDirC = None, None, None maxCoresA, maxCoresB, maxCoresC = 13, 11, 7 maxTotalBlocks = 10 # instantiate A A = randomSparseMatrix(m, k, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # instantiate B B = randomSparseMatrix(k, n, sparsityB) Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB) # instantiate C C = randomSparseMatrix(m, n, sparsityC) Cwrap = MatrixWrapper.wrapMatrix(C, protocolC, dfsDirC, maxCoresC) # multiply Zwrap = dgemm(disco, transA, transB, m, n, k, alpha, Awrap, Bwrap, beta, Cwrap, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(alpha, beta, transA, transB, A, B, C, Z)
def test4(self): ''' Test the rank-1 case where dim(A)=(m,1) and dim(B)=(1,n). ''' random.seed(13) m, k, n = 301, 1, 223 transA, transB = False, False alpha, beta = 2.27, -3.14 sparsityA, sparsityB, sparsityC = 0.4, 0.3, 0.6 protocolA, protocolB, protocolC = MatrixWrapper.RAW, MatrixWrapper.RAW, MatrixWrapper.RAW dfsDirA, dfsDirB, dfsDirC = None, None, None maxCoresA, maxCoresB, maxCoresC = 17, 11, 23 maxTotalBlocks = 13 # instantiate A A = randomSparseMatrix(m, k, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # instantiate B B = randomSparseMatrix(k, n, sparsityB) Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB) # instantiate C C = randomSparseMatrix(m, n, sparsityC) Cwrap = MatrixWrapper.wrapMatrix(C, protocolC, dfsDirC, maxCoresC) # multiply Zwrap = dgemm(disco, transA, transB, m, n, k, alpha, Awrap, Bwrap, beta, Cwrap, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(alpha, beta, transA, transB, A, B, C, Z)
def test1(self): """ Test normal usage. """ random.seed(13) m, n = 101, 51 maxCores = 3 # generate 1- matrix Zwrap = ones(disco, m, n, maxCores) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(m, n, Z)
def test1(self): """ Test normal usage. """ random.seed(13) from numpy import arange lb, ub = -3.0, 4.0 m, n = 500, 333 sparsity = 0.5 maxCores = 4 # generate random matrix Mwrap = rand(disco, m, n, sparsity, lb, ub, maxCores) M = Mwrap.unwrapMatrix(m, n) # validate self.validate(m, n, sparsity, lb, ub, M, 0.05*m*n) # clean up disco.purge(jobname(Mwrap.urls[0]))
def test2(self): """ Test input to diag is a wide matrix with negative k. """ random.seed(13) m, n = 33, 77 k = -3 sparsityA = 0.3 protocolA = MatrixWrapper.RAW dfsDirA = None maxCoresA = 13 maxTotalBlocks = 4 # instantiate A A = randomSparseMatrix(m, n, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # compute diagonal Zwrap = diag(disco, m, n, Awrap, k, maxTotalBlocks) # validate self.validate(k, A, Zwrap) # clean up disco.purge(jobname(Zwrap.urls[0]))
def test3(self): """ Test input to diag is a col vector. """ random.seed(13) m, n = 123, 1 k = 1 sparsityA = 0.2 protocolA = MatrixWrapper.RAW dfsDirA = None maxCoresA = 4 maxTotalBlocks = 5 # instantiate A A = randomSparseMatrix(m, n, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # compute diagonal Zwrap = diag(disco, m, n, Awrap, k, maxTotalBlocks) # validate self.validate(k, A, Zwrap) # clean up disco.purge(jobname(Zwrap.urls[0]))
def test1(self): ''' Test transpose. ''' random.seed(13) m, n = 13, 123 transA = True sparsityA = 0.8 protocolA = MatrixWrapper.RAW dfsDirA = None maxCoresA = 13 maxTotalBlocks = 10 # instantiate A A = randomSparseMatrix(n, m, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # sum Zwrap = dgemsum(disco, transA, m, n, Awrap, maxTotalBlocks) Z = Zwrap.unwrapMatrix(1, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(transA, A, Z)
def test0(self): ''' Test normal basic usage. ''' random.seed(13) m, n = 12, 340 transA = False alpha = -3.14 sparsityA = 0.3 protocolA = MatrixWrapper.RAW dfsDirA = None maxCoresA = 13 maxTotalBlocks = 10 # instantiate A A = randomSparseMatrix(m, n, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # sum Zwrap = dgemscal(disco, transA, m, n, alpha, Awrap, maxTotalBlocks) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(transA, alpha, A, Z)
def test2(self): """ Test transpose. """ random.seed(13) m, n = 101, 51 transA = True minRowId, minColId, maxRowId, maxColId = 10, 20, 30, 40 sparsityA = 0.5 protocolA = MatrixWrapper.RAW dfsDirA = None maxCoresA = 13 maxCores = 3 # instantiate A A = randomSparseMatrix(n, m, sparsityA) Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA) # crop matrix Zwrap = crop(disco, transA, m, n, Awrap, minRowId, minColId, maxRowId, maxColId, maxCores) Z = Zwrap.unwrapMatrix(m, n).todense() # clean up disco.purge(jobname(Zwrap.urls[0])) # validate self.validate(transA, minRowId, minColId, maxRowId, maxColId, A, Z)
tserver.run_server(data_gen)
disco = Disco(sys.argv[1])
results = disco.new_job(name = "test_chain_0", input = tserver.makeurl([""] * 100),
map = fun_map, reduce = fun_reduce, nr_reduces = 4,
sort = False, params = {'suffix': '0'}).wait()
i = 1
while i < 10:
nresults = disco.new_job(name = "test_chain_%d" % i, input = results,
map = fun_map, reduce = fun_reduce, nr_reduces = 4,
map_reader = chain_reader, sort = False,
params = {'suffix': str(i)}).wait()
disco.purge(jobname(results[0]))
results = nresults
i += 1
for key, value in result_iterator(results):
if key[:5] not in ani or key[5:] != "0-1-2-3-4-5-6-7-8-9-":
raise "Corrupted key: %s" % key
if value != "9":
raise "Corrupted value: %s" % value
disco.purge(jobname(results[0]))
print "ok"
