예제 #1
0
파일: testDGEMM.py 프로젝트: davin/disco
	def test5(self):
		'''
		Test transpose.
		'''
		random.seed(13)
		m, k, n = 31, 11, 31 
		transA, transB = True, True
		alpha, beta = -3.1415, 0.5
		sparsityA, sparsityB, sparsityC = 0.2, 0.3, 0.4
		protocolA, protocolB, protocolC = MatrixWrapper.RAW, MatrixWrapper.RAW, MatrixWrapper.RAW
		dfsDirA, dfsDirB, dfsDirC = None, None, None
		maxCoresA, maxCoresB, maxCoresC = 13, 11, 7
		maxTotalBlocks = 10
		# instantiate A
		A = randomSparseMatrix(k, m, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# instantiate B
		B = randomSparseMatrix(n, k, sparsityB)
		Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB)
		# instantiate C
		C = randomSparseMatrix(m, n, sparsityC)
		Cwrap = MatrixWrapper.wrapMatrix(C, protocolC, dfsDirC, maxCoresC)
		# multiply
		Zwrap = dgemm(disco, transA, transB, m, n, k, alpha, Awrap, Bwrap, beta, Cwrap, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(alpha, beta, transA, transB, A, B, C, Z)
예제 #2
0
파일: testReplace.py 프로젝트: davin/disco
	def test1(self):
		"""
		Test normal usage.
		"""
		random.seed(13)
		m, n = 101, 51
		minRowId, minColId, maxRowId, maxColId = 10, 20, 30, 40
		sparsityA = 0.5
		protocolA = MatrixWrapper.RAW
		dfsDirA = None
		maxCoresA = 13
		maxCores = 3
		# instantiate A
		A = randomSparseMatrix(m, n, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# replace elements in matrix
		args = {}
		args['minRowId'] = minRowId 
		args['maxRowId'] = maxRowId
		args['minColId'] = minColId
		args['maxColId'] = maxColId
		f = lambda args: args['v'] if args['i']>=args['minRowId'] and args['i']<=args['maxRowId'] and args['j']>=args['minColId'] and args['j']<=args['maxColId'] else 0
		Zwrap = replace(disco, m, n, Awrap, f, args, maxCores)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(minRowId, minColId, maxRowId, maxColId, A, Z)
예제 #3
0
파일: testDGEMA.py 프로젝트: davin/disco
	def test2(self):
		'''
		Test transpose. 
		'''
		random.seed(13)
		m, n = 33, 3
		transA, transB = True, True
		alpha, beta = 0.2, 0.4
		sparsityA, sparsityB = 0.1, 0.1
		protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW
		dfsDirA, dfsDirB = None, None
		maxCoresA, maxCoresB = 5, 5 
		maxTotalBlocks = 3
		# instantiate A
		A = randomSparseMatrix(n, m, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# instantiate B
		B = randomSparseMatrix(n, m, sparsityB)
		Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB)
		# add
		Zwrap = dgema(disco, transA, transB, m, n, alpha, Awrap, Bwrap, beta, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(alpha, beta, transA, transB, A, B, Z)
예제 #4
0
파일: testDGEMA.py 프로젝트: davin/disco
	def test3(self):
		'''
		Test the case where alpha=0.
		'''
		random.seed(13)
		m, n = 33, 77 
		transA, transB = False, False
		alpha, beta = 0.0, 0.5
		sparsityA, sparsityB = 0.0, 0.0
		protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW
		dfsDirA, dfsDirB = None, None
		maxCoresA, maxCoresB = 7, 5
		maxTotalBlocks = 7 
		# instantiate A
		A = randomSparseMatrix(m, n, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# instantiate B
		B = randomSparseMatrix(m, n, sparsityB)
		Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB)
		# add
		Zwrap = dgema(disco, transA, transB, m, n, alpha, Awrap, Bwrap, beta, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(alpha, beta, transA, transB, A, B, Z)
예제 #5
0
파일: testDGEMA.py 프로젝트: davin/disco
	def test0(self):
		'''
		Test normal basic usage.
		'''
		random.seed(13)
		m, n = 30, 30
		transA, transB = False, False
		alpha, beta = 1.0, 1.0
		sparsityA, sparsityB = 0.3, 0.0
		protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW
		dfsDirA, dfsDirB = None, None
		maxCoresA, maxCoresB = 13, 11
		maxTotalBlocks = 10
		# instantiate A
		A = randomSparseMatrix(m, n, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# instantiate B
		B = randomSparseMatrix(m, n, sparsityB)
		Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB)
		# add 
		Zwrap = dgema(disco, transA, transB, m, n, alpha, Awrap, Bwrap, beta, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(alpha, beta, transA, transB, A, B, Z)
예제 #6
0
파일: testDGEMA.py 프로젝트: davin/disco
	def test1(self):
		'''
		Test the case where size of matrix is smaller than the number of inputs for A and B.
		'''
		random.seed(13)
		m, n = 3, 3
		transA, transB = False, False
		alpha, beta = -1.2, 0.8
		sparsityA, sparsityB = 0.0, 0.0
		protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW
		dfsDirA, dfsDirB = None, None
		maxCoresA, maxCoresB = 13, 11
		maxTotalBlocks = 10
		# instantiate A
		A = randomSparseMatrix(m, n, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# instantiate B
		B = randomSparseMatrix(m, n, sparsityB)
		Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB)
		# add
		Zwrap = dgema(disco, transA, transB, m, n, alpha, Awrap, Bwrap, beta, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(alpha, beta, transA, transB, A, B, Z)
예제 #7
0
파일: matrixWrap.py 프로젝트: davin/disco
	def purge(self, disco):
		"""
		Delete matrix from cluster.
		"""
		if len(self.urls) > 0:
			from disco.util import jobname
			disco.purge(jobname(self.urls[0]))
예제 #8
0
파일: testDGEMM.py 프로젝트: davin/disco
	def test2(self):
		'''
		Test the case where C is an empty wrapper.
		'''
		random.seed(13)
		m, k, n = 22, 11, 21
		transA, transB = False, False
		alpha, beta = 1.0, 1.0
		sparsityA, sparsityB = 0.0, 0.0
		protocolA, protocolB = MatrixWrapper.RAW, MatrixWrapper.RAW
		dfsDirA, dfsDirB = None, None
		maxCoresA, maxCoresB = 13, 11
		maxTotalBlocks = 10
		# instantiate A
		A = randomSparseMatrix(m, k, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# instantiate B
		B = randomSparseMatrix(k, n, sparsityB)
		Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB)
		# instantiate C
		Cwrap = MatrixWrapper()
		# multiply
		Zwrap = dgemm(disco, transA, transB, m, n, k, alpha, Awrap, Bwrap, beta, Cwrap, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(alpha, beta, transA, transB, A, B, C, Z)
예제 #9
0
파일: testDGEMM.py 프로젝트: davin/disco
	def test1(self):
		'''
		Test the case where size of matrix is smaller than the number of inputs for A, B and C.
		'''
		random.seed(13)
		m, k, n = 2, 1, 2
		transA, transB = False, False
		alpha, beta = 1.0, 1.0
		sparsityA, sparsityB, sparsityC = 0.0, 0.0, 0.0
		protocolA, protocolB, protocolC = MatrixWrapper.RAW, MatrixWrapper.RAW, MatrixWrapper.RAW
		dfsDirA, dfsDirB, dfsDirC = None, None, None
		maxCoresA, maxCoresB, maxCoresC = 13, 11, 7
		maxTotalBlocks = 10
		# instantiate A
		A = randomSparseMatrix(m, k, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# instantiate B
		B = randomSparseMatrix(k, n, sparsityB)
		Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB)
		# instantiate C
		C = randomSparseMatrix(m, n, sparsityC)
		Cwrap = MatrixWrapper.wrapMatrix(C, protocolC, dfsDirC, maxCoresC)
		# multiply
		Zwrap = dgemm(disco, transA, transB, m, n, k, alpha, Awrap, Bwrap, beta, Cwrap, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(alpha, beta, transA, transB, A, B, C, Z)
예제 #10
0
파일: testDGEMM.py 프로젝트: davin/disco
	def test4(self):
		'''
		Test the rank-1 case where dim(A)=(m,1) and dim(B)=(1,n).
		'''
		random.seed(13)
		m, k, n = 301, 1, 223
		transA, transB = False, False
		alpha, beta = 2.27, -3.14
		sparsityA, sparsityB, sparsityC = 0.4, 0.3, 0.6
		protocolA, protocolB, protocolC = MatrixWrapper.RAW, MatrixWrapper.RAW, MatrixWrapper.RAW
		dfsDirA, dfsDirB, dfsDirC = None, None, None
		maxCoresA, maxCoresB, maxCoresC = 17, 11, 23 
		maxTotalBlocks = 13
		# instantiate A
		A = randomSparseMatrix(m, k, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# instantiate B
		B = randomSparseMatrix(k, n, sparsityB)
		Bwrap = MatrixWrapper.wrapMatrix(B, protocolB, dfsDirB, maxCoresB)
		# instantiate C
		C = randomSparseMatrix(m, n, sparsityC)
		Cwrap = MatrixWrapper.wrapMatrix(C, protocolC, dfsDirC, maxCoresC)
		# multiply
		Zwrap = dgemm(disco, transA, transB, m, n, k, alpha, Awrap, Bwrap, beta, Cwrap, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(alpha, beta, transA, transB, A, B, C, Z)
예제 #11
0
파일: testOnes.py 프로젝트: davin/disco
	def test1(self):
		"""
		Test normal usage.
		"""
		random.seed(13)
		m, n = 101, 51
		maxCores = 3
		# generate 1- matrix 
		Zwrap = ones(disco, m, n, maxCores)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(m, n, Z)
예제 #12
0
파일: testRand.py 프로젝트: davin/disco
	def test1(self):
		"""
		Test normal usage.
		"""
		random.seed(13)
		from numpy import arange
		lb, ub = -3.0, 4.0
		m, n = 500, 333 
		sparsity = 0.5
		maxCores = 4
		# generate random matrix
		Mwrap = rand(disco, m, n, sparsity, lb, ub, maxCores)
		M = Mwrap.unwrapMatrix(m, n)
		# validate
		self.validate(m, n, sparsity, lb, ub, M, 0.05*m*n)
		# clean up
		disco.purge(jobname(Mwrap.urls[0]))
예제 #13
0
파일: testDiag.py 프로젝트: davin/disco
	def test2(self):
		"""
		Test input to diag is a wide matrix with negative k.
		"""
		random.seed(13)
		m, n = 33, 77
		k = -3 
		sparsityA = 0.3
		protocolA = MatrixWrapper.RAW
		dfsDirA = None
		maxCoresA = 13
		maxTotalBlocks = 4
		# instantiate A
		A = randomSparseMatrix(m, n, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# compute diagonal
		Zwrap = diag(disco, m, n, Awrap, k, maxTotalBlocks)
		# validate
		self.validate(k, A, Zwrap)
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
예제 #14
0
파일: testDiag.py 프로젝트: davin/disco
	def test3(self):
		"""
		Test input to diag is a col vector.
		"""
		random.seed(13)
		m, n = 123, 1
		k = 1 
		sparsityA = 0.2
		protocolA = MatrixWrapper.RAW
		dfsDirA = None
		maxCoresA = 4 
		maxTotalBlocks = 5
		# instantiate A
		A = randomSparseMatrix(m, n, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# compute diagonal
		Zwrap = diag(disco, m, n, Awrap, k, maxTotalBlocks)
		# validate
		self.validate(k, A, Zwrap)
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
예제 #15
0
파일: testDGEMSUM.py 프로젝트: davin/disco
	def test1(self):
		'''
		Test transpose. 
		'''
		random.seed(13)
		m, n = 13, 123
		transA = True 
		sparsityA = 0.8
		protocolA = MatrixWrapper.RAW
		dfsDirA = None
		maxCoresA = 13
		maxTotalBlocks = 10
		# instantiate A
		A = randomSparseMatrix(n, m, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# sum 
		Zwrap = dgemsum(disco, transA, m, n, Awrap, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(1, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(transA, A, Z)
예제 #16
0
파일: testDGEMSCAL.py 프로젝트: davin/disco
	def test0(self):
		'''
		Test normal basic usage.
		'''
		random.seed(13)
		m, n = 12, 340
		transA = False
		alpha = -3.14
		sparsityA = 0.3
		protocolA = MatrixWrapper.RAW
		dfsDirA = None
		maxCoresA = 13
		maxTotalBlocks = 10
		# instantiate A
		A = randomSparseMatrix(m, n, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# sum 
		Zwrap = dgemscal(disco, transA, m, n, alpha, Awrap, maxTotalBlocks)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(transA, alpha, A, Z)
예제 #17
0
파일: testCrop.py 프로젝트: davin/disco
	def test2(self):
		"""
		Test transpose. 
		"""
		random.seed(13)
		m, n = 101, 51
		transA = True 
		minRowId, minColId, maxRowId, maxColId = 10, 20, 30, 40
		sparsityA = 0.5
		protocolA = MatrixWrapper.RAW
		dfsDirA = None
		maxCoresA = 13
		maxCores = 3
		# instantiate A
		A = randomSparseMatrix(n, m, sparsityA)
		Awrap = MatrixWrapper.wrapMatrix(A, protocolA, dfsDirA, maxCoresA)
		# crop matrix 
		Zwrap = crop(disco, transA, m, n, Awrap, minRowId, minColId, maxRowId, maxColId, maxCores)
		Z = Zwrap.unwrapMatrix(m, n).todense()
		# clean up
		disco.purge(jobname(Zwrap.urls[0]))
		# validate
		self.validate(transA, minRowId, minColId, maxRowId, maxColId, A, Z)
예제 #18
0
tserver.run_server(data_gen)
disco = Disco(sys.argv[1])

results = disco.new_job(name = "test_chain_0", input = tserver.makeurl([""] * 100),
                map = fun_map, reduce = fun_reduce, nr_reduces = 4,
                sort = False, params = {'suffix': '0'}).wait()

i = 1
while i < 10:
        nresults = disco.new_job(name = "test_chain_%d" % i, input = results,
                map = fun_map, reduce = fun_reduce, nr_reduces = 4,
                map_reader = chain_reader, sort = False,
                params = {'suffix': str(i)}).wait()

        disco.purge(jobname(results[0]))
        results = nresults
        i += 1

for key, value in result_iterator(results):
        if key[:5] not in ani or key[5:] != "0-1-2-3-4-5-6-7-8-9-":
                raise "Corrupted key: %s" % key
        if value != "9":
                raise "Corrupted value: %s" % value

disco.purge(jobname(results[0]))

print "ok"