def shortestPath(ctx, dim, numIters):
	dist = eager(
			expr.shuffle(
				expr.ndarray(
					(dim, 1),
					dtype = np.int64,
					tile_hint = (dim / ctx.num_workers, 1)
				),
			make_dist,
			)
		)

	linkMatrix = eager(
				expr.shuffle(
					expr.ndarray(
						(dim, dim),
						dtype = np.int64,
						tile_hint = (dim, dim / ctx.num_workers)),
				make_matrix,
				))

	startCompute = time.time()
	for it in range(numIters):
		first = expr.add(dist, linkMatrix)
		second = first.min(axis = 0)
		dist = second.reshape(dim, 1)
	dist.evaluate()
	endCompute = time.time()
	return endCompute - startCompute
Exemplo n.º 2
0
def test_netflix_sgd(ctx):
    U = 100
    M = 100 * 100
    r = 20
    d = 8
    P_RATING = 1000.0 / (U * M)

    # create random factor and value matrices
    Mfactor = spartan.eager(spartan.rand(M, r).astype(np.float32))
    Ufactor = spartan.eager(spartan.rand(U, r).astype(np.float32))

    V = spartan.sparse_empty((U, M),
                             tile_hint=(divup(U, d), divup(M, d)),
                             dtype=np.float32)

    #   V = spartan.shuffle(V, netflix.load_netflix_mapper,
    #                           kw={ 'load_file' : '/big1/netflix.zip' })

    V = spartan.eager(
        spartan.tocoo(
            spartan.shuffle(V,
                            netflix.fake_netflix_mapper,
                            target=V,
                            kw={'p_rating': P_RATING})))

    for i in range(2):
        _ = netflix.sgd(V, Mfactor, Ufactor).force()
def bfs(ctx, dim):
	util.log_info("start to computing......")

	sGenerate = time.time()
	current = eager(
			expr.shuffle(
				expr.ndarray(
					(dim, 1),
					dtype = np.int64,
					tile_hint = (dim / ctx.num_workers, 1)),
				make_current,
			))
	
	linkMatrix = eager(
				expr.shuffle(
					expr.ndarray(
					 (dim, dim),
					 dtype = np.int64,
					 tile_hint = (dim, dim / ctx.num_workers)),
				make_matrix,
				))
	eGenerate = time.time()

	startCompute = time.time()
	while(True):
		next = expr.dot(linkMatrix, current)
		formerNum = expr.count_nonzero(current)
		laterNum = expr.count_nonzero(next)
		hasNew = expr.equal(formerNum, laterNum).glom()
		current = next
		if (hasNew):
			break
	current.evaluate()
	endCompute = time.time()
	return (eGenerate - sGenerate, endCompute - startCompute) 
Exemplo n.º 4
0
def test_netflix_sgd(ctx):
  U = 100
  M = 100*100
  r = 20
  d = 8
  P_RATING = 1000.0 / (U * M)

  # create random factor and value matrices
  Mfactor = spartan.eager(spartan.rand(M, r).astype(np.float32))
  Ufactor = spartan.eager(spartan.rand(U, r).astype(np.float32))

  V = spartan.sparse_empty((U, M),
                           tile_hint=(divup(U, d), divup(M, d)),
                           dtype=np.float32)

#   V = spartan.shuffle(V, netflix.load_netflix_mapper,
#                           kw={ 'load_file' : '/big1/netflix.zip' })

  V = spartan.eager(
        spartan.tocoo(
          spartan.shuffle(V, netflix.fake_netflix_mapper,
                          target=V, kw={'p_rating': P_RATING})))

  for i in range(2):
    _ = netflix.sgd(V, Mfactor, Ufactor).evaluate()
Exemplo n.º 5
0
def benchmark_netflix_sgd(ctx, timer):
    d = ctx.num_workers

    V = spartan.sparse_empty((U, M),
                             tile_hint=(divup(U, d), divup(M, d)),
                             dtype=np.float32)

    V = timer.time_op(
        'prep', lambda: spartan.eager(
            spartan.tocoo(
                spartan.shuffle(V,
                                netflix.fake_netflix_mapper,
                                target=V,
                                kw={'p_rating': P_RATING}))))

    #   V = spartan.shuffle(V, netflix.load_netflix_mapper,
    #                           kw={ 'load_file' : '/big1/netflix.zip' })

    for r in [25, 50]:
        Mfactor = spartan.eager(
            spartan.rand(M, r, tile_hint=(divup(M, d), r)).astype(np.float32))
        Ufactor = spartan.eager(
            spartan.rand(U, r, tile_hint=(divup(U, d), r)).astype(np.float32))

        timer.time_op('rank %d' % r, netflix.sgd(V, Mfactor, Ufactor).force)
def connectedConponents(ctx, dim, numIters):
	linkMatrix = eager(
					expr.shuffle(
						expr.ndarray(
							(dim, dim),
							dtype = np.int64,
							tile_hint = (dim / ctx.num_workers, dim)),
						make_matrix,
					))

	power = eager(
					expr.shuffle(
						expr.ndarray(
							(dim, dim),
							dtype = np.int64,
							tile_hint = (dim / ctx.num_workers, dim)),
						make_matrix,
					))

	eye = expr.eye(dim, tile_hint = (dim / ctx.num_workers,dim))
	startCompute = time.time()
	result = expr.logical_or(eye, linkMatrix).optimized().glom()
	for i in range(numIters):
		power = expr.dot(power, linkMatrix).optimized().glom()
		result = expr.logical_or(result, power)
	result.optimized().glom()
	final = expr.logical_and(result, expr.transpose(result.optimized())).optimized().evaluate()
	endCompute = time.time()
	return endCompute - startCompute
Exemplo n.º 7
0
def benchmark_pagerank(ctx, timer):
  num_pages = PAGES_PER_WORKER * ctx.num_workers
  util.log_info('Total pages: %s', num_pages)

  wts = eager(
    expr.shuffle(
      expr.ndarray(
        (num_pages, num_pages), 
        dtype=np.float32,
        tile_hint=(num_pages, PAGES_PER_WORKER / 8)),
      make_weights,
    ))

  p = eager(expr.ones((num_pages, 1), 
                      tile_hint=(PAGES_PER_WORKER / 8, 1), 
                      dtype=np.float32))

  for i in range(3):
    timer.time_op('pagerank', lambda: expr.dot(wts, p).force())
def saveAsTextFile(ctx, dim):
	matrix = eager(
			expr.shuffle(
				expr.ndarray(
					(dim, dim),
					dtype = np.int32,
					tile_hint = (dim, dim / ctx.num_workers)),
					#tile_hint = (2, 2)),
			make_matrix,
			))
Exemplo n.º 9
0
def benchmark_pagerank(ctx, timer):
    num_pages = PAGES_PER_WORKER * ctx.num_workers
    util.log_info('Total pages: %s', num_pages)

    wts = eager(
        expr.shuffle(
            expr.ndarray((num_pages, num_pages),
                         dtype=np.float32,
                         tile_hint=(num_pages, PAGES_PER_WORKER / 8)),
            make_weights,
        ))

    p = eager(
        expr.ones((num_pages, 1),
                  tile_hint=(PAGES_PER_WORKER / 8, 1),
                  dtype=np.float32))

    for i in range(3):
        timer.time_op('pagerank', lambda: expr.dot(wts, p).force())
Exemplo n.º 10
0
def benchmark_netflix_sgd(ctx, timer):
    d = ctx.num_workers

    V = spartan.sparse_empty((U, M), tile_hint=(divup(U, d), divup(M, d)), dtype=np.float32)

    V = timer.time_op(
        "prep",
        lambda: spartan.eager(
            spartan.tocoo(spartan.shuffle(V, netflix.fake_netflix_mapper, target=V, kw={"p_rating": P_RATING}))
        ),
    )

    #   V = spartan.shuffle(V, netflix.load_netflix_mapper,
    #                           kw={ 'load_file' : '/big1/netflix.zip' })

    for r in [25, 50]:
        Mfactor = spartan.eager(spartan.rand(M, r, tile_hint=(divup(M, d), r)).astype(np.float32))
        Ufactor = spartan.eager(spartan.rand(U, r, tile_hint=(divup(U, d), r)).astype(np.float32))

        timer.time_op("rank %d" % r, netflix.sgd(V, Mfactor, Ufactor).force)
def pagerankDistributed(ctx, numPage, numIters, alpha):
  sGenerate = time.time()
  rank = eager(expr.ones((numPage, 1), tile_hint = (numPage / ctx.num_workers, 1), dtype = np.float32))
  linkMatrix = eager(
              expr.shuffle(
                expr.ndarray(
                  (numPage, numPage),
                  dtype = np.float32,
                  tile_hint = (numPage, numPage / ctx.num_workers)),
              make_weights,
              ))
  eGenerate = time.time()
  util.log_info("**pagerank** rank init finished")
  startCompute = time.time()
  for i in range(numIters):
    #rank = ((1 - alpha) * expr.dot(linkMatrix, rank,tile_hint = (numPage, numPage/10))) + belta
    rank = expr.dot(linkMatrix, rank, tile_hint = (numPage, numPage/10))
  rank.evaluate()
  endCompute = time.time()
  util.log_info("**pagerank** compute finished")
  return (eGenerate - sGenerate, endCompute - startCompute)
Exemplo n.º 12
0
def benchmark_stdev(ctx, timer):
    X = S.eager(S.randn(ctx.num_workers, W, H))
    timer.benchmark_op(lambda: highlight_image(X).optimized().force())
def loadMatrix(ctx, filePath, numPage):
    #return eager(expr.from_file(filePath, tile_hint = (numPage, numPage / 8)))
    return eager(expr.from_file_parallel(filePath, "numpy", sparse = True, tile_hint = (numPage, numPage/10)))
Exemplo n.º 14
0
def benchmark_stdev(ctx, timer):
    X = S.eager(S.randn(ctx.num_workers, W, H))
    timer.benchmark_op(lambda: highlight_image(X).optimized().evaluate())