Exemplos de shuffle em Python

Exemplo n.º 1

Arquivo: test_netflix.py Projeto: GabrielWen/spartan

def test_netflix_sgd(ctx):
  U = 100
  M = 100*100
  r = 20
  d = 8
  P_RATING = 1000.0 / (U * M)

  # create random factor and value matrices
  Mfactor = spartan.eager(spartan.rand(M, r).astype(np.float32))
  Ufactor = spartan.eager(spartan.rand(U, r).astype(np.float32))

  V = spartan.sparse_empty((U, M),
                           tile_hint=(divup(U, d), divup(M, d)),
                           dtype=np.float32)

#   V = spartan.shuffle(V, netflix.load_netflix_mapper,
#                           kw={ 'load_file' : '/big1/netflix.zip' })

  V = spartan.eager(
        spartan.tocoo(
          spartan.shuffle(V, netflix.fake_netflix_mapper,
                          target=V, kw={'p_rating': P_RATING})))

  for i in range(2):
    _ = netflix.sgd(V, Mfactor, Ufactor).evaluate()

Exemplo n.º 2

Arquivo: benchmark_netflix.py Projeto: muddimedia/spartan-1

def benchmark_netflix_sgd(ctx, timer):
    d = ctx.num_workers

    V = spartan.sparse_empty((U, M),
                             tile_hint=(divup(U, d), divup(M, d)),
                             dtype=np.float32)

    V = timer.time_op(
        'prep', lambda: spartan.eager(
            spartan.tocoo(
                spartan.shuffle(V,
                                netflix.fake_netflix_mapper,
                                target=V,
                                kw={'p_rating': P_RATING}))))

    #   V = spartan.shuffle(V, netflix.load_netflix_mapper,
    #                           kw={ 'load_file' : '/big1/netflix.zip' })

    for r in [25, 50]:
        Mfactor = spartan.eager(
            spartan.rand(M, r, tile_hint=(divup(M, d), r)).astype(np.float32))
        Ufactor = spartan.eager(
            spartan.rand(U, r, tile_hint=(divup(U, d), r)).astype(np.float32))

        timer.time_op('rank %d' % r, netflix.sgd(V, Mfactor, Ufactor).force)

Exemplo n.º 3

Arquivo: test_netflix.py Projeto: rossparks/spartan

def test_netflix_sgd(ctx):
    U = 100
    M = 100 * 100
    r = 20
    d = 8
    P_RATING = 1000.0 / (U * M)

    # create random factor and value matrices
    Mfactor = spartan.eager(spartan.rand(M, r).astype(np.float32))
    Ufactor = spartan.eager(spartan.rand(U, r).astype(np.float32))

    V = spartan.sparse_empty((U, M),
                             tile_hint=(divup(U, d), divup(M, d)),
                             dtype=np.float32)

    #   V = spartan.shuffle(V, netflix.load_netflix_mapper,
    #                           kw={ 'load_file' : '/big1/netflix.zip' })

    V = spartan.eager(
        spartan.tocoo(
            spartan.shuffle(V,
                            netflix.fake_netflix_mapper,
                            target=V,
                            kw={'p_rating': P_RATING})))

    for i in range(2):
        _ = netflix.sgd(V, Mfactor, Ufactor).force()

Exemplo n.º 4

Arquivo: benchmark_netflix.py Projeto: GabrielWen/spartan

def benchmark_netflix_sgd(ctx, timer):
    d = ctx.num_workers

    V = spartan.sparse_empty((U, M), tile_hint=(divup(U, d), divup(M, d)), dtype=np.float32)

    V = timer.time_op(
        "prep",
        lambda: spartan.eager(
            spartan.tocoo(spartan.shuffle(V, netflix.fake_netflix_mapper, target=V, kw={"p_rating": P_RATING}))
        ),
    )

    #   V = spartan.shuffle(V, netflix.load_netflix_mapper,
    #                           kw={ 'load_file' : '/big1/netflix.zip' })

    for r in [25, 50]:
        Mfactor = spartan.eager(spartan.rand(M, r, tile_hint=(divup(M, d), r)).astype(np.float32))
        Ufactor = spartan.eager(spartan.rand(U, r, tile_hint=(divup(U, d), r)).astype(np.float32))

        timer.time_op("rank %d" % r, netflix.sgd(V, Mfactor, Ufactor).force)