def test_netflix_sgd(ctx):
U = 100
M = 100*100
r = 20
d = 8
P_RATING = 1000.0 / (U * M)
# create random factor and value matrices
Mfactor = spartan.eager(spartan.rand(M, r).astype(np.float32))
Ufactor = spartan.eager(spartan.rand(U, r).astype(np.float32))
V = spartan.sparse_empty((U, M),
tile_hint=(divup(U, d), divup(M, d)),
dtype=np.float32)
# V = spartan.shuffle(V, netflix.load_netflix_mapper,
# kw={ 'load_file' : '/big1/netflix.zip' })
V = spartan.eager(
spartan.tocoo(
spartan.shuffle(V, netflix.fake_netflix_mapper,
target=V, kw={'p_rating': P_RATING})))
for i in range(2):
_ = netflix.sgd(V, Mfactor, Ufactor).evaluate()
def benchmark_netflix_sgd(ctx, timer):
d = ctx.num_workers
V = spartan.sparse_empty((U, M),
tile_hint=(divup(U, d), divup(M, d)),
dtype=np.float32)
V = timer.time_op(
'prep', lambda: spartan.eager(
spartan.tocoo(
spartan.shuffle(V,
netflix.fake_netflix_mapper,
target=V,
kw={'p_rating': P_RATING}))))
# V = spartan.shuffle(V, netflix.load_netflix_mapper,
# kw={ 'load_file' : '/big1/netflix.zip' })
for r in [25, 50]:
Mfactor = spartan.eager(
spartan.rand(M, r, tile_hint=(divup(M, d), r)).astype(np.float32))
Ufactor = spartan.eager(
spartan.rand(U, r, tile_hint=(divup(U, d), r)).astype(np.float32))
timer.time_op('rank %d' % r, netflix.sgd(V, Mfactor, Ufactor).force)
def test_netflix_sgd(ctx):
U = 100
M = 100 * 100
r = 20
d = 8
P_RATING = 1000.0 / (U * M)
# create random factor and value matrices
Mfactor = spartan.eager(spartan.rand(M, r).astype(np.float32))
Ufactor = spartan.eager(spartan.rand(U, r).astype(np.float32))
V = spartan.sparse_empty((U, M),
tile_hint=(divup(U, d), divup(M, d)),
dtype=np.float32)
# V = spartan.shuffle(V, netflix.load_netflix_mapper,
# kw={ 'load_file' : '/big1/netflix.zip' })
V = spartan.eager(
spartan.tocoo(
spartan.shuffle(V,
netflix.fake_netflix_mapper,
target=V,
kw={'p_rating': P_RATING})))
for i in range(2):
_ = netflix.sgd(V, Mfactor, Ufactor).force()
def benchmark_netflix_sgd(ctx, timer):
d = ctx.num_workers
V = spartan.sparse_empty((U, M), tile_hint=(divup(U, d), divup(M, d)), dtype=np.float32)
V = timer.time_op(
"prep",
lambda: spartan.eager(
spartan.tocoo(spartan.shuffle(V, netflix.fake_netflix_mapper, target=V, kw={"p_rating": P_RATING}))
),
)
# V = spartan.shuffle(V, netflix.load_netflix_mapper,
# kw={ 'load_file' : '/big1/netflix.zip' })
for r in [25, 50]:
Mfactor = spartan.eager(spartan.rand(M, r, tile_hint=(divup(M, d), r)).astype(np.float32))
Ufactor = spartan.eager(spartan.rand(U, r, tile_hint=(divup(U, d), r)).astype(np.float32))
timer.time_op("rank %d" % r, netflix.sgd(V, Mfactor, Ufactor).force)