def create_w2v_embed(df):
'''
We will generate feature vectors using Frovedis Word2Vec for review text.
'''
os.environ["VE_OMP_NUM_THREADS"] = '8'
FrovedisServer.initialize("mpirun -np 1 " + os.environ["FROVEDIS_SERVER"])
frovedis_w2v = Frovedis_Word2Vec(sentences=list(df["Review"]),
hiddenSize=512,
minCount=2,
n_iter=100)
x_emb = frovedis_w2v.transform(list(df["Review"]), func=np.mean)
os.environ["VE_OMP_NUM_THREADS"] = '1'
FrovedisServer.shut_down()
return pd.DataFrame(x_emb)
def run_frovedis(params, X, nproc):
from frovedis.exrpc.server import FrovedisServer
from frovedis.matrix.wrapper import ARPACK
FrovedisServer.initialize("mpirun -np {nproc} {server}".format(
nproc=nproc, server=os.environ['FROVEDIS_SERVER']))
start = time.time()
clf = ARPACK.computeSVD(X, params["n_components"])
end = time.time()
clf.release()
FrovedisServer.shut_down()
return end - start
def run_frovedis(params, X_train, y_train, X_test, y_test, nproc):
from frovedis.exrpc.server import FrovedisServer
from frovedis.mllib.linear_model import LogisticRegression
FrovedisServer.initialize("mpirun -np {nproc} {server}".format(
nproc=nproc, server=os.environ['FROVEDIS_SERVER']))
start = time.time()
clf = LogisticRegression(**params).fit(X_train, y_train)
end = time.time()
y_pred = clf.predict(X_test)
score = 1.0 * sum(y_test == y_pred) / len(y_test)
clf.release()
FrovedisServer.shut_down()
return score, end - start
def run_frovedis(params, X, nproc):
from frovedis.exrpc.server import FrovedisServer
from frovedis.mllib.cluster import KMeans
FrovedisServer.initialize(
"mpirun -np {nproc} {server}".format(
nproc=nproc,
server=os.environ['FROVEDIS_SERVER']
)
)
start = time.time()
clf = KMeans(**params).fit(X)
end = time.time()
clf.release()
FrovedisServer.shut_down()
return end - start
import sys
import numpy as np
from frovedis.exrpc.server import FrovedisServer
from frovedis.linalg import eigsh
desc = "Testing eigsh() for int32 numpy array: "
# initializing the Frovedis server
argvs = sys.argv
argc = len(argvs)
if argc < 2:
print(
'Please give frovedis_server calling command as the first argument \n'
'(e.g. "mpirun -np 2 /opt/nec/frovedis/ve/bin/frovedis_server")')
quit()
FrovedisServer.initialize(argvs[1])
# sample numpy array square symmetric dense data (6x6)
mat = np.asarray(
[[2, -1, 0, 0, -1, 0], [-1, 3, -1, 0, -1, 0], [0, -1, 2, -1, 0, 0],
[0, 0, -1, 3, -1, -1], [-1, -1, 0, -1, 3, 0], [0, 0, 0, -1, 0, 1]],
dtype=np.int32)
try:
eigen_vals, eigen_vecs = eigsh(mat, k=3)
print(desc, "Passed")
except:
print(desc, "Failed")
FrovedisServer.shut_down()
import os
import numpy as np
from sklearn.datasets import load_breast_cancer
from frovedis.exrpc.server import FrovedisServer # frovedis
from frovedis.mllib.linear_model import LogisticRegression # frovedis
#from sklearn.linear_model import LogisticRegression # sklearn
X, y = load_breast_cancer(return_X_y=True)
C = 10.0
max_iter = 10000
solver = "sag"
FrovedisServer.initialize("mpirun -np 4 {}".format(
os.environ['FROVEDIS_SERVER'])) # frovedis
clf = LogisticRegression(random_state=0, solver=solver, C=C,
max_iter=max_iter).fit(X, y)
y_pred = clf.predict(X)
score = 1.0 * sum(y == y_pred) / len(y)
FrovedisServer.shut_down() # frovedis
print("score: {}".format(score))
