def create_w2v_embed(df):
'''
We will generate feature vectors using Frovedis Word2Vec for review text.
'''
os.environ["VE_OMP_NUM_THREADS"] = '8'
FrovedisServer.initialize("mpirun -np 1 " + os.environ["FROVEDIS_SERVER"])
frovedis_w2v = Frovedis_Word2Vec(sentences=list(df["Review"]),
hiddenSize=512,
minCount=2,
n_iter=100)
x_emb = frovedis_w2v.transform(list(df["Review"]), func=np.mean)
os.environ["VE_OMP_NUM_THREADS"] = '1'
FrovedisServer.shut_down()
return pd.DataFrame(x_emb)
def run_frovedis(params, X, nproc):
from frovedis.exrpc.server import FrovedisServer
from frovedis.matrix.wrapper import ARPACK
FrovedisServer.initialize("mpirun -np {nproc} {server}".format(
nproc=nproc, server=os.environ['FROVEDIS_SERVER']))
start = time.time()
clf = ARPACK.computeSVD(X, params["n_components"])
end = time.time()
clf.release()
FrovedisServer.shut_down()
return end - start
def run_frovedis(params, X_train, y_train, X_test, y_test, nproc):
from frovedis.exrpc.server import FrovedisServer
from frovedis.mllib.linear_model import LogisticRegression
FrovedisServer.initialize("mpirun -np {nproc} {server}".format(
nproc=nproc, server=os.environ['FROVEDIS_SERVER']))
start = time.time()
clf = LogisticRegression(**params).fit(X_train, y_train)
end = time.time()
y_pred = clf.predict(X_test)
score = 1.0 * sum(y_test == y_pred) / len(y_test)
clf.release()
FrovedisServer.shut_down()
return score, end - start
def run_frovedis(params, X, nproc):
from frovedis.exrpc.server import FrovedisServer
from frovedis.mllib.cluster import KMeans
FrovedisServer.initialize(
"mpirun -np {nproc} {server}".format(
nproc=nproc,
server=os.environ['FROVEDIS_SERVER']
)
)
start = time.time()
clf = KMeans(**params).fit(X)
end = time.time()
clf.release()
FrovedisServer.shut_down()
return end - start
import sys
import numpy as np
from frovedis.exrpc.server import FrovedisServer
from frovedis.linalg import eigsh
desc = "Testing eigsh() for int32 numpy array: "
# initializing the Frovedis server
argvs = sys.argv
argc = len(argvs)
if argc < 2:
print(
'Please give frovedis_server calling command as the first argument \n'
'(e.g. "mpirun -np 2 /opt/nec/frovedis/ve/bin/frovedis_server")')
quit()
FrovedisServer.initialize(argvs[1])
# sample numpy array square symmetric dense data (6x6)
mat = np.asarray(
[[2, -1, 0, 0, -1, 0], [-1, 3, -1, 0, -1, 0], [0, -1, 2, -1, 0, 0],
[0, 0, -1, 3, -1, -1], [-1, -1, 0, -1, 3, 0], [0, 0, 0, -1, 0, 1]],
dtype=np.int32)
try:
eigen_vals, eigen_vecs = eigsh(mat, k=3)
print(desc, "Passed")
except:
print(desc, "Failed")
FrovedisServer.shut_down()
import os
import numpy as np
from frovedis.exrpc.server import FrovedisServer
from frovedis.matrix.dense import FrovedisBlockcyclicMatrix
from frovedis.matrix.wrapper import PBLAS
FrovedisServer.initialize("mpirun -np 2 {}".format(os.environ['FROVEDIS_SERVER']))
# numpy matrices creation
x = np.matrix([[1],[2],[3],[4]], dtype=np.float64) # 4x1
y = np.matrix([[5],[6],[7],[8]], dtype=np.float64) # 4x1
m = np.matrix([[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]],
dtype=np.float64) # 4x4: eye(I)
n = np.matrix([[1,2,3,4],[5,6,7,8],[8,7,6,5],[4,3,2,1]],
dtype=np.float64) # 4x4
# Creating Frovedis server side blockcyclic matrics from numpy matrices
bcx = FrovedisBlockcyclicMatrix(x) # blockcyclic vector (x)
bcy = FrovedisBlockcyclicMatrix(y) # blockcyclic vector (y)
bcm = FrovedisBlockcyclicMatrix(m) # blockcyclic matrix (m)
bcn = FrovedisBlockcyclicMatrix(n) # blockcyclic matrix (n)
# --- print original data
print ("x:")
print (x)
print ("y:")
print (y)
print ("m:")
print (m)
print ("m:")
print (n)
estimator_name.append(estimator_nm)
start_time = time.time()
estimator.fit(x_train, y_train)
train_time.append(round(time.time() - start_time, 4))
start_time = time.time()
train_score.append(estimator.score(x_train, y_train))
test_score.append(estimator.score(x_test, y_test))
test_time.append(round(time.time() - start_time, 4))
#3.1 LinearRegression
TARGET = "lnr"
FrovedisServer.initialize("mpirun -np 8 " + os.environ["FROVEDIS_SERVER"])
f_est = fLNR()
E_NM = TARGET + "_frovedis_" + frovedis.__version__
evaluate(f_est, E_NM, x_train, y_train, x_test, y_test)
f_est.release()
FrovedisServer.shut_down()
s_est = sLNR()
E_NM = TARGET + "_sklearn_" + sklearn.__version__
evaluate(s_est, E_NM, x_train, y_train, x_test, y_test)
#3.2 SGDRegressor
TARGET = "sgd"
FrovedisServer.initialize("mpirun -np 8 " + os.environ["FROVEDIS_SERVER"])
f_est = fSGDReg(loss="squared_loss", penalty="l2", eta0=0.00001)
