def create_w2v_embed(df): ''' We will generate feature vectors using Frovedis Word2Vec for review text. ''' os.environ["VE_OMP_NUM_THREADS"] = '8' FrovedisServer.initialize("mpirun -np 1 " + os.environ["FROVEDIS_SERVER"]) frovedis_w2v = Frovedis_Word2Vec(sentences=list(df["Review"]), hiddenSize=512, minCount=2, n_iter=100) x_emb = frovedis_w2v.transform(list(df["Review"]), func=np.mean) os.environ["VE_OMP_NUM_THREADS"] = '1' FrovedisServer.shut_down() return pd.DataFrame(x_emb)
def run_frovedis(params, X, nproc): from frovedis.exrpc.server import FrovedisServer from frovedis.matrix.wrapper import ARPACK FrovedisServer.initialize("mpirun -np {nproc} {server}".format( nproc=nproc, server=os.environ['FROVEDIS_SERVER'])) start = time.time() clf = ARPACK.computeSVD(X, params["n_components"]) end = time.time() clf.release() FrovedisServer.shut_down() return end - start
def run_frovedis(params, X_train, y_train, X_test, y_test, nproc): from frovedis.exrpc.server import FrovedisServer from frovedis.mllib.linear_model import LogisticRegression FrovedisServer.initialize("mpirun -np {nproc} {server}".format( nproc=nproc, server=os.environ['FROVEDIS_SERVER'])) start = time.time() clf = LogisticRegression(**params).fit(X_train, y_train) end = time.time() y_pred = clf.predict(X_test) score = 1.0 * sum(y_test == y_pred) / len(y_test) clf.release() FrovedisServer.shut_down() return score, end - start
def run_frovedis(params, X, nproc): from frovedis.exrpc.server import FrovedisServer from frovedis.mllib.cluster import KMeans FrovedisServer.initialize( "mpirun -np {nproc} {server}".format( nproc=nproc, server=os.environ['FROVEDIS_SERVER'] ) ) start = time.time() clf = KMeans(**params).fit(X) end = time.time() clf.release() FrovedisServer.shut_down() return end - start
import sys import numpy as np from frovedis.exrpc.server import FrovedisServer from frovedis.linalg import eigsh desc = "Testing eigsh() for int32 numpy array: " # initializing the Frovedis server argvs = sys.argv argc = len(argvs) if argc < 2: print( 'Please give frovedis_server calling command as the first argument \n' '(e.g. "mpirun -np 2 /opt/nec/frovedis/ve/bin/frovedis_server")') quit() FrovedisServer.initialize(argvs[1]) # sample numpy array square symmetric dense data (6x6) mat = np.asarray( [[2, -1, 0, 0, -1, 0], [-1, 3, -1, 0, -1, 0], [0, -1, 2, -1, 0, 0], [0, 0, -1, 3, -1, -1], [-1, -1, 0, -1, 3, 0], [0, 0, 0, -1, 0, 1]], dtype=np.int32) try: eigen_vals, eigen_vecs = eigsh(mat, k=3) print(desc, "Passed") except: print(desc, "Failed") FrovedisServer.shut_down()
import os import numpy as np from frovedis.exrpc.server import FrovedisServer from frovedis.matrix.dense import FrovedisBlockcyclicMatrix from frovedis.matrix.wrapper import PBLAS FrovedisServer.initialize("mpirun -np 2 {}".format(os.environ['FROVEDIS_SERVER'])) # numpy matrices creation x = np.matrix([[1],[2],[3],[4]], dtype=np.float64) # 4x1 y = np.matrix([[5],[6],[7],[8]], dtype=np.float64) # 4x1 m = np.matrix([[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]], dtype=np.float64) # 4x4: eye(I) n = np.matrix([[1,2,3,4],[5,6,7,8],[8,7,6,5],[4,3,2,1]], dtype=np.float64) # 4x4 # Creating Frovedis server side blockcyclic matrics from numpy matrices bcx = FrovedisBlockcyclicMatrix(x) # blockcyclic vector (x) bcy = FrovedisBlockcyclicMatrix(y) # blockcyclic vector (y) bcm = FrovedisBlockcyclicMatrix(m) # blockcyclic matrix (m) bcn = FrovedisBlockcyclicMatrix(n) # blockcyclic matrix (n) # --- print original data print ("x:") print (x) print ("y:") print (y) print ("m:") print (m) print ("m:") print (n)
estimator_name.append(estimator_nm) start_time = time.time() estimator.fit(x_train, y_train) train_time.append(round(time.time() - start_time, 4)) start_time = time.time() train_score.append(estimator.score(x_train, y_train)) test_score.append(estimator.score(x_test, y_test)) test_time.append(round(time.time() - start_time, 4)) #3.1 LinearRegression TARGET = "lnr" FrovedisServer.initialize("mpirun -np 8 " + os.environ["FROVEDIS_SERVER"]) f_est = fLNR() E_NM = TARGET + "_frovedis_" + frovedis.__version__ evaluate(f_est, E_NM, x_train, y_train, x_test, y_test) f_est.release() FrovedisServer.shut_down() s_est = sLNR() E_NM = TARGET + "_sklearn_" + sklearn.__version__ evaluate(s_est, E_NM, x_train, y_train, x_test, y_test) #3.2 SGDRegressor TARGET = "sgd" FrovedisServer.initialize("mpirun -np 8 " + os.environ["FROVEDIS_SERVER"]) f_est = fSGDReg(loss="squared_loss", penalty="l2", eta0=0.00001)