if (argc < 2): print( 'Please give frovedis_server calling command as the first argument \n(e.g. "mpirun -np 2 -x /opt/nec/nosupport/frovedis/ve/bin/frovedis_server")' ) quit() FrovedisServer.initialize(argvs[1]) # sample 3x3 scipy csr matrix creation data = np.array([1, 2, 3, 4, 5, 6]) indices = np.array([0, 2, 2, 0, 1, 2]) indptr = np.array([0, 2, 3, 6]) mat = csr_matrix((data, indices, indptr), dtype=np.float64, shape=(3, 3)) # Creating Frovedis server side crs matrix from scipy data # "mat" can be any sparse matrix or array-like python object fmat = FrovedisCRSMatrix(mat) # Viewing the created matrix (for debugging) fmat.debug_print() # sparse to dense conversion at server side print("crs -> rowmajor") fmat.to_frovedis_rowmajor_matrix().debug_print() print("crs -> colmajor") fmat.to_frovedis_colmajor_matrix().debug_print() # Saving the created matrix fmat.save("./out/crs_3x3") # asCRS demo
desc = "Testing fit for FrovedisCRSMatrix : " # initializing the frovedis server argvs = sys.argv argc = len(argvs) if (argc < 2): print ('Please give frovedis_server calling command as the first argument\n\ (e.g. "mpirun -np 2 /opt/nec/frovedis/ve/bin/frovedis_server")') quit() FrovedisServer.initialize(argvs[1]) # sample numpy dense data (3x3) mat = np.matrix([[0, 0, 0, 0], [0, 1, 1, 1], [1, 0, 1, 0], [1, 1, 1, 0], [1, 1, 1, 1]], dtype=np.float64) mat = FrovedisCRSMatrix(mat) tsne = TSNE(n_components=2) try: tsne.fit(mat) tsne.embedding_.release() print(desc, "Passed") except: print(desc, "Failed") FrovedisServer.shut_down()
from frovedis.matrix.crs import FrovedisCRSMatrix from frovedis.mllib.cluster import KMeans import sys import numpy as np # initializing the Frovedis server argvs = sys.argv argc = len(argvs) if (argc < 2): print( 'Please give frovedis_server calling command as the first argument \n(e.g. "mpirun -np 2 -x /opt/nec/nosupport/frovedis/ve/bin/frovedis_server")' ) quit() FrovedisServer.initialize(argvs[1]) train_mat = FrovedisCRSMatrix(dtype=np.float64).load("./input/kmeans_data.txt") test_mat = FrovedisCRSMatrix(dtype=np.float64).load("./input/kmeans_tdata.txt") # creating KMeans object kmeans = KMeans(n_clusters=2, verbose=0) # fitting the training matrix on KMEans object kmeans.fit(train_mat) kmeans.debug_print() # saving the trained model kmeans.save("./out/MyKMeansModel") kmeans.release() # loading the same trained model kmeans.load("./out/MyKMeansModel", dtype=np.float64)
from frovedis.exrpc.server import FrovedisServer from frovedis.matrix.crs import FrovedisCRSMatrix from frovedis.mllib.fm import FactorizationMachineRegressor from frovedis.matrix.dvector import FrovedisDoubleDvector #obj: passing negative barch size per node # initializing the Frovedis server argvs = sys.argv argc = len(argvs) if (argc < 2): print( 'Please give frovedis_server calling command as the first argument \n') quit() FrovedisServer.initialize(argvs[1]) mat = FrovedisCRSMatrix(dtype=np.float64).load("./input/classification.txt") lbl = FrovedisDoubleDvector([3, 4, 3, 3, 4, 1, 5, 2, 5, 5]) # fitting input matrix and label on Factorization Machine Classifier object fm_obj = FactorizationMachineRegressor(iteration=10, init_stdev=0.1, init_learn_rate=0.1, optimizer="SGD", dim=(True, True, 8), reg=(0., 0., 0), batch_size_pernode=-1, verbose=0) try: model = fm_obj.fit(mat, lbl)
from frovedis.mllib.fm import FactorizationMachineClassifier from frovedis.mllib.fm import FactorizationMachineRegressor # initializing the Frovedis server argvs = sys.argv argc = len(argvs) if (argc < 2): print( 'Please give frovedis_server calling command as the first argument \n(e.g. "mpirun -np 2 /opt/nec/frovedis/ve/bin/frovedis_server")' ) quit() FrovedisServer.initialize(argvs[1]) mat = np.random.randint(5, size=(6, 50)) mat = np.asmatrix(mat, dtype=np.float64) cmat = FrovedisCRSMatrix(mat, dtype=np.float64) lbl = np.array([1, 1, -1, 1, -1, 1], dtype=np.float64) # fitting input matrix and label on Factorization Machine Classifier object fmc = FactorizationMachineClassifier(iteration=100, init_stdev=0.1, init_learn_rate=0.01, optimizer="SGD", dim=(True, True, 8), reg=(False, False, 0), batch_size_pernode=5000, verbose=0).fit(cmat, lbl) # prediction on created model print("predicting on factorization machine classifier model: ") print(fmc.predict(cmat))
import sys from frovedis.exrpc.server import FrovedisServer from frovedis.matrix.crs import FrovedisCRSMatrix from frovedis.mllib.recommendation import ALS # initializing the Frovedis server argvs = sys.argv argc = len(argvs) if (argc < 2): print( 'Please give frovedis_server calling command as the first argument \n(e.g. "mpirun -np 2 -x /opt/nec/nosupport/frovedis/ve/bin/frovedis_server")' ) quit() FrovedisServer.initialize(argvs[1]) mat = FrovedisCRSMatrix().load("./input/libSVMFile.txt") # fitting the input matrix on a ALS object als = ALS().fit(mat, rank=4) # ids (userId or prodId) start with 0 print(als.predict([(1, 1), (0, 1), (2, 3), (3, 1)])) # saving the model als.save("./out/MyMFModel") # loading the same model als.load("./out/MyMFModel") # recommend 2 users for 2nd product print(als.recommend_users(1, 2))
if (argc < 2): print( 'Please give frovedis_server calling command as the first argument \n(e.g. "mpirun -np 2 -x /opt/nec/nosupport/frovedis/ve/bin/frovedis_server")' ) quit() FrovedisServer.initialize(argvs[1]) corpus = [ 'This is the first document.', 'This document is the second document.', 'And this is the third one.', 'Is this the first document?', ] vectorizer = CountVectorizer() mat = vectorizer.fit_transform(corpus) fmat = FrovedisCRSMatrix(mat, dtype=np.int32, itype=np.int64) #fmat.save('input_corpus_crs_001') test_corpus = [ 'This is the first second third document.', 'This This one third document.' ] vectorizer = CountVectorizer() mat_01 = vectorizer.fit_transform(test_corpus) fmat_01 = FrovedisCRSMatrix(mat_01, dtype=np.int32, itype=np.int64) #fmat_01.save('test_corpus_crs_001') lda = LatentDirichletAllocation(n_components=4, max_iter=5, doc_topic_prior=0.25, algorithm='original',
from frovedis.exrpc.server import FrovedisServer from frovedis.matrix.crs import FrovedisCRSMatrix from frovedis.matrix.dvector import FrovedisDvector from frovedis.mllib.svm import LinearSVC import numpy as np import sys # initializing the Frovedis server argvs = sys.argv argc = len(argvs) if (argc < 2): print ('Please give frovedis_server calling command as the first argument \n(e.g. "mpirun -np 2 -x /opt/nec/nosupport/frovedis/ve/bin/frovedis_server")') quit() FrovedisServer.initialize(argvs[1]) mat = FrovedisCRSMatrix(dtype=np.float64).load("./input/libSVMFile.txt") lbl = FrovedisDvector([1,0,1,1,1,0,1,1],dtype=np.float64) # fitting input matrix and label on linear svm object svm = LinearSVC(solver='lbfgs',verbose=0).fit(mat,lbl) # predicting on loaded model print("predicting on lbfgs svm regression model: ") print(svm.predict(mat)) # fitting input matrix and label on linear svm object svm = LinearSVC(solver='sag',verbose=0).fit(mat,lbl) # predicting on loaded model print("predicting on sgd svm regression model: ") print(svm.predict(mat))
from frovedis.exrpc.server import FrovedisServer from frovedis.matrix.crs import FrovedisCRSMatrix from frovedis.mllib.naive_bayes import MultinomialNB from frovedis.mllib.naive_bayes import BernoulliNB #Objective: Run without error # initializing the Frovedis server argvs = sys.argv argc = len(argvs) if (argc < 2): print 'Please give frovedis_server calling command as the first argument \n(e.g. "mpirun -np 2 /opt/nec/frovedis/ve/bin/frovedis_server")' quit() FrovedisServer.initialize(argvs[1]) mat = FrovedisCRSMatrix(dtype=np.float64).load("./input/multi.txt") lbl = np.array([1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0]) nbm = MultinomialNB(alpha=1.0, fit_prior=True, class_prior=None, verbose=0).fit(mat, lbl) nbm.debug_print() print("predicting on nbm multinomial classifier model: ") mnb = nbm.predict(mat) print(mnb) print("Accuracy of model: ", nbm.score(mat, lbl)) ret1 = (mnb == lbl).all() mat = FrovedisCRSMatrix(dtype=np.float64).load("./input/bern.txt") mat.debug_print() lbl = np.array([1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 1.0, 1.0]) nbm2 = BernoulliNB(alpha=1.0, fit_prior=True,
from math import sqrt from sklearn.metrics import mean_squared_error from frovedis.exrpc.server import FrovedisServer from frovedis.matrix.crs import FrovedisCRSMatrix from frovedis.matrix.dvector import FrovedisDoubleDvector from frovedis.mllib.fm import FactorizationMachineRegressor # initializing the Frovedis server argvs = sys.argv argc = len(argvs) if (argc < 2): print ('Please give frovedis_server calling command as the first argument \n') quit() FrovedisServer.initialize(argvs[1]) mat = FrovedisCRSMatrix(dtype=np.float64).load("./input/regression.txt") lbl = FrovedisDoubleDvector([3,4,3,3,4,1,5,2,5,5]) # fitting input matrix and label on Factorization Machine Classifier object fm_obj = FactorizationMachineRegressor(iteration = 100, init_stdev = 0.1, init_learn_rate = 0.1, optimizer="SGD", dim = (True, True, 8), reg = (0., 0., 0), batch_size_pernode = 1, verbose = 0) # prediction on created model model = fm_obj.fit(mat, lbl) pr_res = model.predict(mat) # saving the model model.save("./out/FMCModel1")
import os import numpy as np from scipy.sparse import csr_matrix from frovedis.exrpc.server import FrovedisServer from frovedis.matrix.crs import FrovedisCRSMatrix # sample 3x3 scipy csr matrix creation data = np.array([1, 2, 3, 4, 5, 6]) indices = np.array([0, 2, 2, 0, 1, 2]) indptr = np.array([0, 2, 3, 6]) mat = csr_matrix((data, indices, indptr), dtype=np.float64, shape=(3, 3)) FrovedisServer.initialize("mpirun -np 2 {}".format( os.environ['FROVEDIS_SERVER'])) print("-- matrix created from scipy csr matrix --") fmat = FrovedisCRSMatrix(mat) fmat.debug_print() fmat.save("./result") fmat.release() print("\n-- matrix loaded from text file --") fmat2 = FrovedisCRSMatrix().load_text("./result") fmat2.debug_print() fmat2.release() FrovedisServer.shut_down()