def test_dynamic_updates(self): """ TensorCoFi dynamic update We will take a tensor cofi. Train the model, evaluate it. Then we remove all the user factors and recompute them using the online_user_factors to check if the performance is almost the same... """ pyTF = PyTensorCoFi(n_factors=20, n_iterations=5, c_lambda=0.05, c_alpha=40) evaluator = Evaluator() tf = TensorCoFi(n_factors=2, n_iterations=100, c_lambda=0.05, c_alpha=40) df = pd.read_csv(resource_filename(testfm.__name__, "data/movielenshead.dat"), sep="::", header=None, names=["user", "item", "rating", "date", "title"]) training, testing = testfm.split.holdoutByRandom(df, 0.7) users = {user: list(entries) for user, entries in training.groupby("user")["item"]} tf.fit(training) map1 = evaluator.evaluate_model(tf, testing) # map of the original model #now we try to replace the original factors with on the fly computed factors #lets iterate over the training data of items and the users for u, items in users.items(): #user id in the tf uid = tf.data_map[tf.get_user_column()][u] # user id iids = [tf.data_map[tf.get_item_column()][i] for i in items] # item ids that user has seen #original_factors = tf.factors["user"][uid] new_factors = pyTF.online_user_factors(tf.factors[1], iids, p_param=40, lambda_param=0.05) #replace original factors with the new factors tf.factors[0][uid, :] = new_factors #tf.update_user_factors(uid, new_factors) #lets evaluate the new model with real-time updated factors map2 = evaluator.evaluate_model(tf, testing) #The difference should be smaller than 20% assert abs(map1[0]-map2[0]) < 0.2*map1[0]
def tune(model,training,testing, non_relevant_count=100, **kwargs): ''' Return a mean for the predictive power ''' model.setParams(**kwargs) model.fit(training) eval = Evaluator() # Return the MAPMeasure in position 0 measure = eval.evaluate_model(model,testing, non_relevant_count=non_relevant_count)[0] print 'tried {} = {}'.format(kwargs, measure) return measure
def test_rmse(self): eval = Evaluator() model = ConstantModel(constant=3.0) testing = pd.DataFrame([{'user':10, 'item':100, 'rating':3}, {'user':10,'item':110,'rating':1}, {'user':12,'item':100,'rating':4}, ]) rmse = eval.evaluate_model_rmse(model, testing) self.assertEqual(sqrt((0+4+1)/3.0), rmse) eval.close()
def test_rmse(self): """ [EVALUATOR] Test the rmse measure """ eval = Evaluator() model = ConstantModel(constant=3.0) testing = pd.DataFrame([{"user": 10, "item": 100, "rating": 3}, {"user": 10, "item": 110, "rating": 1}, {"user": 12, "item": 100, "rating": 4}]) rmse = eval.evaluate_model_rmse(model, testing) self.assertEqual(sqrt((0+4+1)/3.0), rmse)
def tune(model, training, testing, non_relevant_count=100, **kwargs): """ Return a mean for the predictive power """ model.set_params(**kwargs) model.fit(training) evaluator = Evaluator() # Return the MAPMeasure in position 0 measure = evaluator.evaluate_model( model, testing, non_relevant_count=non_relevant_count)[0] print "tried {} = {}".format(kwargs, measure) return measure
def test_dynamic_updates(self): """ TensorCoFi dynamic update We will take a tensor cofi. Train the model, evaluate it. Then we remove all the user factors and recompute them using the online_user_factors to check if the performance is almost the same... """ pyTF = PyTensorCoFi(n_factors=20, n_iterations=5, c_lambda=0.05, c_alpha=40) evaluator = Evaluator() tf = TensorCoFi(n_factors=2, n_iterations=100, c_lambda=0.05, c_alpha=40) df = pd.read_csv(resource_filename(testfm.__name__, "data/movielenshead.dat"), sep="::", header=None, names=["user", "item", "rating", "date", "title"]) training, testing = testfm.split.holdoutByRandom(df, 0.7) users = { user: list(entries) for user, entries in training.groupby("user")["item"] } tf.fit(training) map1 = evaluator.evaluate_model(tf, testing) # map of the original model #now we try to replace the original factors with on the fly computed factors #lets iterate over the training data of items and the users for u, items in users.items(): #user id in the tf uid = tf.data_map[tf.get_user_column()][u] # user id iids = [tf.data_map[tf.get_item_column()][i] for i in items] # item ids that user has seen #original_factors = tf.factors["user"][uid] new_factors = pyTF.online_user_factors(tf.factors[1], iids, p_param=40, lambda_param=0.05) #replace original factors with the new factors tf.factors[0][uid, :] = new_factors #tf.update_user_factors(uid, new_factors) #lets evaluate the new model with real-time updated factors map2 = evaluator.evaluate_model(tf, testing) #The difference should be smaller than 20% assert abs(map1[0] - map2[0]) < 0.2 * map1[0]
def test_tensor_score_against_testfm(self): """ [recommendation.models.TensorCoFi] Test tensorcofi scores with test.fm benchmark """ evaluator = Evaluator() tc = TensorCoFi(n_users=len(self.df.user.unique()), n_items=len(self.df.item.unique()), n_factors=2) ptc = PyTensorCoFi() training, testing = testfm.split.holdoutByRandom(self.df, 0.9) items = training.item.unique() tc.fit(training) ptc.fit(training) tc_score = evaluator.evaluate_model(tc, testing, all_items=items)[0] ptc_score = evaluator.evaluate_model(ptc, testing, all_items=items)[0] assert abs(tc_score-ptc_score) < .15, \ "TensorCoFi score is not close enough to testfm benchmark (%.3f != %.3f)" % (tc_score, ptc_score)
def test_popularity_score_against_testfm(self): """ [recommendation.models.TensorCoFi] Test popularity scores with test.fm benchmark """ evaluator = Evaluator() training, testing = testfm.split.holdoutByRandom(self.df, 0.9) items = training.item.unique() tc = Popularity(len(items)) ptc = TFMPopularity() tc.fit(training) ptc.fit(training) tc_score = evaluator.evaluate_model(tc, testing, all_items=items)[0] ptc_score = evaluator.evaluate_model(ptc, testing, all_items=items)[0] assert abs(tc_score-ptc_score) < .1, \ "Popularity score is not close enough to testfm benchmark (%.3f != %.3f)" % (tc_score, ptc_score)
def test_default(self): """ [EVALUATOR] Test the measure """ mapm = MAPMeasure() model = IdModel() evaluation = Evaluator() df = pd.DataFrame({"user": [1, 1, 3, 4], "item": [1, 2, 3, 4], "rating": [5, 3, 2, 1], "date": [11, 12, 13, 14]}) e = evaluation.evaluate_model(model, df, non_relevant_count=2, measures=[mapm]) assert len(e) == 1, "Evaluator result is not what is expected" #r = mapm.measure([]) #assert not r, "MAPMeasure for empty list is not returning NAN (%f, %s)" % (r, type(r)) r = mapm.measure([(True, 0.)]) assert r == 1., "MAPMeasure for 1 entry (True, 0.) list is not returning 1. (%f)" % r r = mapm.measure([(False, 0.)]) assert r == 0., "MAPMeasure for 1 entry (False, 0.) list is not returning 0. (%f)" % r r = mapm.measure([(False, 0.01), (True, 0.00)]) assert r == 0.5, "MAPMeasure for 2 entries (False, 0.01) and (True, 0.) list is not returning 0.5 (%f)" % r r = mapm.measure([(False, 0.9), (True, 0.8), (False, 0.7), (False, 0.6), (True, 0.5), (True, 0.4), (True, 0.3), (False, 0.2), (False, 0.1), (False, 0)]) assert r == 0.4928571428571428, "Measure should be around 0.4928571428571428 (%f)" % r
def test_nogil_against_std_05(self): """ [EVALUATOR] Test the groups measure differences between python and c implementations for 5% training """ df = pd.read_csv(resource_filename(testfm.__name__, 'data/movielenshead.dat'), sep="::", header=None, names=['user', 'item', 'rating', 'date', 'title']) model = PyTensorCoFi() ev = Evaluator(False) ev_nogil = Evaluator() results = {"implementation": [], "measure": []} for i in range(SAMPLE_SIZE_FOR_TEST): training, testing = testfm.split.holdoutByRandom(df, 0.05) model.fit(training) results["implementation"].append("Cython"), results["measure"].append(ev_nogil.evaluate_model(model, testing)[0]) results["implementation"].append("Python"), results["measure"].append(ev.evaluate_model(model, testing)[0]) ##################### # ANOVA over result # ##################### assert_equality_in_groups(results, alpha=ALPHA, groups="implementation", test_var="measure")
# Tell me what models we want to evaluate models = [ RandomModel(), PopularityOkapi(hadoop_source="/data/b.ajf/hadoop1_env.sh", host="igraph-01", okapi_jar_dir="okapi/jar/", #host='54.72.18.118', user='******', #okapi_jar_dir='/Users/linas/devel/okapi/target/', #okapi_jar_base_name='okapi-0.3.2-SNAPSHOT-jar-with-dependencies.jar', #public_key_path='/Users/linas/.ssh/hack-okapi.pem' ), Popularity(normalize=False), BPROkapi(hadoop_source="/data/b.ajf/hadoop1_env.sh", host="igraph-01", okapi_jar_dir="okapi/jar/", #host='54.72.18.118', user='******', #okapi_jar_dir='/Users/linas/devel/okapi/target/', # #okapi_jar_base_name='okapi-0.3.2-SNAPSHOT-jar-with-dependencies.jar', #public_key_path='/Users/linas/.ssh/hack-okapi.pem' ) ] # Setup the environment evaluator = Evaluator() for m in models: m.fit(df) print m.get_name().ljust(50), print evaluator.evaluate_model(m, df)
sep="::", header=None, names=['user', 'item', 'rating', 'date', 'title']) print df.head() #tell me what models we want to evaluate models = [ RandomModel(), PopularityOkapi(host='54.72.18.118', username='******', #okapi_jar_dir='/Users/linas/devel/okapi/target/', #okapi_jar_base_name='okapi-0.3.2-SNAPSHOT-jar-with-dependencies.jar', public_key_path='/Users/linas/.ssh/hack-okapi.pem' ), Popularity(normalize=False), BPROkapi(host='54.72.18.118', username='******', #okapi_jar_dir='/Users/linas/devel/okapi/target/', #okapi_jar_base_name='okapi-0.3.2-SNAPSHOT-jar-with-dependencies.jar', public_key_path='/Users/linas/.ssh/hack-okapi.pem' ) ] #setup the environment evaluator = Evaluator() for m in models: m.fit(df) print m.getName().ljust(50), print evaluator.evaluate_model(m, df) evaluator.close()#need this call to clean up the worker processes
__author__ = "linas" import testfm import pandas as pd from testfm.evaluation.evaluator import Evaluator from testfm.models.baseline_model import Popularity, RandomModel, Item2Item from testfm.models.tensorcofi import PyTensorCoFi, TensorCoFi, CTensorCoFi from testfm.models.content_based import TFIDFModel, LSIModel from testfm.models.bpr import BPR from pkg_resources import resource_filename import datetime if __name__ == "__main__": evaluator = Evaluator() #prepare the data df = pd.read_csv(resource_filename(testfm.__name__, "data/movielenshead.dat"), sep="::", header=None, names=["user", "item", "rating", "date", "title"]) print df.head() training, testing = testfm.split.holdoutByRandom(df, 0.5) #tell me what models we want to evaluate models = [ RandomModel(), BPR(), TFIDFModel("title"), Popularity(), TensorCoFi(n_factors=20, n_iterations=5, c_lambda=0.05, c_alpha=40), PyTensorCoFi(n_factors=20, n_iterations=5, c_lambda=0.05, c_alpha=40), CTensorCoFi(n_factors=20, n_iterations=5, c_lambda=0.05, c_alpha=40),
__author__ = "linas" import pandas as pd import testfm from testfm.models.tensorCoFi import TensorCoFi from testfm.evaluation.evaluator import Evaluator from pkg_resources import resource_filename from testfm.evaluation.parameterTuning import ParameterTuning eval = Evaluator() # call this before loading the data to save memory (fork of process takes place) # prepare the data df = pd.read_csv( resource_filename(testfm.__name__, "data/movielenshead.dat"), sep="::", header=None, names=["user", "item", "rating", "date", "title"], ) print df.head() training, testing = testfm.split.holdoutByRandom(df, 0.9) print "Tuning the parameters." tr, validation = testfm.split.holdoutByRandom(training, 0.7) pt = ParameterTuning() pt.setMaxIterations(10) pt.setZvalue(80) tf_params = pt.getBestParams(TensorCoFi, tr, validation) print tf_params tf = TensorCoFi()
__author__ = 'linas' import pandas as pd import testfm from testfm.models.tensorcofi import PyTensorCoFi as TensorCoFi from testfm.models.bpr import BPR as TensorCoFi from testfm.evaluation.evaluator import Evaluator from pkg_resources import resource_filename from testfm.evaluation.parameter_tuning import ParameterTuning if __name__ == "__main__": eval = Evaluator( ) # Call this before loading the data to save memory (fork of process takes place) # Prepare the data df = pd.read_csv(resource_filename(testfm.__name__, 'data/movielenshead.dat'), sep="::", header=None, names=['user', 'item', 'rating', 'date', 'title']) print df.head() training, testing = testfm.split.holdoutByRandom(df, 0.9) print "Tuning the parameters." tr, validation = testfm.split.holdoutByRandom(training, 0.7) pt = ParameterTuning() pt.set_max_iterations(100) pt.set_z_value(90) tf_params = pt.get_best_params(TensorCoFi, tr, validation) print tf_params
__author__ = 'linas' import testfm import pandas as pd from testfm.evaluation.evaluator import Evaluator from testfm.models.baseline_model import Popularity, RandomModel, Item2Item from testfm.models.tensorCoFi import TensorCoFi from testfm.models.content_based import TFIDFModel, LSIModel from testfm.models.ensemble_models import LinearRank from testfm.models.bpr import BPR from pkg_resources import resource_filename #because of Global Interpreter Lock we need to initialize evaluator here (it forks processes) eval = Evaluator() #prepare the data df = pd.read_csv(resource_filename(testfm.__name__,'data/movielenshead.dat'), sep="::", header=None, names=['user', 'item', 'rating', 'date', 'title']) print df.head() training, testing = testfm.split.holdoutByRandom(df, 0.9) #tell me what models we want to evaluate models = [ RandomModel(), BPR(), TFIDFModel('title'), Popularity(), TensorCoFi(dim=20, nIter=5, lamb=0.05, alph=40, user_features=['user'], item_features=['item', 'title']), #Item2Item(), #LSIModel('title'), ]