class FWLSCalculator(object):
def __init__(self, data_size=1000):
self.logger = logging.getLogger('FWLS')
self.train_data = None
self.test_data = None
self.rating_count = None
self.cb = ContentBasedRecs()
self.cf = NeighborhoodBasedRecs()
self.fwls = FeatureWeightedLinearStacking()
self.data_size = data_size
def get_real_training_data(self):
columns = ['user_id', 'movie_id', 'rating', 'type']
ratings_data = Rating.objects.all().values(*columns)[:self.data_size]
df = pd.DataFrame.from_records(ratings_data, columns=columns)
self.train_data, self.test_data = train_test_split(df, test_size=0.2)
self.logger.debug("training data loaded {}".format(len(ratings_data)))
def calculate_predictions_for_training_data(self):
self.logger.debug("[BEGIN] getting predictions")
self.train_data['cb'] = self.train_data.apply(
lambda data: self.cb.predict_score(data['user_id'], data['movie_id'
]),
axis=1)
self.train_data['cf'] = self.train_data.apply(
lambda data: self.cf.predict_score(data['user_id'], data['movie_id'
]),
axis=1)
self.logger.debug("[END] getting predictions")
return None
def calculate_feature_functions_for_training_data(self):
self.logger.debug("[BEGIN] calculating functions")
self.train_data['cb1'] = self.train_data.apply(
lambda data: data['cb'] * self.fwls.fun1(), axis=1)
self.train_data['cb2'] = self.train_data.apply(
lambda data: data['cb'] * self.fwls.fun2(data['user_id']), axis=1)
self.train_data['cf1'] = self.train_data.apply(
lambda data: data['cf'] * self.fwls.fun1(), axis=1)
self.train_data['cf2'] = self.train_data.apply(
lambda data: data['cf'] * self.fwls.fun2(data['user_id']), axis=1)
self.logger.debug("[END] calculating functions")
return None
def train(self):
#model = sm.ols(formula="rating ~ cb1+cb2+cf1+cf2", data=self.train_data[['rating', 'cb1','cb2','cf1','cf2']])
#results = model.fit()
#self.logger.info(results.summary())
#self.logger.info(results.params)
regr = linear_model.LinearRegression()
regr.fit(self.train_data[['cb1', 'cb2', 'cf1', 'cf2']],
self.train_data['rating'])
self.logger.info(regr.coef_)
return regr.coef_
def __init__(self, data_size=1000):
self.logger = logging.getLogger('FWLS')
self.train_data = None
self.test_data = None
self.rating_count = None
self.cb = ContentBasedRecs()
self.cf = NeighborhoodBasedRecs()
self.fwls = FeatureWeightedLinearStacking()
self.data_size = data_size
def test_recFWLS(self):
save_path = './models/fwls/'
builder = FWLSCalculator(save_path)
train_data = self.get_training_data()
builder.train_data = train_data
parameters = builder.train()
assert(parameters is not None)
rec = FeatureWeightedLinearStacking()
rec.set_save_path(save_path)
assert(rec.wcb1 == parameters['cb1'])
assert(rec.wcb2 == parameters['cb2'])
assert(rec.wcf1 == parameters['cf1'])
assert(rec.wcf2 == parameters['cf2'])
def recs_fwls(request, user_id, num=6):
sorted_items = FeatureWeightedLinearStacking().recommend_items(user_id, num)
data = {
'user_id': user_id,
'data': sorted_items
}
return JsonResponse(data, safe=False)
def __init__(self, save_path, data_size=1000):
self.save_path = save_path
self.logger = logging.getLogger('FWLS')
self.train_data = None
self.test_data = None
self.rating_count = None
self.cb = ContentBasedRecs()
self.cf = NeighborhoodBasedRecs()
self.fwls = FeatureWeightedLinearStacking()
self.data_size = data_size
def evaluate_fwls_recommender():
save_path = './models/fwls/'
min_number_of_ratings = 10
min_overlap = 3
min_sim = 0.1
K = 5
min_rank = 5
number_test_users = 1000
timestr = time.strftime("%Y%m%d-%H%M%S")
file_name = '{}-fwls.csv'.format(timestr)
with open(file_name, 'a', 1) as logfile:
logfile.write(
"ar, map, mae, min_overlap, min_sim, K, min_num_of_ratings, min_rank, data_sample\n"
)
builder = FWLSCalculator(min_overlap, save_path)
for data_sample in np.arange(500, 5000, 200):
min_rank = min_number_of_ratings / 2
recommender = FeatureWeightedLinearStacking()
er = EvaluationRunner(0,
builder,
recommender,
K,
params={
'save_path': save_path,
'data_sample': data_sample
})
result = er.calculate(min_number_of_ratings, min_rank,
number_test_users)
map = result['map']
mae = result['mae']
ar = result['ar']
logfile.write("{}, {}, {}, {}, {}, {}, {}, {}, {}\n".format(
ar, map, mae, min_overlap, min_sim, K, min_number_of_ratings,
min_rank, data_sample))
logfile.flush()
def evaluate_fwls_recommender(coverage=False):
save_path = './models/fwls/'
min_number_of_ratings = 10
min_overlap = 3
min_sim = 0.1
K = 5
min_rank = 5
timestr = time.strftime("%Y%m%d-%H%M%S")
file_name = '{}-fwls.csv'.format(timestr)
with open(file_name, 'a', 1) as logfile:
logfile.write(
"ar, map, mae, min_overlap, min_sim, K, min_num_of_ratings, min_rank, user_coverage, "
"movie_coverage\n")
builder = FWLSCalculator(min_overlap, save_path)
for k in np.arange(2, 20, 2):
min_rank = min_number_of_ratings / 2
recommender = FeatureWeightedLinearStacking()
er = EvaluationRunner(0,
builder,
recommender,
k,
params={'save_path': save_path})
result = er.calculate(min_number_of_ratings, min_rank)
user_coverage, movie_coverage = 0, 0
if coverage:
user_coverage, movie_coverage = RecommenderCoverage(
recommender).calculate_coverage(k)
builder = None
map = result['map']
mae = result['mae']
ar = result['ar']
logfile.write("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}\n".format(
ar, map, mae, min_overlap, min_sim, k, min_number_of_ratings,
min_rank, user_coverage, movie_coverage))
logfile.flush()
class FWLSCalculator(object):
def __init__(self, save_path, data_size=1000):
self.save_path = save_path
self.logger = logging.getLogger('FWLS')
self.train_data = None
self.test_data = None
self.rating_count = None
self.cb = ContentBasedRecs()
self.cf = NeighborhoodBasedRecs()
self.fwls = FeatureWeightedLinearStacking()
self.data_size = data_size
def get_real_training_data(self):
columns = ['user_id', 'movie_id', 'rating', 'type']
ratings_data = Rating.objects.all().values(*columns)[:self.data_size]
df = pd.DataFrame.from_records(ratings_data, columns=columns)
self.train_data, self.test_data = train_test_split(df, test_size=0.2)
self.logger.debug("training data loaded {}".format(len(ratings_data)))
def calculate_predictions_for_training_data(self):
self.logger.debug("[BEGIN] getting predictions")
self.train_data['cb'] = self.train_data.apply(lambda data:
self.cb.predict_score(data['user_id'],
data['movie_id']), axis=1)
self.train_data['cf'] = self.train_data.apply(lambda data:
self.cf.predict_score(data['user_id'],
data['movie_id']), axis=1)
self.logger.debug("[END] getting predictions")
return None
def calculate_feature_functions_for_training_data(self):
self.logger.debug("[BEGIN] calculating functions")
self.train_data['cb1'] = self.train_data.apply(lambda data:
data['cb'] * self.fwls.fun1(), axis=1)
self.train_data['cb2'] = self.train_data.apply(lambda data:
data['cb'] * self.fwls.fun2(data['user_id']), axis=1)
self.train_data['cf1'] = self.train_data.apply(lambda data:
data['cf'] * self.fwls.fun1(), axis=1)
self.train_data['cf2'] = self.train_data.apply(lambda data:
data['cf'] * self.fwls.fun2(data['user_id']), axis=1)
self.logger.debug("[END] calculating functions")
return None
def build(self, train_data=None, params=None):
if params:
self.save_path = params['save_path']
self.data_size = params['data_sample']
if train_data is not None:
self.train_data = train_data
if self.data_size > 0:
self.train_data = self.train_data.sample(self.data_size)
self.logger.debug("training sample of size {}".format(self.train_data.shape[0]))
else:
self.get_real_training_data()
self.calculate_predictions_for_training_data()
self.calculate_feature_functions_for_training_data()
return self.train()
def train(self, ratings=None, train_feature_recs=False):
if train_feature_recs:
ItemSimilarityMatrixBuilder().build(ratings)
LdaModel.build()
regr = linear_model.LinearRegression(fit_intercept=True,
n_jobs=-1,
normalize=True)
regr.fit(self.train_data[['cb1', 'cb2', 'cf1', 'cf2']], self.train_data['rating'])
self.logger.info(regr.coef_)
result = {'cb1': regr.coef_[0],
'cb2': regr.coef_[1],
'cf1': regr.coef_[2],
'cf2': regr.coef_[3],
'intercept': regr.intercept_}
self.logger.debug(result)
self.logger.debug(self.train_data.iloc[100])
ensure_dir(self.save_path)
with open(self.save_path + 'fwls_parameters.data', 'wb') as ub_file:
pickle.dump(result, ub_file)
return result
class FWLSCalculator(object):
def __init__(self, save_path, data_size = 1000):
self.save_path = save_path
self.logger = logging.getLogger('FWLS')
self.train_data = None
self.test_data = None
self.rating_count = None
self.cb = ContentBasedRecs()
self.cf = NeighborhoodBasedRecs()
self.fwls = FeatureWeightedLinearStacking()
self.data_size = data_size
def get_real_training_data(self):
columns = ['user_id', 'movie_id', 'rating', 'type']
ratings_data = Rating.objects.all().values(*columns)[:self.data_size]
df = pd.DataFrame.from_records(ratings_data, columns=columns)
self.train_data, self.test_data = train_test_split(df, test_size=0.2)
self.logger.debug("training data loaded {}".format(len(ratings_data)))
def calculate_predictions_for_training_data(self):
self.logger.debug("[BEGIN] getting predictions")
self.train_data['cb'] = self.train_data.apply(lambda data:
self.cb.predict_score(data['user_id'], data['movie_id']), axis=1)
self.train_data['cf'] = self.train_data.apply(lambda data:
self.cf.predict_score(data['user_id'], data['movie_id']), axis=1)
self.logger.debug("[END] getting predictions")
return None
def calculate_feature_functions_for_training_data(self):
self.logger.debug("[BEGIN] calculating functions")
self.train_data['cb1'] = self.train_data.apply(lambda data:
data['cb'] * self.fwls.fun1(), axis=1)
self.train_data['cb2'] = self.train_data.apply(lambda data:
data['cb'] * self.fwls.fun2(data['user_id']), axis = 1)
self.train_data['cf1'] = self.train_data.apply(lambda data:
data['cf'] * self.fwls.fun1(), axis=1)
self.train_data['cf2'] = self.train_data.apply(lambda data:
data['cf'] * self.fwls.fun2(data['user_id']), axis = 1)
self.logger.debug("[END] calculating functions")
return None
def build(self, train_data = None, params = None):
if params:
self.save_path = params['save_path']
if train_data is None:
self.get_real_training_data()
self.train_data = train_data
self.calculate_predictions_for_training_data()
self.calculate_feature_functions_for_training_data()
return self.train()
def train(self, ratings = None, train_feature_recs= False):
if train_feature_recs:
ItemSimilarityMatrixBuilder().build(ratings)
LdaModel.build()
regr = linear_model.LinearRegression()
regr.fit(self.train_data[['cb1','cb2','cf1','cf2']], self.train_data['rating'])
self.logger.info(regr.coef_)
result = {'cb1': regr.coef_[0],
'cb2': regr.coef_[1],
'cf1': regr.coef_[2],
'cf2': regr.coef_[3]
}
ensure_dir(self.save_path)
with open(self.save_path + 'fwls_parameters.data', 'wb') as ub_file:
pickle.dump(result, ub_file)
return result