"""

Load DataFrames that describe the variables in the shared latent feature space

:param fp: Filepath to load from

:return: Tuple of two DataFrames or None if DataFrames could not be found/loaded

"""

if fp and os.path.isfile(fp):

with open(fp, 'rb') as f:

res = pickle.load(f)

else:

res = None

"""

Save DataFrames that describe the variables in the shared latent feature space

:param dfs: Length-2 iterable of DataFrames

:param fp: Filepath to save to

"""

with open(fp, 'wb') as f:

"""

Get DataFrames that describe the variables in the shared latent feature space

:param ratings: Series of ratings that has a multiindex w/ levels (user, item)

:param fp: Filepath to load DataFrames from

:param n_latent_features: Dimensionality of latent feature space

:return: Tuple of two DataFrames respectively describing users and items in the latent feature space

"""

res = load_var_features_dfs(fp)

if res:

return res

# Do Glorot initialization for features

try:

user_index = sorted(ratings.index.levels[0])

item_index = sorted(ratings.index.levels[1])

except AttributeError:

user_index = [0]

item_index = ratings.index

shape_user = (len(user_index), n_latent_features)

shape_item = (len(item_index), n_latent_features)

scale_user = np.sqrt(2. / (sum(shape_user)))

scale_item = np.sqrt(2. / (sum(shape_item)))

user_vals = np.random.randn(*shape_user) * scale_user

item_vals = np.random.randn(*shape_item) * scale_item

# Add an extra column for the biases

user_vals = np.concatenate((np.zeros((user_vals.shape[0], 1)), user_vals), axis=1)

item_vals = np.concatenate((np.zeros((item_vals.shape[0], 1)), item_vals), axis=1)

# Return DataFrames holding the features

users = pd.DataFrame(user_vals, index=user_index, dtype=np.float32)

items = pd.DataFrame(item_vals, index=item_index, dtype=np.float32)

"""

Build MF model in theano

:param reg_constant: Regularization constant

:param var1_name: Name of first variable (e.g. users)

:param var2_name: Name of second variable (e.g. items)

:return: theano function implementing MF model

"""

ratings = T.vector('ratings')

var1_vector = T.vector('{}_vector'.format(var1_name))

var2_matrix = T.matrix('{}_matrix'.format(var2_name))

predictions = T.dot(var2_matrix[:, 1:], var1_vector[1:]) + var2_matrix[:, 0] + var1_vector[0]

prediction_error = ((ratings - predictions) ** 2).sum()

l2_penalty = (var1_vector ** 2).sum() + (var2_matrix ** 2).sum().sum()

cost = prediction_error + reg_constant * l2_penalty

var1_grad, var2_grad = T.grad(cost, [var1_vector, var2_matrix])

var1_grad /= var2_matrix.shape[0]

f = theano.function(inputs=[ratings, var1_vector, var2_matrix], outputs=[cost, var1_grad, var2_grad])

"""

Train the MF model

:param f: theano model

:param data: Series of ratings that has a multiindex w/ levels (user, item)

:param var1_all: DataFrame of either users or items in latent feature space

:param var2_all: DataFrame of either users or items in latent feature space

:param learning_rate: Learning rate used for SGD

:param level: Level of data's multiindex to groupby on for training

"""

for (var1_idx, ratings_series) in data.groupby(level=level):

var2_idxs = ratings_series.index.get_level_values(1 - level)

var1 = var1_all.loc[var1_idx]

var2 = var2_all.loc[var2_idxs]

c, g1, g2 = f(ratings_series, var1, var2)

# TODO: implement momentum

var1_all.loc[var1_idx] -= learning_rate * g1

"""

Validate model

:param data: Series of ratings that has a multiindex w/ levels (user, item)

:param users_all: DataFrame of all users in latent feature space

:param items_all: DataFrame of all items in latent feature space

:return: Error averaged over validation data

"""

# This can take very long time if |users| >> |items| in data

errors = []

for (user_idx, user_ratings) in data.groupby(level=0):

item_idxs = user_ratings.index.get_level_values(1).values

user = users_all.loc[user_idx]

items = items_all.loc[item_idxs]

# much faster than using pandas operations because data already aligned

user_predictions = np.dot(items.iloc[:, 1:].values, user[1:].values) + items.iloc[:, 0].values + user[0]

user_errors = (user_ratings - user_predictions).abs()

errors.extend(user_errors)

error = np.mean(errors)

"""

Get user representation in latent feature space

:param ratings: Series of item ratings made by user

:param items_all: Series of all items

:param n_latent_features: Number of latent features

:param learning_rate: Learning rate used for SGD

:return:

"""

user, _ = get_latent_feature_dfs(ratings=ratings, n_latent_features=n_latent_features)

user = user.iloc[0]

ratings = ratings[~ratings.isnull() & ratings > 0]

items = items_all.loc[ratings.index]

f = build_model()

while True:

try:

c, ug, ig = f(ratings, user, items)

user -= learning_rate * ug

except KeyboardInterrupt:

"""

Get user ratings

:param s: Series of item identifiers

:param save_fp: Filepath to save/load to

:param load_only: Boolean indicating whether or not to only load (as opposed to asking for more ratings)

:return: Series of item ratings

"""

if save_fp and os.path.isfile(save_fp):

with open(save_fp, 'rb') as f:

user_ratings = pickle.load(f)

else:

user_ratings = pd.Series(index=s.index)

if load_only:

return user_ratings

unrated = s[user_ratings.isnull()]

possible_ratings = {0.5 * i for i in range(11)}

print('\nEnter 0 or \\n for unknown rating')

print('Press ctrl-c to stop\n')

try:

for idx, title in unrated.items():

rating = None

while rating not in possible_ratings:

rating = input('{} : '.format(title))

if rating == '':

rating = '0'

try:

rating = float(rating)

except ValueError:

print('Didn\'t recognize rating {}. Try again.'.format(rating))

user_ratings[idx] = rating

except KeyboardInterrupt:

pass

finally:

if save_fp:

with open(save_fp, 'wb') as f:

pickle.dump(user_ratings, f)

learning_rate=5e-4, level=0, max_epochs=1000,

min_ratings_user=0, min_ratings_item=0,

valid_frequency=0, perc_valid=0,

save_frequency=0, save_fp=None):

"""

Main loop for matrix factorization

:param f: theano model

:param data: Series of ratings that has a multiindex w/ levels (user, item)

:param users_all: DataFrame of all users in latent feature space

:param items_all: DataFrame of all items in latent feature space

:param learning_rate: Learning rate used for SGD

:param level: Level of data's multiindex to groupby on for training

:param max_epochs: Number of epochs before

:param min_ratings_user: Minimum number of ratings a user must have made

:param min_ratings_item: Minimum number of ratings an item must have

:param valid_frequency: Validation frequency (number epochs)

:param perc_valid: Percentage of eligible datapoints to validate on

:param save_frequency: Save frequency (number epochs) - if this is >0, must specify save_fp

:param save_fp: Filepath to save user and item feature matrices to

"""

if min_ratings_item:

data = data.groupby(level=1).filter(lambda s: len(s) >= min_ratings_item)

if min_ratings_user:

data = data.groupby(level=0).filter(lambda s: len(s) >= min_ratings_user)

epoch = 0

data_train, data_valid = cross_validation.train_test_split(data, test_size=perc_valid)

if level == 0:

var1_all, var2_all = users_all, items_all

elif level == 1:

var1_all, var2_all = items_all, users_all

try:

while epoch < max_epochs:

epoch += 1

logging.debug('Training on epoch %i', epoch)

train(f, data_train, var1_all, var2_all, learning_rate, level)

if valid_frequency and (epoch % valid_frequency == 0):

logging.debug('Validating on epoch %i', epoch)

error_valid = validate(data_valid, users_all, items_all)

logger.info('Validation absolute value error at epoch {}: {}'.format(epoch, error_valid))

if save_frequency and (epoch % save_frequency == 0):

save_var_features_dfs((users_all, items_all), fp=save_fp)

except KeyboardInterrupt:

logger.info('Stopping training early on epoch {}'.format(epoch))

finally: