def read_imgs_and_measurement():
slice_list = []
# read gt image addresses
gt_names = utils.all_files_under('../gt', extension='.jpg')
for idx in range(len(gt_names)):
if np.mod(idx, 300) == 0:
print('Method: {}, Measure: {}, idx: {}'.format(
args.method, args.measure, idx))
img_name = os.path.basename(gt_names[idx])
# read gt and prediction image
gt_img = cv2.imread(gt_names[idx],
cv2.IMREAD_GRAYSCALE).astype(np.float32)
pred_img = cv2.imread(
os.path.join('../{}'.format(args.method), img_name),
cv2.IMREAD_GRAYSCALE).astype(np.float32)
# calculate measurement
measure_value = 0.
if args.measure.lower() == 'mae':
measure_value = utils.mean_absoulute_error(pred_img, gt_img)
elif args.measure.lower() == 'rmse':
measure_value = utils.root_mean_square_error(pred_img, gt_img)
elif args.measure.lower() == 'psnr':
measure_value = utils.peak_signal_to_noise_ratio(pred_img, gt_img)
elif args.measure.lower() == 'ssim':
measure_value = utils.structural_similarity_index(pred_img, gt_img)
slice_list.append(SliceExample(img_name, args.method, measure_value))
return slice_list
def main(gt, method):
# read gt image addresses
gt_names = utils.all_files_under(os.path.join('../', gt), extension='.jpg')
# read prediction image addresses
filenames = utils.all_files_under(os.path.join('../', method),
extension='.jpg')
# print(methods[idx_method])
mae_method, rmse_method, psnr_method, ssim_method, pcc_method = [], [], [], [], []
for idx_name in range(len(gt_names)):
# read gt and prediction image
gt_img = cv2.imread(gt_names[idx_name],
cv2.IMREAD_GRAYSCALE).astype(np.float32)
pred_img = cv2.imread(filenames[idx_name],
cv2.IMREAD_GRAYSCALE).astype(np.float32)
# check gt and prediction image name
if gt_names[idx_name].split('p0')[-1] != filenames[idx_name].split(
'p0')[-1]:
sys.exit(' [!] Image name can not match!')
# calcualte mae and psnr
mae = utils.mean_absoulute_error(pred_img, gt_img)
rmse = utils.root_mean_square_error(pred_img, gt_img)
psnr = utils.peak_signal_to_noise_ratio(pred_img, gt_img)
ssim = utils.structural_similarity_index(pred_img, gt_img)
pcc = utils.pearson_correlation_coefficient(pred_img, gt_img)
if np.mod(idx_name, 300) == 0:
print('Method: {}, idx: {}'.format(method, idx_name))
# collect each image results
mae_method.append(mae)
rmse_method.append(rmse)
psnr_method.append(psnr)
ssim_method.append(ssim)
pcc_method.append(pcc)
# list to np.array
mae_method = np.asarray(mae_method)
rmse_method = np.asarray(rmse_method)
psnr_method = np.asarray(psnr_method)
ssim_method = np.asarray(ssim_method)
pcc_method = np.asarray(pcc_method)
print(' MAE - mean: {:.3f}, std: {:.3f}'.format(np.mean(mae_method),
np.std(mae_method)))
print('RMSE - mean: {:.3f}, std: {:.3f}'.format(np.mean(rmse_method),
np.std(rmse_method)))
print('PSNR - mean: {:.3f}, std: {:.3f}'.format(np.mean(psnr_method),
np.std(psnr_method)))
print('SSIM - mean: {:.3f}, std: {:.3f}'.format(np.mean(ssim_method),
np.std(ssim_method)))
print(' PCC - mean: {:.3f}, std: {:.3f}'.format(np.mean(pcc_method),
np.std(pcc_method)))
data_list = [mae_method, rmse_method, psnr_method, ssim_method, pcc_method]
write_to_csv(method, data_list, gt_names)
def cal_meausre(methods, measure, case_dict, num_cases_require):
mean_bar = np.zeros((len(methods), num_cases_require), dtype=np.float32)
var_bar = np.zeros((len(methods), num_cases_require), dtype=np.float32)
case_idx = 0
for key_idx in sorted(case_dict.keys()):
if case_idx < num_cases_require:
print('key_idx: {}'.format(key_idx))
img_paths = case_dict[key_idx]
measure_result = []
for method in methods:
method_measure = []
for _, img_path in enumerate(img_paths):
gt_img = cv2.imread(os.path.join('../gt', img_path),
cv2.IMREAD_GRAYSCALE).astype(
np.float32)
pred_img = cv2.imread(
os.path.join('../{}'.format(method), img_path),
cv2.IMREAD_GRAYSCALE).astype(np.float32)
if measure.lower() == 'mae':
measure_val = utils.mean_absoulute_error(
pred_img, gt_img)
elif measure.lower() == 'rmse':
measure_val = utils.root_mean_square_error(
pred_img, gt_img)
elif measure.lower() == 'psnr':
measure_val = utils.peak_signal_to_noise_ratio(
pred_img, gt_img)
elif measure.lower() == 'ssim':
measure_val = utils.structural_similarity_index(
pred_img, gt_img)
elif measure.lower() == 'pcc':
measure_val = utils.pearson_correlation_coefficient(
pred_img, gt_img)
else:
raise NotImplementedError
# Save one slice results
method_measure.append(measure_val)
# Save whole slice results
measure_result.append(method_measure)
measure_array = np.asarray(measure_result)
mean_bar[:, case_idx] = np.mean(measure_array, axis=1)
var_bar[:, case_idx] = np.std(measure_array, axis=1)
case_idx += 1
else:
break
return mean_bar, var_bar
def iterative_svd(args):
args.shuffle = True
kf = KFold(n_splits=args.splits_num, shuffle=args.shuffle, random_state=42)
score_lst = list()
for fold, (train_index, valid_index) in enumerate(kf.split(users)):
# Initialize matrices
train_users = users[train_index]
train_movies = movies[train_index]
train_ratings = ratings[train_index]
valid_users = users[valid_index]
valid_movies = movies[valid_index]
valid_ratings = ratings[valid_index]
A_train = np.zeros(shape=(number_of_users, number_of_movies))
A_train[:] = np.nan
for i, (user, movie) in enumerate(zip(train_users, train_movies)):
A_train[user][movie] = train_ratings[i]
# mask
unknown_train = np.isnan(A_train)
known_train = ~unknown_train
# Fill-in missing entries
X_train = pd.DataFrame(A_train, copy=True)
movie_avg_ratings = X_train.mean(axis=0, skipna=True)
A_train_average = X_train.fillna(movie_avg_ratings, axis=0)
A_train_average = A_train_average.values
model = IterativeSVD(shrinkage=args.shrinkage)
model.fit(A_train_average,
known_train,
iterations=args.iterations,
verbose=False)
preds = model.predict(valid_users, valid_movies)
score = root_mean_square_error(valid_ratings, preds)
score_lst.append(score)
print("Fold:", fold + 1, "RMSE:", score)
print('Mean CV RMSE:', np.mean(score_lst))
def embeddings(args):
kf = KFold(n_splits=args.splits_num, shuffle=args.shuffle, random_state=42)
score_lst = list()
for fold, (train_index, valid_index) in enumerate(kf.split(users)):
train_users = users[train_index]
train_movies = movies[train_index]
train_ratings = ratings[train_index]
valid_users = users[valid_index]
valid_movies = movies[valid_index]
valid_ratings = ratings[valid_index]
model = Embeddings(
number_of_users,
number_of_movies,
embeddings_size=args.embeddings_size,
dropout_embeddings=args.embeddings_dropout_embeddings,
dropout=args.embeddings_dropout)
model.fit(train_users,
train_movies,
train_ratings,
valid_users=valid_users,
valid_movies=valid_movies,
valid_ratings=valid_ratings,
epochs=args.embeddings_num_epochs,
verbose=args.verbose,
decay=args.embeddings_decay,
decay_steps=args.embeddings_decay_steps,
learning_rate=args.embeddings_learning_rate,
batch_size=args.embeddings_batch_size)
preds = model.predict(valid_users, valid_movies)
score = root_mean_square_error(valid_ratings, preds)
score_lst.append(score)
print("Fold:", fold + 1, "score:", score)
print('Mean CV RMSE:', np.mean(score_lst))
def autoencoder(args):
kf = KFold(n_splits=args.splits_num, shuffle=args.shuffle, random_state=42)
score_lst = list()
for fold, (train_index, valid_index) in enumerate(kf.split(users)):
# Initialize matrices
train_users = users[train_index]
train_movies = movies[train_index]
train_ratings = ratings[train_index]
valid_users = users[valid_index]
valid_movies = movies[valid_index]
valid_ratings = ratings[valid_index]
data_zeros = np.full((number_of_users, number_of_movies), 0)
data_mask = np.full((number_of_users, number_of_movies), 0)
for i, (user, movie) in enumerate(zip(train_users, train_movies)):
data_zeros[user][movie] = train_ratings[i]
data_mask[user][movie] = 1
model = Autoencoder(number_of_users,
number_of_movies,
layers=args.hidden_layers,
masking=args.masking)
model.fit(data_zeros,
data_mask,
valid_users=valid_users,
valid_movies=valid_movies,
valid_ratings=valid_ratings,
n_epochs=args.num_epochs,
verbose=False)
preds = model.predict(data_zeros, valid_users, valid_movies)
score = root_mean_square_error(valid_ratings, preds)
score_lst.append(score)
print("Fold:", fold + 1, "score:", score)
print('Mean CV RMSE:', np.mean(score_lst))
def bias_sgd(args):
# shuffle to ensure train examples exist for all users with high probability
kf = KFold(n_splits=args.splits_num, shuffle=args.shuffle, random_state=42)
score_lst = list()
for fold, (train_index, valid_index) in enumerate(kf.split(users)):
train_users = users[train_index]
train_movies = movies[train_index]
train_ratings = ratings[train_index]
valid_users = users[valid_index]
valid_movies = movies[valid_index]
valid_ratings = ratings[valid_index]
model = BiasSGD(number_of_users=number_of_users,
number_of_movies=number_of_movies,
hidden_size=args.hidden_size,
regularization_matrix=args.regularization_matrix,
regularization_vector=args.regularization_vector)
model.fit(train_users,
train_movies,
train_ratings,
valid_users=valid_users,
valid_movies=valid_movies,
valid_ratings=valid_ratings,
num_epochs=args.num_epochs,
decay=args.decay,
lr=args.lr,
decay_every=args.decay_every,
verbose=args.verbose)
preds = model.predict(valid_users, valid_movies)
score = root_mean_square_error(valid_ratings, preds)
score_lst.append(score)
print("Fold:", fold + 1, "score:", score)
print('Mean CV RMSE:', np.mean(score_lst))
def predict_with_config(args,
hidden_size=12,
lr=0.04,
reg_matrix=0.08,
reg_vector=0.04):
predictor = BiasSGD(N_USERS,
N_MOVIES,
hidden_size=hidden_size,
regularization_matrix=reg_matrix,
regularization_vector=reg_vector)
predictor.fit(train_users,
train_movies,
train_ratings,
valid_users=valid_users,
valid_movies=valid_movies,
valid_ratings=valid_ratings,
num_epochs=args.num_iter,
lr=lr)
preds = predictor.predict(valid_users, valid_movies)
err = root_mean_square_error(valid_ratings, preds)
if args.sleep > 0:
time.sleep(args.sleep)
return err
def fit(self,
data,
data_mask,
valid_users=None,
valid_movies=None,
valid_ratings=None,
n_epochs=350,
decay_steps=None,
learning_rate=None,
decay=None,
log_path=None,
verbose=True):
if decay_steps is None:
# empirical
decay_steps = self.number_of_users // 64 * 5
if learning_rate is None:
learning_rate = 10
if decay is None:
decay = 0.96
if log_path is None:
log_path = DEFAULT_LOG_PATH
validation = False
if valid_users is not None and valid_movies is not None and valid_ratings is not None:
assert len(valid_users) == len(valid_movies) == len(valid_ratings), \
"Invalid validation data provided"
validation = True
with tf.Graph().as_default():
with tf.Session() as sess:
self.build_graph()
global_step = tf.Variable(1,
name='global_step',
trainable=False)
learning_rate = tf.Variable(learning_rate,
trainable=False,
dtype=tf.float32,
name="learning_rate")
learning_rate = tf.train.exponential_decay(
learning_rate, global_step, decay_steps, decay)
# Gradients and update operation for training the model.
opt = tf.train.AdadeltaOptimizer(learning_rate)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
# Update all the trainable parameters
train_step = opt.minimize(self.loss,
global_step=global_step)
saver = tf.train.Saver(max_to_keep=3)
writer = tf.summary.FileWriter(log_path, sess.graph)
writer.flush()
tf.summary.scalar('loss', self.loss)
tf.summary.scalar('learning_rate', learning_rate)
summaries_merged = tf.summary.merge_all()
sess.run(tf.global_variables_initializer())
for epoch in range(n_epochs):
for users in get_batches(list(range(
self.number_of_users))):
user_ratings = [data[i, :] for i in users]
user_masks = [data_mask[i, :] for i in users]
_, loss, step, summary = sess.run(
[
train_step, self.loss, global_step,
summaries_merged
],
feed_dict={
self.input_: user_ratings,
self.input_pos: users,
self.input_mask: user_masks,
self.training: True
})
writer.add_summary(summary, step)
writer.flush()
if epoch % 10 == 0:
if verbose and validation:
predictions_validation = self.predict_with_session(
sess, data, valid_users, valid_movies)
print(
'Autoencoder: At epoch', epoch,
'validation error:',
root_mean_square_error(valid_ratings,
predictions_validation))
saver.save(sess,
os.path.join(log_path, "model"),
global_step=epoch)
def fit_data(self,
users_train,
movies_train,
ratings_train,
users_validation=None,
movies_validation=None,
ratings_validation=None,
num_iters=150000000,
update_every=1000000,
learning_rate=0.04,
decay=0.95,
verbose=True):
validation = False
if users_validation is not None and movies_validation is not None and ratings_validation is not None:
validation = True
# for better parameter tuning please refer to past shown report (uploaded by Adamos)
# for the moment pure stochastic
# also make this in epochs by taking user-movie-rating sequencially and shuffling each epoch
step = 0
average_rating = np.mean(ratings_train)
for iteration in range(num_iters):
step += 1
lr = update_learning_rate(learning_rate, step, update_every, decay)
index = randint(0, len(users_train) - 1)
user = users_train[index]
movie = movies_train[index]
U_d = self.U[user, :]
V_n = self.V[movie, :]
biasU_d = self.biasU[user]
biasV_n = self.biasV[movie]
guess = U_d.dot(V_n) + biasU_d + biasV_n
delta = ratings_train[index] - guess
try:
new_U_d = U_d + lr * (delta * V_n -
self.regularization_matrix * U_d)
new_V_n = V_n + lr * (delta * U_d -
self.regularization_matrix * V_n)
new_biasU_d = biasU_d + lr * (
delta - self.regularization_vector *
(biasU_d + biasV_n - average_rating))
new_biasV_n = biasV_n + lr * (
delta - self.regularization_vector *
(biasV_n + biasU_d - average_rating))
except FloatingPointError:
continue
else:
self.U[user, :] = new_U_d
self.V[movie, :] = new_V_n
self.biasU[user] = new_biasU_d
self.biasV[movie] = new_biasV_n
if self.verbose > 0:
if iteration % 1000000 == 0 and validation and verbose:
predictions = self.predict(users_validation,
movies_validation)
print(
'Validation error at iteration', iteration, 'is',
root_mean_square_error(ratings_validation,
predictions))
non_zero_inds = np.where(data_mask == 1)
users_train = non_zero_inds[0]
movies_train = non_zero_inds[1]
ratings_train = data_zeros[non_zero_inds]
list_preds_train = []
list_preds_test = []
for _ in range(10):
model3 = Autoencoder(number_of_users,
number_of_movies,
layers=[21, 21, 21],
masking=0.5)
model3.train(data_zeros,
data_mask,
users_validation=users_test,
movies_validation=movies_test,
ratings_validations=ratings_test,
n_epochs=200,
verbose=False)
preds_train = model3.predict(data_zeros, users_train, movies_train)
preds_test = model3.predict(data_zeros, users_test, movies_test)
list_preds_train.append(preds_train)
list_preds_test.append(preds_test)
ratings_test_pred = np.mean(list_preds_test, axis=0)
score = root_mean_square_error(ratings_test, ratings_test_pred)
print("score:", score)
val_error = list()
shrinkage_array = np.arange(20, 51)
known_train = ~np.isnan(A_train)
known_validation = ~np.isnan(A_valid)
# Fill-in missing entries
X_train = pd.DataFrame(A_train, copy=True)
movie_avg_ratings = X_train.mean(axis=0, skipna=True)
A_train_average = X_train.fillna(movie_avg_ratings, axis=0).values
for shrinkage in shrinkage_array:
model = IterativeSVD(shrinkage=shrinkage)
model.fit(A_train_average, known_train, verbose=False)
preds = model.predict(valid_users, valid_movies)
val_error.append(root_mean_square_error(valid_ratings, preds))
val_error = np.array(val_error)
plt.figure()
plt.plot(shrinkage_array, val_error)
plt.ylabel('RMSE')
plt.xlabel('Threshold')
plt.show()
def generate(data):
# Turn on evaluation mode which disables dropout.
model.eval()
idss_predict = []
context_predict = []
rating_predict = []
with torch.no_grad():
while True:
user, item, rating, seq, feature = data.next_batch()
user = user.to(device) # (batch_size,)
item = item.to(device)
bos = seq[:, 0].unsqueeze(0).to(device) # (1, batch_size)
feature = feature.t().to(device) # (1, batch_size)
if args.use_feature:
text = torch.cat([feature, bos],
0) # (src_len - 1, batch_size)
else:
text = bos # (src_len - 1, batch_size)
start_idx = text.size(0)
for idx in range(args.words):
# produce a word at each step
if idx == 0:
log_word_prob, log_context_dis, rating_p, _ = model(
user, item, text, False
) # (batch_size, ntoken) vs. (batch_size, ntoken) vs. (batch_size,)
rating_predict.extend(rating_p.tolist())
context = predict(log_context_dis,
topk=args.words) # (batch_size, words)
context_predict.extend(context.tolist())
else:
log_word_prob, _, _, _ = model(
user, item, text, False, False,
False) # (batch_size, ntoken)
word_prob = log_word_prob.exp() # (batch_size, ntoken)
word_idx = torch.argmax(
word_prob, dim=1
) # (batch_size,), pick the one with the largest probability
text = torch.cat([text, word_idx.unsqueeze(0)],
0) # (len++, batch_size)
ids = text[start_idx:].t().tolist() # (batch_size, seq_len)
idss_predict.extend(ids)
if data.step == data.total_step:
break
# rating
predicted_rating = [
(r, p) for (r, p) in zip(data.rating.tolist(), rating_predict)
]
RMSE = root_mean_square_error(predicted_rating, corpus.max_rating,
corpus.min_rating)
print(now_time() + 'RMSE {:7.4f}'.format(RMSE))
MAE = mean_absolute_error(predicted_rating, corpus.max_rating,
corpus.min_rating)
print(now_time() + 'MAE {:7.4f}'.format(MAE))
# text
tokens_test = [
ids2tokens(ids[1:], word2idx, idx2word) for ids in data.seq.tolist()
]
tokens_predict = [
ids2tokens(ids, word2idx, idx2word) for ids in idss_predict
]
BLEU1 = bleu_score(tokens_test, tokens_predict, n_gram=1, smooth=False)
print(now_time() + 'BLEU-1 {:7.4f}'.format(BLEU1))
BLEU4 = bleu_score(tokens_test, tokens_predict, n_gram=4, smooth=False)
print(now_time() + 'BLEU-4 {:7.4f}'.format(BLEU4))
USR, USN = unique_sentence_percent(tokens_predict)
print(now_time() + 'USR {:7.4f} | USN {:7}'.format(USR, USN))
feature_batch = feature_detect(tokens_predict, feature_set)
DIV = feature_diversity(feature_batch) # time-consuming
print(now_time() + 'DIV {:7.4f}'.format(DIV))
FCR = feature_coverage_ratio(feature_batch, feature_set)
print(now_time() + 'FCR {:7.4f}'.format(FCR))
feature_test = [idx2word[i]
for i in data.feature.squeeze(1).tolist()] # ids to words
FMR = feature_matching_ratio(feature_batch, feature_test)
print(now_time() + 'FMR {:7.4f}'.format(FMR))
text_test = [' '.join(tokens) for tokens in tokens_test]
text_predict = [' '.join(tokens) for tokens in tokens_predict]
tokens_context = [
' '.join([idx2word[i] for i in ids]) for ids in context_predict
]
ROUGE = rouge_score(text_test, text_predict) # a dictionary
for (k, v) in ROUGE.items():
print(now_time() + '{} {:7.4f}'.format(k, v))
text_out = ''
for (real, ctx, fake) in zip(text_test, tokens_context, text_predict):
text_out += '{}\n{}\n{}\n\n'.format(real, ctx, fake)
return text_out
def main(gt, methods, display_names):
# read gt image addresses
gt_names = utils.all_files_under(os.path.join('../', gt), extension='.jpg')
# read prediction image addresses
filenames_list = []
for method in methods:
filenames = utils.all_files_under(os.path.join('../', method),
extension='.jpg')
filenames_list.append(filenames)
mae_overall, rmse_overall, psnr_overall, ssim_overall, pcc_overall = [], [], [], [], []
for idx_method in range(len(methods)):
# print(methods[idx_method])
mae_method, rmse_method, psnr_method, ssim_method, pcc_method = [], [], [], [], []
for idx_name in range(len(gt_names)):
# read gt and prediction image
gt_img = cv2.imread(gt_names[idx_name],
cv2.IMREAD_GRAYSCALE).astype(np.float32)
pred_img = cv2.imread(filenames_list[idx_method][idx_name],
cv2.IMREAD_GRAYSCALE).astype(np.float32)
# check gt and prediction image name
if gt_names[idx_name].split('p0')[-1] != filenames_list[
idx_method][idx_name].split('p0')[-1]:
sys.exit(' [!] Image name can not match!')
# calcualte mae and psnr
mae = utils.mean_absoulute_error(pred_img, gt_img)
rmse = utils.root_mean_square_error(pred_img, gt_img)
psnr = utils.peak_signal_to_noise_ratio(pred_img, gt_img)
ssim = utils.structural_similarity_index(pred_img, gt_img)
pcc = utils.pearson_correlation_coefficient(pred_img, gt_img)
if np.mod(idx_name, 300) == 0:
print('Method: {}, idx: {}'.format(methods[idx_method],
idx_name))
# collect each image results
mae_method.append(mae)
rmse_method.append(rmse)
psnr_method.append(psnr)
ssim_method.append(ssim)
pcc_method.append(pcc)
# list to np.array
mae_method = np.asarray(mae_method)
rmse_method = np.asarray(rmse_method)
psnr_method = np.asarray(psnr_method)
ssim_method = np.asarray(ssim_method)
pcc_method = np.asarray(pcc_method)
# collect all methods results
mae_overall.append(mae_method)
rmse_overall.append(rmse_method)
psnr_overall.append(psnr_method)
ssim_overall.append(ssim_method)
pcc_overall.append(pcc_method)
# list to np.array
mae_overall = np.asarray(mae_overall)
rmse_overall = np.asarray(rmse_overall)
psnr_overall = np.asarray(psnr_overall)
ssim_overall = np.asarray(ssim_overall)
pcc_overall = np.asarray(pcc_overall)
# draw boxplot
utils.draw_box_plot(
[mae_overall, rmse_overall, psnr_overall, ssim_overall, pcc_overall],
display_names)
# write to csv file
utils.write_to_csv(
[mae_overall, rmse_overall, psnr_overall, ssim_overall, pcc_overall],
methods, gt_names)
def fit(self,
train_users,
train_movies,
train_ratings,
valid_users=None,
valid_movies=None,
valid_ratings=None,
num_epochs=50,
decay=1.5,
lr=0.05,
decay_every=5,
verbose=True):
"""
Parameters
----------
train_users array of user ids for train
train_movies array of movie ids for train
train_ratings array of ratings for train
valid_users array of user ids for validation
valid_movies array of movie ids for validation
valid_ratings array of ratings for validation
num_epochs number of epochs
decay divide the learning rate by this number every requested epochs
lr initial learning rate
decay_every number of epoch every which to decay the learning rate
verbose
"""
validation = False
if valid_users is not None and valid_movies is not None and valid_ratings is not None:
validation = True
average_rating = np.mean(train_ratings)
for epoch in range(1, num_epochs + 1):
users_train_sh, movies_train_sh, ratings_train_sh = shuffle(
train_users, train_movies, train_ratings)
for user, movie, rating in zip(users_train_sh, movies_train_sh,
ratings_train_sh):
U_d = self.U[user, :]
V_n = self.V[movie, :]
biasU_d = self.biasU[user]
biasV_n = self.biasV[movie]
guess = U_d.dot(V_n) + biasU_d + biasV_n
delta = rating - guess
try:
new_U_d = U_d + lr * (delta * V_n -
self.regularization_matrix * U_d)
new_V_n = V_n + lr * (delta * U_d -
self.regularization_matrix * V_n)
new_biasU_d = biasU_d + lr * (
delta - self.regularization_vector *
(biasU_d + biasV_n - average_rating))
new_biasV_n = biasV_n + lr * (
delta - self.regularization_vector *
(biasV_n + biasU_d - average_rating))
except FloatingPointError:
continue
else:
self.U[user, :] = new_U_d
self.V[movie, :] = new_V_n
self.biasU[user] = new_biasU_d
self.biasV[movie] = new_biasV_n
if validation and verbose:
predictions = self.predict(valid_users, valid_movies)
print('Validation error at epoch', epoch, 'is',
root_mean_square_error(valid_ratings, predictions))
if epoch % decay_every == 0:
lr /= decay
name = "Micromax"
target = "micromax_fing_pred.csv"
df = pd.read_csv(target)
df = df[['pred_loc', 'act_loc', 'ts']].values
df = dict([(x[2], (list(map(float, (x[0].strip('[]').split(','))))))
for x in df])
print(df_path1)
num_loc = 0
count = 0
for ts in df_path1:
act_x, act_y = df_path1[ts]
update(validation, [act_x], [act_y])
if ts in df:
pred_x, pred_y = df[ts]
num_loc += 1
update(path, [pred_x], [pred_y])
ax.legend([path, validation],
["# points: %d\ntime: %s" % (num_loc, ts)])
count += 1
plt.savefig("plots/%s/%03d.png" % (name, count))
ax.set_title("%s\nFingerprinting" % (name))
plt.pause(0.02)
ax.set_title("%s\nFingerprinting \nRMSQ: %f" %
(name, utils.root_mean_square_error(df_path1, df)))
plt.savefig("plots/%s/%03d.png" % (name, count + 1))
plt.show()
#