def __init__(self,
model_dir,
batch_size=8192,
epochs=50,
lr=1e-3,
early_stop=10,
dropout=0.5,
overwrite=True,
**kwargs):
self.model_manager = ModelManager(model_dir)
self.ckpt_path = os.path.join(self.model_manager.path_name, 'ckpt')
self.batch_size = batch_size
self.epochs = epochs
self.lr = lr
self.early_stop = early_stop
self.dropout = dropout
self.user_list = load_np_array(CONFIG.user_list_file)
self.item_list = load_np_array(CONFIG.movie_list_file)
tf.reset_default_graph()
self.model = NeuralCF(len(self.user_list), len(self.item_list),
**kwargs)
self.sess = None
self.saver = None
self.global_step = tf.Variable(0, trainable=False)
with tf.variable_scope("Optimizer"):
params = tf.trainable_variables()
gradients = tf.gradients(self.model.loss, params)
clipped_gradients, _ = tf.clip_by_global_norm(gradients, 5.0)
optimizer = tf.train.AdamOptimizer(self.lr)
self.train_op = optimizer.apply_gradients(
zip(clipped_gradients, params), global_step=self.global_step)
self.create_or_load_model(overwrite)
def make_profiles(overwrite=False):
def parse_profile(_id, _type):
file = os.path.join(CONFIG.data_path, _type, _id, 'analysis',
'profile.json')
if not os.path.exists(file):
return []
profile = load_json_file(file)
result = []
for target, value in profile.items():
for sentiment, val in value.items():
for description, sentences in val.items():
result.extend([(target, description, sentiment)] *
len(sentences))
return result
if not overwrite and os.path.exists(CONFIG.user_profile_file):
user_profile = load_json_file(CONFIG.user_profile_file)
else:
users = load_np_array(CONFIG.user_list_file)
user_profile = list(map(lambda x: parse_profile(x, 'user'), users))
save_json_file(CONFIG.user_profile_file, user_profile)
if not overwrite and os.path.exists(CONFIG.movie_profile_file):
movie_profile = load_json_file(CONFIG.movie_profile_file)
else:
movies = load_np_array(CONFIG.movie_list_file)
movie_profile = list(map(lambda x: parse_profile(x, 'subject'),
movies))
save_json_file(CONFIG.movie_profile_file, movie_profile)
return user_profile, movie_profile
def get_rate_matrix(user_th=5, mode='rate', overwrite=False):
if not overwrite and os.path.exists(CONFIG.rate_record_all):
try:
user_list = load_np_array(CONFIG.user_list_file)
movie_list = load_np_array(CONFIG.movie_list_file)
rate_matrix = load_np_array(CONFIG.rate_matrix_file)
return user_list, movie_list, rate_matrix
except:
pass
user_index_dict = dict()
user_list = []
user_index = 0
movie_index_dict = dict()
movie_list = []
movie_index = 0
users = os.listdir(CONFIG.user_path)
records = []
for user in users:
collect_file = os.path.join(CONFIG.user_path, user, 'collect.json')
try:
collects = load_json_file(collect_file)
except:
collects = []
if mode == 'comment':
collects = list(
filter(lambda x: x['comment'] != '' and x["rate"] != 0,
collects))
elif mode == 'rate':
collects = list(filter(lambda x: x["rate"] != 0, collects))
if len(collects) < user_th:
continue
user_index_dict[user] = user_index
user_list.append(user)
user_index += 1
# user_count[user] = len(collects)
for item in collects:
if mode == 'bool':
value = 1
else:
value = item['rate']
movie = item['movie_id']
if movie not in movie_index_dict:
movie_index_dict[movie] = movie_index
movie_list.append(movie)
movie_index += 1
date = item["date"]
row = user_index_dict[user]
col = movie_index_dict[movie]
# movie_count[movie] = movie_count.get(movie, 0) + 1
records.append((row, col, value, date))
write_lines(CONFIG.rate_record_all, records, lambda x: '%s %s %d %s' %
(x[0], x[1], x[2], x[3]))
rows, cols, values, dates = list(zip(*records))
rate_matrix = coo_matrix((values, (rows, cols)),
shape=(user_index, movie_index)).todense()
save_np_array(CONFIG.rate_matrix_file, rate_matrix)
save_np_array(CONFIG.user_list_file, user_list)
save_np_array(CONFIG.movie_list_file, movie_list)
return user_list, movie_list, rate_matrix
def make_tags(overwrite=False):
def parse_movie_tags(movie):
info_file = os.path.join(CONFIG.movie_path, movie, 'info.json')
if not os.path.exists(info_file):
return []
info = load_json_file(info_file)
return info.get("genres", [])
def parse_user_tags(user):
collect_profile_file = os.path.join(CONFIG.user_path, user, 'profile',
'collect_distribution.json')
if not os.path.exists(collect_profile_file):
return []
collect_profile = load_json_file(collect_profile_file)
tag_distribution = collect_profile.get("type", {})
tags = list(
itertools.chain.from_iterable(
[[tag for _ in range(freq)]
for tag, freq in tag_distribution.items()]))
return tags
if not overwrite and os.path.exists(CONFIG.user_tags_file):
user_tags = load_json_file(CONFIG.user_tags_file)
else:
users = load_np_array(CONFIG.user_list_file)
user_tags = list(map(parse_user_tags, users))
save_json_file(CONFIG.user_tags_file, user_tags)
if not overwrite and os.path.exists(CONFIG.movie_tags_file):
movie_tags = load_json_file(CONFIG.movie_tags_file)
else:
movies = load_np_array(CONFIG.movie_list_file)
movie_tags = list(map(parse_movie_tags, movies))
save_json_file(CONFIG.movie_tags_file, movie_tags)
if not overwrite and os.path.exists(CONFIG.tag_word_list):
tag_words = read_lines(CONFIG.tag_word_list, lambda x: x.strip())
else:
tag_words = set(itertools.chain.from_iterable(user_tags + movie_tags))
write_lines(CONFIG.tag_word_list, tag_words)
return user_tags, movie_tags, tag_words
def get_all_comments_and_reviews():
users = load_np_array(CONFIG.user_list_file)
movies = load_np_array(CONFIG.movie_list_file)
user_comment = []
user_review = []
movie_comment = []
movie_review = []
for user in users:
try:
review = load_json_file(
os.path.join(CONFIG.user_path, user, 'review', 'review.json'))
except Exception as e:
print(e)
review = []
try:
comment = load_json_file(
os.path.join(CONFIG.user_path, user, 'collect',
'collect.json'))
except Exception as e:
print(e)
comment = []
review = [(x["content"], x["rate"]) for x in review
if len(x) > 0 and x["rate"] > 0]
comment = [(x["comment"], x["rate"]) for x in comment
if x["rate"] > 0 and x["comment"].strip() != ""]
user_review.extend(review)
user_comment.extend(comment)
print('user: comment: %d, review: %d' %
(len(user_comment), len(user_review)))
for movie in movies:
review_folder = os.path.join(CONFIG.movie_path, movie, 'reviews')
review = []
if os.path.exists(review_folder):
files = [
os.path.join(review_folder, x)
for x in os.listdir(review_folder) if x.endswith('0.json')
]
for file in files:
data = load_json_file(file)
review.extend([(x["content"], x["rating"]["value"])
for x in data['reviews']
if x["rating"]["value"] > 0])
movie_review.extend(review)
comment_folder = os.path.join(CONFIG.movie_path, movie, 'comments')
comment = []
if os.path.exists(comment_folder):
files = [
os.path.join(comment_folder, x)
for x in os.listdir(comment_folder) if x.endswith('0.json')
]
for file in files:
data = load_json_file(file)
comment.extend([(x["content"], x["rating"]["value"])
for x in data['comments']
if x["rating"]["value"] > 0])
movie_comment.extend(comment)
print('movie: comment: %d, review: %d' %
(len(movie_comment), len(movie_review)))
comment_rate = user_comment + movie_comment
review_rate = user_review + movie_review
save_json_file(CONFIG.comment_rate_file, comment_rate)
save_json_file(CONFIG.review_rate_file, review_rate)