def merge_cut_profile_rate(filter_no_profile=True, overwrite=False):
if not overwrite and os.path.exists(CONFIG.sing_rate_data_all):
return
# cuts_comment = load_json_file(CONFIG.single_rate_comment_cut)
# rates_comment = [x[1] for x in load_json_file(CONFIG.single_rate_comment)]
# profiles_comment = load_json_file(CONFIG.single_rate_comment_profile)
# data_comment = zip(cuts_comment, profiles_comment, rates_comment)
# if filter_no_profile:
# data_comment = filter(lambda x: len(x[1]) > 0, data_comment)
# data_comment = list(data_comment)
# del cuts_comment, rates_comment, profiles_comment
# gc.collect()
data_comment = []
cuts_review = load_json_file(CONFIG.single_rate_review_cut)
rates_review = [x[1] for x in load_json_file(CONFIG.single_rate_review)]
profiles_review = load_json_file(CONFIG.sing_rate_review_profile)
data_review = zip(cuts_review, profiles_review, rates_review)
if filter_no_profile:
data_review = filter(lambda x: len(x[1]) > 0, data_review)
data_review = list(data_review)
del cuts_review, rates_review, profiles_review
gc.collect()
data = data_comment + data_review
save_json_file(CONFIG.sing_rate_data_all, data)
print('got %d cut_profile_rate data' % len(data))
del data_comment, data_review, data
gc.collect()
def make_profiles(overwrite=False):
def parse_profile(_id, _type):
file = os.path.join(CONFIG.data_path, _type, _id, 'analysis',
'profile.json')
if not os.path.exists(file):
return []
profile = load_json_file(file)
result = []
for target, value in profile.items():
for sentiment, val in value.items():
for description, sentences in val.items():
result.extend([(target, description, sentiment)] *
len(sentences))
return result
if not overwrite and os.path.exists(CONFIG.user_profile_file):
user_profile = load_json_file(CONFIG.user_profile_file)
else:
users = load_np_array(CONFIG.user_list_file)
user_profile = list(map(lambda x: parse_profile(x, 'user'), users))
save_json_file(CONFIG.user_profile_file, user_profile)
if not overwrite and os.path.exists(CONFIG.movie_profile_file):
movie_profile = load_json_file(CONFIG.movie_profile_file)
else:
movies = load_np_array(CONFIG.movie_list_file)
movie_profile = list(map(lambda x: parse_profile(x, 'subject'),
movies))
save_json_file(CONFIG.movie_profile_file, movie_profile)
return user_profile, movie_profile
def make_tags(overwrite=False):
def parse_movie_tags(movie):
info_file = os.path.join(CONFIG.movie_path, movie, 'info.json')
if not os.path.exists(info_file):
return []
info = load_json_file(info_file)
return info.get("genres", [])
def parse_user_tags(user):
collect_profile_file = os.path.join(CONFIG.user_path, user, 'profile',
'collect_distribution.json')
if not os.path.exists(collect_profile_file):
return []
collect_profile = load_json_file(collect_profile_file)
tag_distribution = collect_profile.get("type", {})
tags = list(
itertools.chain.from_iterable(
[[tag for _ in range(freq)]
for tag, freq in tag_distribution.items()]))
return tags
if not overwrite and os.path.exists(CONFIG.user_tags_file):
user_tags = load_json_file(CONFIG.user_tags_file)
else:
users = load_np_array(CONFIG.user_list_file)
user_tags = list(map(parse_user_tags, users))
save_json_file(CONFIG.user_tags_file, user_tags)
if not overwrite and os.path.exists(CONFIG.movie_tags_file):
movie_tags = load_json_file(CONFIG.movie_tags_file)
else:
movies = load_np_array(CONFIG.movie_list_file)
movie_tags = list(map(parse_movie_tags, movies))
save_json_file(CONFIG.movie_tags_file, movie_tags)
if not overwrite and os.path.exists(CONFIG.tag_word_list):
tag_words = read_lines(CONFIG.tag_word_list, lambda x: x.strip())
else:
tag_words = set(itertools.chain.from_iterable(user_tags + movie_tags))
write_lines(CONFIG.tag_word_list, tag_words)
return user_tags, movie_tags, tag_words
def split_cut_profile_rate(k=5, overwrite=False):
if not overwrite:
files = [
x for x in os.listdir(CONFIG.single_rate_training_folder)
if re.match(r'%s_\d+.json' %
CONFIG.single_rate_file_name, x) is not None
]
if len(files) == k:
return
all_records = load_json_file(CONFIG.sing_rate_data_all)
print_with_time('all records loaded')
random.shuffle(all_records)
print_with_time('shuffled')
for i in range(k):
out_file = os.path.join(
CONFIG.single_rate_training_folder,
'%s_%d.json' % (CONFIG.single_rate_file_name, i))
subset = all_records[i::k]
save_json_file(out_file, subset)
del subset
gc.collect()
del all_records
gc.collect()
def get_all_comments_and_reviews():
users = load_np_array(CONFIG.user_list_file)
movies = load_np_array(CONFIG.movie_list_file)
user_comment = []
user_review = []
movie_comment = []
movie_review = []
for user in users:
try:
review = load_json_file(
os.path.join(CONFIG.user_path, user, 'review', 'review.json'))
except Exception as e:
print(e)
review = []
try:
comment = load_json_file(
os.path.join(CONFIG.user_path, user, 'collect',
'collect.json'))
except Exception as e:
print(e)
comment = []
review = [(x["content"], x["rate"]) for x in review
if len(x) > 0 and x["rate"] > 0]
comment = [(x["comment"], x["rate"]) for x in comment
if x["rate"] > 0 and x["comment"].strip() != ""]
user_review.extend(review)
user_comment.extend(comment)
print('user: comment: %d, review: %d' %
(len(user_comment), len(user_review)))
for movie in movies:
review_folder = os.path.join(CONFIG.movie_path, movie, 'reviews')
review = []
if os.path.exists(review_folder):
files = [
os.path.join(review_folder, x)
for x in os.listdir(review_folder) if x.endswith('0.json')
]
for file in files:
data = load_json_file(file)
review.extend([(x["content"], x["rating"]["value"])
for x in data['reviews']
if x["rating"]["value"] > 0])
movie_review.extend(review)
comment_folder = os.path.join(CONFIG.movie_path, movie, 'comments')
comment = []
if os.path.exists(comment_folder):
files = [
os.path.join(comment_folder, x)
for x in os.listdir(comment_folder) if x.endswith('0.json')
]
for file in files:
data = load_json_file(file)
comment.extend([(x["content"], x["rating"]["value"])
for x in data['comments']
if x["rating"]["value"] > 0])
movie_comment.extend(comment)
print('movie: comment: %d, review: %d' %
(len(movie_comment), len(movie_review)))
comment_rate = user_comment + movie_comment
review_rate = user_review + movie_review
save_json_file(CONFIG.comment_rate_file, comment_rate)
save_json_file(CONFIG.review_rate_file, review_rate)
def save_json(self, data, save_name: str):
if '.json' not in save_name:
save_name += '.json'
self.make_dir_for_file(save_name)
save_json_file(os.path.join(self.path_name, save_name), data)