def merge_cut_profile_rate(filter_no_profile=True, overwrite=False): if not overwrite and os.path.exists(CONFIG.sing_rate_data_all): return # cuts_comment = load_json_file(CONFIG.single_rate_comment_cut) # rates_comment = [x[1] for x in load_json_file(CONFIG.single_rate_comment)] # profiles_comment = load_json_file(CONFIG.single_rate_comment_profile) # data_comment = zip(cuts_comment, profiles_comment, rates_comment) # if filter_no_profile: # data_comment = filter(lambda x: len(x[1]) > 0, data_comment) # data_comment = list(data_comment) # del cuts_comment, rates_comment, profiles_comment # gc.collect() data_comment = [] cuts_review = load_json_file(CONFIG.single_rate_review_cut) rates_review = [x[1] for x in load_json_file(CONFIG.single_rate_review)] profiles_review = load_json_file(CONFIG.sing_rate_review_profile) data_review = zip(cuts_review, profiles_review, rates_review) if filter_no_profile: data_review = filter(lambda x: len(x[1]) > 0, data_review) data_review = list(data_review) del cuts_review, rates_review, profiles_review gc.collect() data = data_comment + data_review save_json_file(CONFIG.sing_rate_data_all, data) print('got %d cut_profile_rate data' % len(data)) del data_comment, data_review, data gc.collect()
def make_profiles(overwrite=False): def parse_profile(_id, _type): file = os.path.join(CONFIG.data_path, _type, _id, 'analysis', 'profile.json') if not os.path.exists(file): return [] profile = load_json_file(file) result = [] for target, value in profile.items(): for sentiment, val in value.items(): for description, sentences in val.items(): result.extend([(target, description, sentiment)] * len(sentences)) return result if not overwrite and os.path.exists(CONFIG.user_profile_file): user_profile = load_json_file(CONFIG.user_profile_file) else: users = load_np_array(CONFIG.user_list_file) user_profile = list(map(lambda x: parse_profile(x, 'user'), users)) save_json_file(CONFIG.user_profile_file, user_profile) if not overwrite and os.path.exists(CONFIG.movie_profile_file): movie_profile = load_json_file(CONFIG.movie_profile_file) else: movies = load_np_array(CONFIG.movie_list_file) movie_profile = list(map(lambda x: parse_profile(x, 'subject'), movies)) save_json_file(CONFIG.movie_profile_file, movie_profile) return user_profile, movie_profile
def make_tags(overwrite=False): def parse_movie_tags(movie): info_file = os.path.join(CONFIG.movie_path, movie, 'info.json') if not os.path.exists(info_file): return [] info = load_json_file(info_file) return info.get("genres", []) def parse_user_tags(user): collect_profile_file = os.path.join(CONFIG.user_path, user, 'profile', 'collect_distribution.json') if not os.path.exists(collect_profile_file): return [] collect_profile = load_json_file(collect_profile_file) tag_distribution = collect_profile.get("type", {}) tags = list( itertools.chain.from_iterable( [[tag for _ in range(freq)] for tag, freq in tag_distribution.items()])) return tags if not overwrite and os.path.exists(CONFIG.user_tags_file): user_tags = load_json_file(CONFIG.user_tags_file) else: users = load_np_array(CONFIG.user_list_file) user_tags = list(map(parse_user_tags, users)) save_json_file(CONFIG.user_tags_file, user_tags) if not overwrite and os.path.exists(CONFIG.movie_tags_file): movie_tags = load_json_file(CONFIG.movie_tags_file) else: movies = load_np_array(CONFIG.movie_list_file) movie_tags = list(map(parse_movie_tags, movies)) save_json_file(CONFIG.movie_tags_file, movie_tags) if not overwrite and os.path.exists(CONFIG.tag_word_list): tag_words = read_lines(CONFIG.tag_word_list, lambda x: x.strip()) else: tag_words = set(itertools.chain.from_iterable(user_tags + movie_tags)) write_lines(CONFIG.tag_word_list, tag_words) return user_tags, movie_tags, tag_words
def split_cut_profile_rate(k=5, overwrite=False): if not overwrite: files = [ x for x in os.listdir(CONFIG.single_rate_training_folder) if re.match(r'%s_\d+.json' % CONFIG.single_rate_file_name, x) is not None ] if len(files) == k: return all_records = load_json_file(CONFIG.sing_rate_data_all) print_with_time('all records loaded') random.shuffle(all_records) print_with_time('shuffled') for i in range(k): out_file = os.path.join( CONFIG.single_rate_training_folder, '%s_%d.json' % (CONFIG.single_rate_file_name, i)) subset = all_records[i::k] save_json_file(out_file, subset) del subset gc.collect() del all_records gc.collect()
def get_all_comments_and_reviews(): users = load_np_array(CONFIG.user_list_file) movies = load_np_array(CONFIG.movie_list_file) user_comment = [] user_review = [] movie_comment = [] movie_review = [] for user in users: try: review = load_json_file( os.path.join(CONFIG.user_path, user, 'review', 'review.json')) except Exception as e: print(e) review = [] try: comment = load_json_file( os.path.join(CONFIG.user_path, user, 'collect', 'collect.json')) except Exception as e: print(e) comment = [] review = [(x["content"], x["rate"]) for x in review if len(x) > 0 and x["rate"] > 0] comment = [(x["comment"], x["rate"]) for x in comment if x["rate"] > 0 and x["comment"].strip() != ""] user_review.extend(review) user_comment.extend(comment) print('user: comment: %d, review: %d' % (len(user_comment), len(user_review))) for movie in movies: review_folder = os.path.join(CONFIG.movie_path, movie, 'reviews') review = [] if os.path.exists(review_folder): files = [ os.path.join(review_folder, x) for x in os.listdir(review_folder) if x.endswith('0.json') ] for file in files: data = load_json_file(file) review.extend([(x["content"], x["rating"]["value"]) for x in data['reviews'] if x["rating"]["value"] > 0]) movie_review.extend(review) comment_folder = os.path.join(CONFIG.movie_path, movie, 'comments') comment = [] if os.path.exists(comment_folder): files = [ os.path.join(comment_folder, x) for x in os.listdir(comment_folder) if x.endswith('0.json') ] for file in files: data = load_json_file(file) comment.extend([(x["content"], x["rating"]["value"]) for x in data['comments'] if x["rating"]["value"] > 0]) movie_comment.extend(comment) print('movie: comment: %d, review: %d' % (len(movie_comment), len(movie_review))) comment_rate = user_comment + movie_comment review_rate = user_review + movie_review save_json_file(CONFIG.comment_rate_file, comment_rate) save_json_file(CONFIG.review_rate_file, review_rate)
def save_json(self, data, save_name: str): if '.json' not in save_name: save_name += '.json' self.make_dir_for_file(save_name) save_json_file(os.path.join(self.path_name, save_name), data)