def __init__(self): self.auto = AutoCoder() self.name = "fast encoder" self.data ={} path = Dir.res+"/encoder/cleandata_8700/" fllist = ftools.get_files(path) for name in fllist: self.data[name] = tools.load_object(path+name)
def load_parameter(self): flist = tools.get_files(self.path) # print(flist) for name in flist: fpath = self.path+name tmp = tools.load_object(fpath) for key in tmp.keys(): self.parameter[key] = tmp[key]
def analyze(main_name, compare_index, name="cleandata_small"): save_path = Dir.res + "/result/judge.txt" jude_dict = tools.load_object(save_path) # print(list(jude_dict.keys())[0]) print(len(jude_dict)) entry_path = Dir.res + "/result/" + name + "/EntryBigraph/detials.txt" entry_data = load_data(entry_path) first_path = Dir.res + "/result/" + name + "/" + main_name + "/detials.txt" first_data = load_data(first_path) textrank_path = Dir.res + "/result/" + name + "/TextRank/detials.txt" tr_data = load_data(textrank_path) result = {} for key in first_data.keys(): a = first_data[key][0] - entry_data[key][0] b = first_data[key][1] - entry_data[key][1] c = first_data[key][0] - tr_data[key][0] d = first_data[key][1] - tr_data[key][1] e = first_data[key][0] - tr_data[key][0] + entry_data[key][ 0] - tr_data[key][0] f = first_data[key][1] - tr_data[key][1] + entry_data[key][ 1] - tr_data[key][1] result[key] = [a, b, c, d, e, f] count = 0 news_root = Dir.res + "/" + name + "/news/" abst_root = Dir.res + "/" + name + "/abstract/" fname = ftools.get_files(news_root) new_result = {} for filename in fname: # print(filename,count,len(fname)) # news = ftools.read_lines(news_root+filename) # weibo = ftools.read_lines(abst_root+filename) # jude = data_filter(news,weibo) # jude_dict[filename] = jude jude = jude_dict[filename] if jude > 0.5: new_result[filename] = result[filename] new_result[filename].append(jude) count += 1 tools.save_object(jude_dict, Dir.res + "/result/judge.txt") tmp = dict( sorted(new_result.items(), key=lambda d: d[1][compare_index], reverse=True)) save_dict = {} names = [] for key in tmp.keys(): save_dict[key] = tmp[key] names.append(key) save_path = Dir.res + "/result/" + name + "/" + main_name + ".txt" ftools.write_com_dict(save_path, save_dict) return names
def analyze_rate(name="cleandata_small", num=None): save_path = Dir.res + "/result/judge.txt" jude_dict = tools.load_object(save_path) print(len(jude_dict)) rate = [0, 0, 0, 0] # nums = [int(num*var/sum(rate)) for var in rate] textrank_path = Dir.res + "/result/" + name + "/TextRank/detials.txt" tr_data = load_data(textrank_path) tr_data = dict( sorted(tr_data.items(), key=lambda d: d[1][0], reverse=False)) entry_path = Dir.res + "/result/" + name + "/EntryBigraph/detials.txt" entry_data = load_data(entry_path) entry_data = dict( sorted(entry_data.items(), key=lambda d: d[1][0], reverse=False)) fvae = Dir.res + "/result/" + name + "/Fourth Version auto encoder/detials.txt" fvae_data = load_data(fvae) fvae_data = dict( sorted(fvae_data.items(), key=lambda d: d[1][0], reverse=True)) fvsae = Dir.res + "/result/" + name + "/Fourth Version simple auto encoder/detials.txt" fvsae_data = load_data(fvsae) fvsae_data = dict( sorted(fvsae_data.items(), key=lambda d: d[1][0], reverse=True)) res = [] new_dict = dict(sorted(jude_dict.items(), key=lambda d: d[0], reverse=True)) for na in new_dict.keys(): if num != None and len(res) >= num: break if new_dict[na] > 0.8: res.append(na) # for key in list(fvsae_data.keys()): # if jude_dict[key] >0.5: # res.append(key) # # if len(res)>=nums[0]: # break # # for key in fvae_data.keys(): # if key not in res: # if jude_dict[key] > 0.5: # res.append(key) # if len(res) > sum(nums[:2]): # break # # for key in entry_data.keys(): # if key not in res: # if jude_dict[key] > 0.5: # res.append(key) # if len(res) > sum(nums[:3]): # break # # for key in tr_data.keys(): # if key not in res: # if jude_dict[key] > 0.5: # res.append(key) # if len(res) == num: # break return res