def __init__(self):
self.auto = AutoCoder()
self.name = "fast encoder"
self.data ={}
path = Dir.res+"/encoder/cleandata_8700/"
fllist = ftools.get_files(path)
for name in fllist:
self.data[name] = tools.load_object(path+name)
def load_parameter(self):
flist = tools.get_files(self.path)
# print(flist)
for name in flist:
fpath = self.path+name
tmp = tools.load_object(fpath)
for key in tmp.keys():
self.parameter[key] = tmp[key]
def analyze(main_name, compare_index, name="cleandata_small"):
save_path = Dir.res + "/result/judge.txt"
jude_dict = tools.load_object(save_path)
# print(list(jude_dict.keys())[0])
print(len(jude_dict))
entry_path = Dir.res + "/result/" + name + "/EntryBigraph/detials.txt"
entry_data = load_data(entry_path)
first_path = Dir.res + "/result/" + name + "/" + main_name + "/detials.txt"
first_data = load_data(first_path)
textrank_path = Dir.res + "/result/" + name + "/TextRank/detials.txt"
tr_data = load_data(textrank_path)
result = {}
for key in first_data.keys():
a = first_data[key][0] - entry_data[key][0]
b = first_data[key][1] - entry_data[key][1]
c = first_data[key][0] - tr_data[key][0]
d = first_data[key][1] - tr_data[key][1]
e = first_data[key][0] - tr_data[key][0] + entry_data[key][
0] - tr_data[key][0]
f = first_data[key][1] - tr_data[key][1] + entry_data[key][
1] - tr_data[key][1]
result[key] = [a, b, c, d, e, f]
count = 0
news_root = Dir.res + "/" + name + "/news/"
abst_root = Dir.res + "/" + name + "/abstract/"
fname = ftools.get_files(news_root)
new_result = {}
for filename in fname:
# print(filename,count,len(fname))
# news = ftools.read_lines(news_root+filename)
# weibo = ftools.read_lines(abst_root+filename)
# jude = data_filter(news,weibo)
# jude_dict[filename] = jude
jude = jude_dict[filename]
if jude > 0.5:
new_result[filename] = result[filename]
new_result[filename].append(jude)
count += 1
tools.save_object(jude_dict, Dir.res + "/result/judge.txt")
tmp = dict(
sorted(new_result.items(),
key=lambda d: d[1][compare_index],
reverse=True))
save_dict = {}
names = []
for key in tmp.keys():
save_dict[key] = tmp[key]
names.append(key)
save_path = Dir.res + "/result/" + name + "/" + main_name + ".txt"
ftools.write_com_dict(save_path, save_dict)
return names
def analyze_rate(name="cleandata_small", num=None):
save_path = Dir.res + "/result/judge.txt"
jude_dict = tools.load_object(save_path)
print(len(jude_dict))
rate = [0, 0, 0, 0]
# nums = [int(num*var/sum(rate)) for var in rate]
textrank_path = Dir.res + "/result/" + name + "/TextRank/detials.txt"
tr_data = load_data(textrank_path)
tr_data = dict(
sorted(tr_data.items(), key=lambda d: d[1][0], reverse=False))
entry_path = Dir.res + "/result/" + name + "/EntryBigraph/detials.txt"
entry_data = load_data(entry_path)
entry_data = dict(
sorted(entry_data.items(), key=lambda d: d[1][0], reverse=False))
fvae = Dir.res + "/result/" + name + "/Fourth Version auto encoder/detials.txt"
fvae_data = load_data(fvae)
fvae_data = dict(
sorted(fvae_data.items(), key=lambda d: d[1][0], reverse=True))
fvsae = Dir.res + "/result/" + name + "/Fourth Version simple auto encoder/detials.txt"
fvsae_data = load_data(fvsae)
fvsae_data = dict(
sorted(fvsae_data.items(), key=lambda d: d[1][0], reverse=True))
res = []
new_dict = dict(sorted(jude_dict.items(), key=lambda d: d[0],
reverse=True))
for na in new_dict.keys():
if num != None and len(res) >= num:
break
if new_dict[na] > 0.8:
res.append(na)
# for key in list(fvsae_data.keys()):
# if jude_dict[key] >0.5:
# res.append(key)
#
# if len(res)>=nums[0]:
# break
#
# for key in fvae_data.keys():
# if key not in res:
# if jude_dict[key] > 0.5:
# res.append(key)
# if len(res) > sum(nums[:2]):
# break
#
# for key in entry_data.keys():
# if key not in res:
# if jude_dict[key] > 0.5:
# res.append(key)
# if len(res) > sum(nums[:3]):
# break
#
# for key in tr_data.keys():
# if key not in res:
# if jude_dict[key] > 0.5:
# res.append(key)
# if len(res) == num:
# break
return res