def depart_type(sql, type_list):
type_depart = {} # 每组对应的机构信息
id_depart = getdata(sql) # 已有机构信息的文章编号和作者机构信息
for line in id_depart:
type_depart[line[0]] = line[1].strip()
for a in range(len(type_list)):
# 统计每一组对应的机构信息
a_types = {}
ake = 0
for b in type_list[a]:
if b in type_depart:
ake += 1
# print(type_depart[b]) # 输出对应的机构信息
# 统计每种机构信息出现的次数
if type_depart[b] in a_types:
temp = a_types[type_depart[b]] + 1
a_types[type_depart[b]] = temp
else:
a_types[type_depart[b]] = 1
print("第" + str(a + 1) + "组：")
print(str(len(type_list[a])) + "篇文章共中有" + str(ake) + "篇有机构信息")
if len(a_types) > 0:
aa = reversed(sorted(a_types.items(), key=lambda value: value[1]))
for line in aa:
print(line)
print()
def get_test_corpus(name, dict1):
sql = "select title,keywords,abstract from topics.{0} where author_unique is NULL;".format(
name)
test_docs = []
data = getdata(sql)
for DOC in data:
title = DOC[0]
keywords = DOC[1]
abstract = DOC[2]
temp = "{};{};{}".format(title, keywords, abstract)
with open("./data/temp1.txt", "w", encoding='utf-8') as f:
f.write(temp.rstrip("\n"))
ltp(
r"D:\MyProject\pythonProjects\TopicMine\GetData\geology\data\temp1.txt",
r"D:\MyProject\pythonProjects\TopicMine\GetData\geology\data\temp2.txt"
)
documents = parse_xml("./data/temp2.txt")
test_docs.append(' '.join(documents))
texts = [[word for word in t.split()] for t in test_docs]
corpus_temp = [dict1.doc2bow(text) for text in texts]
for v in range(len(corpus_temp)):
for a in range(len(corpus_temp[v])):
temp1 = corpus_temp[v][a][1]
temp2 = corpus_temp[v][a][0]
corpus_temp[v][a] = (temp2, temp1 / len(corpus_temp[v]))
corpora.MmCorpus.serialize('./data/test_corpus.mm', corpus_temp)
def depart_type(sql, type_list):
type_depart = {} # 每组对应的机构信息
id_depart = getdata(sql) # 已有机构信息的文章编号和作者机构信息
for line in id_depart:
type_depart[line[0]] = line[1].strip()
for a in range(len(type_list)):
# 统计每一组对应的机构信息
a_types = {}
ake = 0
for b in type_list[a]:
if b in type_depart:
ake += 1
# print(type_depart[b]) # 输出对应的机构信息
# 统计每种机构信息出现的次数
if type_depart[b] in a_types:
temp = a_types[type_depart[b]] + 1
a_types[type_depart[b]] = temp
else:
a_types[type_depart[b]] = 1
print("第" + str(a + 1) + "组：")
print(str(len(type_list[a])) + "篇文章共中有" + str(ake) + "篇有机构信息")
if len(a_types) > 0:
aa = reversed(sorted(a_types.items(), key=lambda value: value[1]))
for line in aa:
print(line)
print()
def fill_new_table(sql, name, ins1, ins2):
data = getdata(sql)
id = 0
for row in data:
title = row[1].strip().strip('\n').strip('\r')
authors = row[2]
mname = row[3]
index_terms = row[4]
if row[5] is not None:
keywords = row[5].strip().strip('\n').strip('\r')
else:
keywords = row[5]
abstract = row[6].strip().strip('\n').strip('\r')
pdate = row[7]
try:
# 找到包含名字是name的记录
authorlist = authors.split(";")
for author in authorlist:
new_author = re.split(r'\^c', author.strip())
if new_author[0] == name:
if len(new_author) > 1: # 标记已有机构信息的作者
temp = new_author[1]
id += 1
sql = ins1.format(id, title, temp, authors, mname, index_terms, keywords, abstract, pdate,
name)
handledata(sql)
else:
id += 1
sql = ins2.format(id, title, authors, mname, index_terms, keywords, abstract, pdate,
name)
handledata(sql)
except Exception as e:
print(e)
print("填充新表" + "\"" + name + "\"" + "完成!")
def update_departs(sql, lis, name):
data = getdata(sql)
for a in data:
dd = a[0]
depart = a[1]
if depart not in lis:
temp = "update %s set author_unique=NULL where id =%d" % (name, dd)
handledata(temp)
def update_departs(sql, lis, name):
data = getdata(sql)
for a in data:
dd = a[0]
depart = a[1]
if depart not in lis:
temp = "update %s set author_unique=NULL where id =%d" % (name, dd)
handledata(temp)
def get_paper(name):
sql = 'select title,keywords,abstract from topics.{0};'.format(name)
data = getdata(sql)
for DOC in data:
title = DOC[0]
keywords = DOC[1]
abstract = DOC[2]
# 获得训练数据
temp = '{};{};{}'.format(title, keywords, abstract)
with open('data/temp1.txt', 'a', encoding='utf-8') as f:
f.write('%s\n' % temp.strip())
def get_author_paper(sql, num_coauthor):
# 获取作者列表
author_list = []
data = getdata(sql)
for line in data:
author_list.append([])
id = int(line[0]) - 1
temp = line[1].split(';')
for authors in temp:
author = authors.split('^c')
author_list[id].append(author[0].strip())
# print(author_list)
# 统计拥有相同合著者的文章
related_list = [] # 相关列表
for a in range(len(author_list) - 1):
related_list.append([]) # 存储每篇文章的同一作者文章，根据合著者相同判断
related_list[a].append(a + 1) # 存储文章自身编号
for b in range(a + 1, len(author_list)): # 循环判断其后每篇文章是否有相同合著者
same = 0
for item in author_list[a]:
if item in author_list[b]:
same += 1
if same > num_coauthor: # 参数可以调整，最低为1，即除了作者自身有一个共现作者
related_list[a].append(b + 1) # 存储有相同合作者的文章编号
# print(related_list)
# 去除重复项
for a in range(len(author_list) - 1):
for b in range(a + 1, len(author_list) - 1):
tempa = related_list[a]
tempb = related_list[b]
if set(tempa) & set(tempb): # 如果文章有交集
related_list[b] = related_list[a] + related_list[b]
related_list[a] = []
break
# 输出
new_list = []
id2 = 0
id1 = 0
num = 0
for item in related_list:
if len(item) > 1:
id2 += 1
num += len(set(item))
print(sorted(list(set(item))))
# new_list.append(sorted(list(set(item))))
if len(item) == 1:
id1 += 1
# print(list(set(item)))
print("拥有合著者共现的类别：" + str(id2) + "个； 包括文章：" + str(num) + "篇")
print("没有合著者共现的文章：" + str(id1))
print("总文章数：" + str(len(author_list)))
def get_author_paper(sql, num_coauthor):
# 获取作者列表
author_list = []
data = getdata(sql)
for line in data:
author_list.append([])
id = int(line[0]) - 1
temp = line[1].split(';')
for authors in temp:
author = authors.split('^c')
author_list[id].append(author[0].strip())
# print(author_list)
# 统计拥有相同合著者的文章
related_list = [] # 相关列表
for a in range(len(author_list) - 1):
related_list.append([]) # 存储每篇文章的同一作者文章，根据合著者相同判断
related_list[a].append(a + 1) # 存储文章自身编号
for b in range(a + 1, len(author_list)): # 循环判断其后每篇文章是否有相同合著者
same = 0
for item in author_list[a]:
if item in author_list[b]:
same += 1
if same > num_coauthor: # 参数可以调整，最低为1，即除了作者自身有一个共现作者
related_list[a].append(b + 1) # 存储有相同合作者的文章编号
# print(related_list)
# 去除重复项
for a in range(len(author_list) - 1):
for b in range(a + 1, len(author_list) - 1):
tempa = related_list[a]
tempb = related_list[b]
if set(tempa) & set(tempb): # 如果文章有交集
related_list[b] = related_list[a] + related_list[b]
related_list[a] = []
break
# 输出
new_list = []
id2 = 0
id1 = 0
num = 0
for item in related_list:
if len(item) > 1:
id2 += 1
num += len(set(item))
print(sorted(list(set(item))))
# new_list.append(sorted(list(set(item))))
if len(item) == 1:
id1 += 1
# print(list(set(item)))
print("拥有合著者共现的类别：" + str(id2) + "个； 包括文章：" + str(num) + "篇")
print("没有合著者共现的文章：" + str(id1))
print("总文章数：" + str(len(author_list)))
def get_keywords(name):
sql = 'select keywords from topics.{0};'.format(name)
data = getdata(sql)
keywords = []
for doc in data:
keyword = doc[0]
if keyword != 'None' and keyword != '':
for key in re.split(r',|，|;|；|：|:|“|”|"', keyword):
if len(key) > 1 and key not in keywords:
keywords.append(key)
with open('data/user_dict.txt', 'a', encoding='utf-8') as f:
f.write('\n'.join(keywords))
def get_noun(sql, name, dep_lis):
all_docs = []
for dep in dep_lis:
data = getdata(sql.format(name, dep))
ss = ""
for DOC in data:
title = DOC[0]
keywords = DOC[1]
abstract = DOC[2]
temp = "{};{};{}".format(title, keywords, abstract)
ss += temp + "\n"
with open("./data/temp1.txt", "w", encoding='utf-8') as f:
f.write(ss.rstrip("\n"))
ltp(r"D:\MyProject\pythonProjects\TopicMine\GetData\geology\data\temp1.txt",
r"D:\MyProject\pythonProjects\TopicMine\GetData\geology\data\temp2.txt")
documents = parse_xml("./data/temp2.txt")
all_docs.append(' '.join(documents))
return all_docs
def coauthor_dict_match(sql, dictionary, name):
data = getdata(sql) # 没有机构信息的文章数据
paper_get_type = 0
only_one_type = 0
get_more_type = 0
for line in data:
departs = {}
paper_id = line[1]
for coauthor in line[0].split(";"): # 合著者列表
author = coauthor.split("^c")
if author[0] != name: # 合著者不等于重名者
for item in dictionary.items():
if author[0] in item[1]: # 合著者的姓名在合著者列表中
if item[0] not in departs:
departs[item[0]] = 1
else:
temp = departs[item[0]] + 1
departs[item[0]] = temp
# 输出匹配到合著者匹配结果
# if len(departs) > 0:
# paper_get_type += 1
# print("第%d篇文章的可能机构信息：" % paper_id)
# aa = reversed(sorted(departs.items(), key=lambda value: value[1]))
# for a in aa:
# if a[1] > 1: # 限制 匹配到合著者的个数
# print(a)
# print()
if len(departs) == 1:
for key in departs:
if departs[key] > 1: # 至少两个合著者
print("%d:%s\n" % (paper_id, key))
with open('./data/depart_coauthor_match.txt', 'a', encoding='utf-8') as f:
f.write("%d:%s\n" % (paper_id, key))
only_one_type += 1
aa = "update {0} set author_unique = '{1}' where id ={2};".format(name, key, int(paper_id))
handledata(aa)
# print("可能匹配到机构信息的文章个数：" + str(paper_get_type))
print("匹配到一个机构信息且至少两个合著者的文章个数：" + str(only_one_type))
def get_coauthor_dict(sql, name):
dicts = {} # 存储合作者字典
data = getdata(sql)
for line in data:
if line[0] in dicts: # 唯一作者标识已经存在字典中
new_coauthor = dicts[line[0]]
coauthors = line[1].split(";")
for coauthor in coauthors:
author = coauthor.split("^c")
if author[0] != name and author[0] not in new_coauthor:
new_coauthor.append(author[0])
else:
new_coauthor = []
coauthors = line[1].split(";")
for coauthor in coauthors:
author = coauthor.split("^c")
if author[0] != name:
new_coauthor.append(author[0])
dicts[line[0]] = new_coauthor
return dicts
def get_noun(sql, name, dep_lis):
all_docs = []
for dep in dep_lis:
data = getdata(sql.format(name, dep))
ss = ""
for DOC in data:
title = DOC[0]
keywords = DOC[1]
abstract = DOC[2]
temp = "{};{};{}".format(title, keywords, abstract)
ss += temp + "\n"
with open("./data/temp1.txt", "w", encoding='utf-8') as f:
f.write(ss.rstrip("\n"))
ltp(
r"D:\MyProject\pythonProjects\TopicMine\GetData\geology\data\temp1.txt",
r"D:\MyProject\pythonProjects\TopicMine\GetData\geology\data\temp2.txt"
)
documents = parse_xml("./data/temp2.txt")
all_docs.append(' '.join(documents))
return all_docs
def get_depart_list(sql, name):
departs = getdata(sql)
list1 = []
for depart in departs:
if depart[0] not in list1:
list1.append(depart[0])
# 保留最详细的机构信息，如“成都理工大学信息管理学院”与“成都理工大学”，保留后者
for a in range(len(list1) - 1):
if list1[a] is not None:
for b in range(a + 1, len(list1)):
if list1[b] is not None:
if list1[a].startswith(list1[b]):
list1[b] = None
continue
elif list1[b].startswith(list1[a]):
list1[a] = None
break
list2 = [a for a in list1 if a is not None]
if len(list2) != len(list1):
temp = "select id,author_unique from topics.{0} where author_unique is not NUll;".format(name)
update_departs(temp, list2, name)
return list2
def get_depart_list(sql, name):
departs = getdata(sql)
list1 = []
for depart in departs:
if depart[0] not in list1:
list1.append(depart[0])
# 保留最详细的机构信息，如“成都理工大学信息管理学院”与“成都理工大学”，保留后者
for a in range(len(list1) - 1):
if list1[a] is not None:
for b in range(a + 1, len(list1)):
if list1[b] is not None:
if list1[a].startswith(list1[b]):
list1[b] = None
continue
elif list1[b].startswith(list1[a]):
list1[a] = None
break
list2 = [a for a in list1 if a is not None]
if len(list2) != len(list1):
temp = "select id,author_unique from topics.{0} where author_unique is not NUll;".format(
name)
update_departs(temp, list2, name)
return list2
def fill_new_table(sql, name, ins1, ins2):
data = getdata(sql)
id = 0
for row in data:
title = row[1].strip().strip('\n').strip('\r')
authors = row[2]
mname = row[3]
index_terms = row[4]
if row[5] is not None:
keywords = row[5].strip().strip('\n').strip('\r')
else:
keywords = row[5]
abstract = row[6].strip().strip('\n').strip('\r')
pdate = row[7]
try:
# 找到包含名字是name的记录
authorlist = authors.split(";")
for author in authorlist:
new_author = re.split(r'\^c', author.strip())
if new_author[0] == name:
if len(new_author) > 1: # 标记已有机构信息的作者
temp = new_author[1]
id += 1
sql = ins1.format(id, title, temp, authors, mname,
index_terms, keywords, abstract,
pdate, name)
handledata(sql)
else:
id += 1
sql = ins2.format(id, title, authors, mname,
index_terms, keywords, abstract,
pdate, name)
handledata(sql)
except Exception as e:
print(e)
print("填充新表" + "\"" + name + "\"" + "完成!")
def get_test_corpus(name, dict1):
sql = "select title,keywords,abstract from topics.{0} where author_unique is NULL;".format(name)
test_docs = []
data = getdata(sql)
for DOC in data:
title = DOC[0]
keywords = DOC[1]
abstract = DOC[2]
temp = "{};{};{}".format(title, keywords, abstract)
with open("./data/temp1.txt", "w", encoding='utf-8') as f:
f.write(temp.rstrip("\n"))
ltp(r"D:\MyProject\pythonProjects\TopicMine\GetData\geology\data\temp1.txt",
r"D:\MyProject\pythonProjects\TopicMine\GetData\geology\data\temp2.txt")
documents = parse_xml("./data/temp2.txt")
test_docs.append(' '.join(documents))
texts = [[word for word in t.split()] for t in test_docs]
corpus_temp = [dict1.doc2bow(text) for text in texts]
for v in range(len(corpus_temp)):
for a in range(len(corpus_temp[v])):
temp1 = corpus_temp[v][a][1]
temp2 = corpus_temp[v][a][0]
corpus_temp[v][a] = (temp2, temp1 / len(corpus_temp[v]))
corpora.MmCorpus.serialize('./data/test_corpus.mm', corpus_temp)
#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: ake
@software: PyCharm Community Edition
@time: 2016/4/24 16:51
"""
from GetData.geology.evaluate import getlist
from GetData.preprocess import getdata
if __name__ == '__main__':
departlist = getlist('data/departs.txt')
name1 = '李勇'
sql0 = "select id,authors from {0} where author_unique is NUll;".format(name1)
data = getdata(sql0)
for line in data:
id1 = line[0]
departs = []
doc = line[1].strip().split(';')
for author in doc:
aa = author.split('^c')
if aa[0] != name1 and len(aa) > 1 and aa[1] in departlist:
departs.append(aa[1])
if len(departs) == 1:
print('%d:%s' % (id1, departs[0]))
with open('data/合著者机构匹配.txt', 'a', encoding='utf-8') as f:
f.write('%d:%s' % (id1, departs[0]) + '\n')
# 相对词频 = 某词在文章中出现的次数/文章的总词数
for v in range(len(new_corpus)):
for a in range(len(new_corpus[v])):
temp1 = new_corpus[v][a][1]
temp2 = new_corpus[v][a][0]
new_corpus[v][a] = (temp2, temp1 / len(new_corpus[v]))
corpora.MmCorpus.serialize('./data/type.mm', new_corpus)
# 生成并保存相似矩阵
index = similarities.Similarity('./data',
new_corpus,
num_features=len(dictionary1),
num_best=5)
index.save('./data/sim.index')
if __name__ == '__main__':
name1 = "李勇"
# 统计已有的机构类别信息
depart_list = []
sql1 = "select author_unique from {0} where author_unique is not NUll;".format(
name1)
for depart in getdata(sql1):
if depart[0] not in depart_list:
depart_list.append(depart[0])
with open("./data/departs.txt", "w", encoding="utf-8") as f:
f.write("\n".join(depart_list))
# 生成并本地化保存词典和按词典表示的类别语料
sql2 = "select title,keywords,abstract from topics.{0} where author_unique = '{1}';"
get_corpus(sql2, name1, depart_list)
# print(a)
# print()
if len(departs) == 1:
for key in departs:
if departs[key] > 1: # 至少两个合著者
print("%d:%s\n" % (paper_id, key))
with open('./data/depart_coauthor_match.txt', 'a', encoding='utf-8') as f:
f.write("%d:%s\n" % (paper_id, key))
only_one_type += 1
aa = "update {0} set author_unique = '{1}' where id ={2};".format(name, key, int(paper_id))
handledata(aa)
# print("可能匹配到机构信息的文章个数：" + str(paper_get_type))
print("匹配到一个机构信息且至少两个合著者的文章个数：" + str(only_one_type))
if __name__ == '__main__':
name1 = "李勇"
sql3 = "select author_unique from topics.{0} where author_unique is NUll;".format(name1)
num_no_depart = len(getdata(sql3))
print("没有机构信息的文章有%d篇" % num_no_depart)
# 获取已有单位信息的作者合著者字典
sql1 = "select author_unique,authors from topics.{0} where author_unique is not NUll;".format(name1)
coauthor_dict = get_coauthor_dict(sql1, name1)
# 根据合作者信息字典匹配作者
sql2 = "select authors,id from topics.{0} where author_unique is NUll;".format(name1)
coauthor_dict_match(sql2, coauthor_dict, name1)
new_dict.append(aa)
dictionary1 = corpora.Dictionary(new_dict)
dictionary1.save('./data/type.dict') # 保存为本地词典
new_corpus = [dictionary1.doc2bow(text) for text in texts]
# 相对词频 = 某词在文章中出现的次数/文章的总词数
for v in range(len(new_corpus)):
for a in range(len(new_corpus[v])):
temp1 = new_corpus[v][a][1]
temp2 = new_corpus[v][a][0]
new_corpus[v][a] = (temp2, temp1 / len(new_corpus[v]))
corpora.MmCorpus.serialize('./data/type.mm', new_corpus)
# 生成并保存相似矩阵
index = similarities.Similarity('./data', new_corpus, num_features=len(dictionary1), num_best=5)
index.save('./data/sim.index')
if __name__ == '__main__':
name1 = "李勇"
# 统计已有的机构类别信息
depart_list = []
sql1 = "select author_unique from {0} where author_unique is not NUll;".format(name1)
for depart in getdata(sql1):
if depart[0] not in depart_list:
depart_list.append(depart[0])
with open("./data/departs.txt", "w", encoding="utf-8") as f:
f.write("\n".join(depart_list))
# 生成并本地化保存词典和按词典表示的类别语料
sql2 = "select title,keywords,abstract from topics.{0} where author_unique = '{1}';"
get_corpus(sql2, name1, depart_list)