async def Send_MP_absents(self, absents: list, entry: str,
message: discord.Message, delay: int):
"""
Send a mp message to all absents
"""
language_msg = returnLanguage(
readGuild(message.guild.id)["language"], "sendabsents")
embed = discord.Embed(color=discord.Colour.red(), title="Absence")
embed.set_author(name=Tools.name(message.author),
icon_url=message.author.avatar_url)
embed.add_field(name=language_msg[0], value=Tools.name(message.author))
embed.add_field(name=language_msg[1], value=message.guild)
embed.add_field(name=language_msg[2], value=message.guild)
embed.add_field(name="Date", value=date.today().strftime("%d/%m/%Y"))
embed.add_field(name=language_msg[3][0],
value=f"[{language_msg[3][1]}]({message.jump_url})")
if delay:
embed.add_field(
name="Time on receipt of the message to report late",
value=f"{delay} minutes")
for member in absents:
await member.send(embed=embed)
if delay:
message = await member.send(
"Click to send a message to the teacher that you're late")
await message.add_reaction("⏰")
self.missing[message.id] = entry
async def Call(self, context, classroom, showAll: bool):
class_role = Tools.convert(classroom)
data = readGuild(context.guild.id)
if class_role is None:
await Tools.embedError("This is not a role, but a specific user")
else:
if Tools.got_the_role(data["teacher"], context.author):
self.callList[
f"{context.guild.id}-{context.message.id}"] = Check(
class_role, context.message.author, showAll,
data["delay"] if data["delay"] > 0 else 0)
message = returnLanguage(data["language"], "startcall")
embed = discord.Embed(color=discord.Colour.green(),
title=message[0],
description=message[1])
embed.set_author(name=Tools.name(context.message.author),
icon_url=context.message.author.avatar_url)
embed.add_field(name=f"**__{message[2]}__**", value=classroom)
embed.add_field(name="Date",
value=date.today().strftime("%d/%m/%Y"))
embed.set_footer(text=message[3])
await context.channel.send(embed=embed)
await context.message.add_reaction(
"✅") # on rajoute les réactions ✅ & 🆗
await context.message.add_reaction("🆗")
await context.message.add_reaction("🛑")
else:
await context.channel.send("<@{}> : {}".format(
context.author.id,
returnLanguage(data["language"], "notTeacher")))
def init_callbacks(self, config):
if config['callbacks'] is None:
return self
if 'csv_logger' in config['callbacks']:
log_dir = os.path.join(self.model.save_directory, 'logger')
Tools.create_dir_if_not_exists(log_dir)
self.csv_logger = CSVLogger(
filename = '{}/{}training.log'.format(log_dir, self.model.model_name),
append = config['callbacks']['csv_logger']['append']
)
self.callbacks.extend([self.csv_logger])
if 'checkpoint' in config['callbacks']:
chekpt_dir = config['callbacks']['checkpoint']['directory']
Tools.create_dir_if_not_exists(chekpt_dir)
self.checkpointer = ModelCheckpoint(
filepath = chekpt_dir + '/' + self.model.model_name + '-{epoch:02d}.hdf5',
verbose = config['callbacks']['checkpoint']['verbose'],
period = config['callbacks']['checkpoint']['period']
)
self.callbacks.extend([self.checkpointer])
if 'display_picture' in config['callbacks']:
self.picture_displayer = DisplayPictureCallback(
model = self.model,
data_loader = self.data_loader.get_train_generator(),
epoch_laps = config['callbacks']['display_picture']['epoch_laps']
)
self.callbacks.extend([self.picture_displayer])
return self
def generate_data(self,text):
if isinstance(text,str):
sens = tools.seperate_sentences(text)
else:
sens = text
words =[]
sen_words = []
for sen in sens:
wp = tools.sen_pog( sen)
tmp = []
for w,p in wp:
if "n" in p or "v" in p or "m" in p:
tmp.append(w)
if w not in words:
words.append(w)
sen_words.append(tmp)
vector = []
for sen_w in sen_words:
tmp =[0]*len(words)
for i in range(len(words)):
w= words[i]
if w in sen_w:
tmp[i] = 1
vector.append(tmp)
return words,vector
def textrank(self, text):
sentences = tools.seperate_sentences(text)
words = {}
words_list = []
res = {}
sen_words = []
for sen in sentences:
ws = tools.seperate(sen)
sen_words.append(ws)
for w in ws:
if w not in words.keys():
words_list.append(w)
words[w] = len(words)
matrix = np.zeros((len(words), len(words)))
# matrix = [[0] * len(words) for var in range(len(words))]
for sen_w in sen_words:
for i in range(len(sen_w)):
for j in range(i, len(sen_w)):
# print(words[sen_w[i]],words[sen_w[j]],len(words))
matrix[words[sen_w[i]], words[sen_w[j]]] += 1
matrix[words[sen_w[j]], words[sen_w[i]]] += 1
nx_graph = nx.from_numpy_matrix(matrix)
nx_parameter = {'alpha': 0.85}
score = nx.pagerank(nx_graph, **nx_parameter)
sorted_score = sorted(score.items(),
key=lambda item: item[1],
reverse=True)
for index, value in sorted_score:
if words_list[index] not in res.keys():
res[words_list[index]] = value
return res
def weighted_vectorize(self, text):
res = []
sentences = tools.seperate_sentences(text)
tr_text = self.tr.textrank(text)
for sen in sentences:
tmp = []
tmp_weight = []
sen_words = tools.seperate(sen)
for w in sen_words:
if self.model.wv.vocab.__contains__(w):
tmp.append(self.model.__getitem__(w))
if w in tr_text:
tmp_weight.append(tr_text[w])
else:
tmp_weight.append(1 / len(sen_words))
else:
tmp.append([0] * self.vec_length)
tmp_weight.append(1 / len(sen_words))
for i in range(len(tmp)):
tmp[i] = tools.vector_multi(tmp[i],
tmp_weight[i] / sum(tmp_weight))
sen_vec = tools.vector_add_multi(tmp)
if len(sen_vec) == 0:
print(sen)
res.append(sen_vec)
return res
def craw_result_process(root=Dir.res + "/data/"):
files = ftools.get_files(root)
data = []
for i in range(len(files)):
filename = files[i]
if len(data) > 10:
break
lines = ftools.read_lines(root + filename)
for line in lines:
tmp = line.split(",")
# print("news",len(tmp[2]))
# print("news",tmp[2])
#
# print("abstract",len(tmp[1]))
# print("abstract",tmp[1])
abstract = tools.seperate_sentences(tmp[1])
news = tools.seperate_sentences(tmp[2])
print(abstract)
print(news)
# input()
jude = data_filter(news, abstract)
if jude > 0.5:
data.append(['\n'.join(abstract), '\n'.join(news)])
return data
def get_similarity(self,sen1, sen2):
"""默认的用于计算两个句子相似度的函数。
Keyword arguments:
word_list1, word_list2 -- 分别代表两个句子，都是由单词组成的列表
"""
word_list1 = tools.seperate(sen1)
word_list2 = tools.seperate(sen2)
words = list(set(word_list1 + word_list2))
vector1 = [float(word_list1.count(word)) for word in words]
vector2 = [float(word_list2.count(word)) for word in words]
vector3 = [vector1[x] * vector2[x] for x in range(len(vector1))]
vector4 = [1 for num in vector3 if num > 0.]
co_occur_num = sum(vector4)
if abs(co_occur_num) <= 1e-12:
return 0.
denominator = math.log(float(len(word_list1))) + math.log(float(len(word_list2))) # 分母
if abs(denominator) < 1e-12:
return 0.
return co_occur_num / denominator
def vectorize_files(fileroot,savepath):
data = ftools.read_dir_lines_dict(fileroot)
auto = AutoCoder()
count = 0
print(len(data.keys()))
for key in data.keys():
text = '。'.join(data[key])
sens, sens_words, sens_tags = auto.preprocess(text)
start = time.time()
sens_vector,essay_vector = auto.vectorize(sens_words, sens_tags)
end = time.time()
key_text =''.join([''.join(var) for var in sens_words])
save_key = tools.md5(key_text)
tmp =[list(var) for var in sens_vector]
save_object = [tmp,list(essay_vector)]
tools.save_object(save_object,Dir.res + "/encoder/cleandata_8700/"+save_key)
count+=1
print(count,len(data.keys()),end-start)
def data_filter(news, abstract):
if len(news) < 10 or len(news) >= 80:
return -2
if len(abstract) < 2 or len(''.join(abstract)) < 50:
return -1
news_gram2, abstract_gram2 = [], []
for i in range(len(news)):
tmp = tools.seperate(news[i])
news_gram2.append(
set([tmp[k] + tmp[k + 1] for k in range(len(tmp) - 1)]))
# news_gram2.append(set(tmp))
result = 0
for i in range(len(abstract)):
tmp = tools.seperate(abstract[i])
abstract_gram2.append(
set([tmp[k] + tmp[k + 1] for k in range(len(tmp) - 1)]))
# abstract_gram2.append(set(tmp))
value = 0
for j in range(len(news_gram2)):
v = len(abstract_gram2[i].intersection(news_gram2[j]))
if v > value:
value = v
result += value
# print(news_gram2[12])
# print(abstract_gram2[0])
# print(abstract_gram2[0].intersection(news_gram2[12]))
result /= sum([len(abstract_gram2[i]) for i in range(len(abstract_gram2))])
# print(result)
return result
def returnPresent(guild_id: int, role_list: list, class_list: list):
"""
Return presents and absents students who have reacted on a message
"""
class_list.sort(key=lambda x: Tools.name(x).lower())
role_list.sort(key=lambda x: Tools.name(x).lower())
messages = Server(guild_id).returnLanguage("endcall")
presents_msg = messages[0]
absents_msg = str()
students = []
for member in class_list:
if member.id not in students:
presents_msg += f"• *{Tools.name(member)}* <@{member.id}>\n" # [user.display_name,user.id]
students.append(member.id)
if role_list is not None:
role_list.remove(member)
# if there is no more people
if not role_list:
absents_msg += messages[2]
# if there is still people
else:
absents_msg = "\n" + messages[1]
for member in role_list:
absents_msg += f"• *{Tools.name(member)}* <@{member.id}>\n"
return presents_msg, absents_msg, role_list, students
async def Send_MP_absents(self, absents: list, message: discord.Message,
delay: int, teacher: discord.User):
"""
Send a mp message to all absents
"""
language_msg = Server(message.guild.id).returnLanguage("sendabsents")
embed = discord.Embed(color=discord.Colour.red(), title="Absence")
embed.set_author(name=Tools.name(message.author),
icon_url=message.author.avatar_url)
embed.add_field(name=language_msg[0], value=Tools.name(message.author))
embed.add_field(name=language_msg[1], value=message.guild)
embed.add_field(name=language_msg[2], value=message.guild)
embed.add_field(name="Date", value=date.today().strftime("%d/%m/%Y"))
embed.add_field(name=language_msg[3][0],
value=f"[{language_msg[3][1]}]({message.jump_url})")
missing = {}
for member in absents:
try:
await member.send(embed=embed)
if delay:
time = datetime.now()
message = await member.send(
"Click on the emote below to automatically send a late ticket message to the teacher."
)
await message.add_reaction("⏰")
missing[message.id] = LateStudent(teacher, member, message)
except discord.errors.Forbidden:
await teacher.send(
f"Unable to send a late ticket to {member.mention}")
return missing
def __init__(self):
cfg = configparse()
self.parser = Parser()
cfg_mongo = cfg['mongodb']
self.repository = MongoRepository(cfg_mongo['host'], cfg_mongo['port'],
cfg_mongo['database'],
cfg_mongo['collection'])
self.tools = Tools()
def save_value(self, path, key, coverage_list, relative_matrix, clues_list,
entities_list):
ftools.check_filename(path)
save_dict = {}
save_dict[key] = [
coverage_list, relative_matrix, clues_list, entities_list
]
tools.save_object(save_dict, path)
def analyze(self, text):
sens_words, sens_tag = [], []
sens = tools.seperate_sentences(text)
for sen in sens:
tmp_words, tmp_tag = tools.seperate_pog(sen)
sens_words.append(tmp_words)
sens_tag.append(tmp_tag)
return sens, sens_words, sens_tag
def save_value(self, path, text, coverage_list, relative_matrix,
clues_list, entities_list):
ftools.check_filename(path)
save_dict = {}
save_dict['#$#'.join(text)] = [
coverage_list, relative_matrix, clues_list, entities_list
]
tools.save_object(save_dict, path)
def load_parameter(self):
flist = tools.get_files(self.path)
# print(flist)
for name in flist:
fpath = self.path+name
tmp = tools.load_object(fpath)
for key in tmp.keys():
self.parameter[key] = tmp[key]
def coverage_values(self, sens_vector, essay_vector):
coverage_list = []
for sv in sens_vector:
coverage_list.append(self.dist.sim(sv, essay_vector,
self.disttype))
tools.normarlize(coverage_list)
return coverage_list
async def addRole(context, value):
server = Server(context.guild.id)
a = Tools.got_the_role(server.admin, context.author)
if (server.admin["roles"] != []
or server.admin["users"] != []) and not Tools.got_the_role(
server.admin, context.author):
await Tools.SendError(context.channel,
server.GetLanguage().NoPrivileges)
else:
embed = Embed.BasicEmbed(color=discord.Colour.orange())
added_roles = str()
already_added_roles = str()
added_user = str()
already_added_user = str()
if value == "teacher":
roles = server.teacher
elif value == "admin":
roles = server.admin
for role in context.message.role_mentions:
if role.id not in roles["roles"]:
roles["roles"].append(role.id)
added_roles += role.mention + "\n"
else:
already_added_roles += role.mention + "\n"
for user in context.message.mentions:
if user.id not in roles["users"]:
roles["users"].append(user.id)
added_user += user.mention + "\n"
else:
already_added_user += user.mention + "\n"
if added_roles == "" and already_added_roles == "" and added_user == "" and already_added_user == "":
await Tools.SendError(
context.channel,
"You need to add roles or users to use the command")
else:
if added_roles != "":
embed.add_field(name="Added roles", value=added_roles)
if already_added_roles != "":
embed.add_field(name="Roles already added",
value=already_added_roles)
if added_user != "":
embed.add_field(name="Added users", value=added_user)
if already_added_user != "":
embed.add_field(name="Users already added",
value=already_added_user)
server.Save_Settings()
await Tools.AdminCommand(context, embed, "Add teacher Command")
def get_sens_words(self,text):
sens = tools.seperate_sentences(text)
sens_words = []
for line in sens:
words, tags = tools.seperate_pog(line)
for i in range(len(words)):
w = words[i]
if w not in self.words_tags_dict.keys():
self.words_tags_dict[w] = tags[i]
sens_words.append(words)
return sens_words
def preprocess(self,text):
sens_words, sens_tag = [], []
sens = tools.seperate_sentences(text)
for sen in sens:
tmp_words, tmp_tag = [], []
for w, t in tools.sen_pog(sen):
tmp_words.append(w)
tmp_tag.append(t)
sens_words.append(tmp_words)
sens_tag.append(tmp_tag)
return sens, sens_words, sens_tag
def vectorize(self, text):
sens = tools.seperate_sentences(text)
matrix = []
for sen in sens:
tmp = tools.sen_pog(sen)
pog_tmp = []
for w, p in tmp:
if p == "n" or "v" in p:
pog_tmp.append(w)
matrix.append(pog_tmp)
tr_res = self.tr.textrank_matrix(matrix)
def get_similarity_(self, sen1, sen2):
word_list1 = tools.seperate(sen1)
word_list2 = tools.seperate(sen2)
words = list(set(word_list1 + word_list2))
vector1 = [float(word_list1.count(word)) for word in words]
vector2 = [float(word_list2.count(word)) for word in words]
vector1, vector2 = np.mat(vector1), np.mat(vector2)
dist = np.sqrt(np.sum(np.square(vector1 - vector2)))
return dist
async def CheckReaction(self, reaction: discord.Reaction, user,
entry: str):
reactionContent = str(reaction).strip(" ")
if reactionContent == "✅": # si l'utilisateur a coché présent
if Tools.got_the_role(
self.callList[entry].classroom,
user): # si user a le role de la classe correspondante
await self.callList[entry].AddStudent(
user) # on le rajoute à la liste d'appel
elif not Tools.got_the_role(
readGuild(reaction.message.guild.id)['botID'], user):
await reaction.message.remove_reaction("✅", user)
await Tools.embedError(
user,
"You do not have the right to click on ✅, you're not part of this class!"
)
elif reactionContent in ("🆗", "🛑"):
# Check if user got teacher privileges
if self.callList[entry].teacher == user:
await reaction.message.clear_reactions()
if reactionContent == "🆗":
await reaction.message.channel.send(
"<@{}> :{} <@&{}>".format(
user.id,
returnLanguage(
readGuild(
reaction.message.guild.id)["language"],
"FinishCall"), self.callList[entry].classroom))
await self.finishCall(reaction.message.channel, entry,
reaction.message.guild.id, reaction)
else:
await reaction.message.channel.send(
returnLanguage(
readGuild(reaction.message.guild.id)["language"],
"cancelCall"))
del self.callList[entry]
elif not Tools.got_the_role(
readGuild(reaction.message.guild.id)['botID'],
user): # pas le bot
await reaction.message.remove_reaction(reactionContent, user)
await Tools.embedError(
user,
"You can't stop the call, you do not have teacher privileges!"
)
else: # other emoji
await reaction.message.remove_reaction(reactionContent, user)
await Tools.embedError(
user,
"Do not use another emoji on this message while the call is not finished!"
)
def analyze(main_name, compare_index, name="cleandata_small"):
save_path = Dir.res + "/result/judge.txt"
jude_dict = tools.load_object(save_path)
# print(list(jude_dict.keys())[0])
print(len(jude_dict))
entry_path = Dir.res + "/result/" + name + "/EntryBigraph/detials.txt"
entry_data = load_data(entry_path)
first_path = Dir.res + "/result/" + name + "/" + main_name + "/detials.txt"
first_data = load_data(first_path)
textrank_path = Dir.res + "/result/" + name + "/TextRank/detials.txt"
tr_data = load_data(textrank_path)
result = {}
for key in first_data.keys():
a = first_data[key][0] - entry_data[key][0]
b = first_data[key][1] - entry_data[key][1]
c = first_data[key][0] - tr_data[key][0]
d = first_data[key][1] - tr_data[key][1]
e = first_data[key][0] - tr_data[key][0] + entry_data[key][
0] - tr_data[key][0]
f = first_data[key][1] - tr_data[key][1] + entry_data[key][
1] - tr_data[key][1]
result[key] = [a, b, c, d, e, f]
count = 0
news_root = Dir.res + "/" + name + "/news/"
abst_root = Dir.res + "/" + name + "/abstract/"
fname = ftools.get_files(news_root)
new_result = {}
for filename in fname:
# print(filename,count,len(fname))
# news = ftools.read_lines(news_root+filename)
# weibo = ftools.read_lines(abst_root+filename)
# jude = data_filter(news,weibo)
# jude_dict[filename] = jude
jude = jude_dict[filename]
if jude > 0.5:
new_result[filename] = result[filename]
new_result[filename].append(jude)
count += 1
tools.save_object(jude_dict, Dir.res + "/result/judge.txt")
tmp = dict(
sorted(new_result.items(),
key=lambda d: d[1][compare_index],
reverse=True))
save_dict = {}
names = []
for key in tmp.keys():
save_dict[key] = tmp[key]
names.append(key)
save_path = Dir.res + "/result/" + name + "/" + main_name + ".txt"
ftools.write_com_dict(save_path, save_dict)
return names
def sim(self, sen1, sen2):
count = 0
words1 = tools.seperate(sen1)
words2 = tools.seperate(sen2)
for var in words1:
if var in words2:
count += 1
if len(words1) == 0:
print("sentences", sen1)
input()
count /= len(words1)
return count
def analyze(self, text):
sens_words, sens_tag = [], []
sens = tools.seperate_sentences(text)
tmp = []
for sen in sens:
if "原标题" in sen:
continue
tmp.append(sen)
tmp_words, tmp_tag = tools.seperate_pog(sen)
sens_words.append(tmp_words)
sens_tag.append(tmp_tag)
return tmp, sens_words, sens_tag
def text2pic(text):
sens = tools.seperate_sentences(text)
nodes = []
nodes_dict = {}
sen_words =[]
sen_noun_words =[]
for sen in sens:
wp = tools.sen_pog(sen)
tmp_w =[]
tmp_p = []
tmp_noun =[]
for w,p in wp:
if "n" in p or "v" in p or "m" in p:
if w not in nodes:
nodes.append(w)
if w not in nodes_dict.keys():
nodes_dict[w] = 0
nodes_dict[w]+=1
tmp_noun.append(w)
# tmp.append([w,p])
tmp_w.append(w)
tmp_p.append(p)
sen_noun_words.append(tmp_noun)
sen_words .append([tmp_w,tmp_p])
# nodes = []
# tmp = sorted(nodes_dict.items(), key= lambda d:d[1],reverse=True)
# for var,count in tmp:
# nodes.append(var)
#
# print(tmp)
matrix = [[0]*len(nodes) for var in range(len(nodes))]
for k in range(len(sen_noun_words)):
var = sen_noun_words[k]
for i in range(len(var)-1):
for j in range(i+1,len(var)):
#
matrix[nodes.index(var[i])][nodes.index(var[j])] += 1
matrix[nodes.index(var[j])][nodes.index(var[i])] += 1
# nouni_index = sen_words[k][0].index(var[i])
# nounj_index = sen_words[k][0].index(var[j])
# if nouni_index == nounj_index-1 and True:
# matrix[nodes.index(var[i])][nodes.index(var[j])] +=1
# matrix[nodes.index(var[j])][nodes.index(var[i])] +=1
# else:
# for p in sen_words[k][1][nouni_index:nounj_index]:
# if "v" in p or "m" in p:
# matrix[nodes.index(var[i])][nodes.index(var[j])] += 1
# matrix[nodes.index(var[j])][nodes.index(var[i])] += 1
# break
return matrix,nodes
class Unittest(unittest.TestCase):
def setUp(self):
self.tools = Tools()
def test_get_utc_epoch(self):
date = '13.04.2018 13:40'
result = self.tools.get_utc_epoch(date)
self.assertEqual(1523608800000, result)
def test_get_utc_epoch_none(self):
date = None
result = self.tools.get_utc_epoch(date)
self.assertIsNone(result)
def preprocess(self, text):
sens_words, sens_tag = [], []
sens = tools.seperate_sentences(text)
tmp = []
for i in range(1, len(sens)):
sen = sens[i]
# for sen in sens:
if "原标题" in sen:
continue
tmp.append(sen)
tmp_words, tmp_tag = tools.seperate_pog(sen)
sens_words.append(tmp_words)
sens_tag.append(tmp_tag)
return tmp, sens_words, sens_tag