def preprocess(utterance):
stop_words, missing, quantifier, replacements, start, wrongNE = loadConfig(
'Retrieve')
sent = choppunc(utterance)
b = sent.split()
b[0] = b[0].lower().capitalize()
c = nltk.pos_tag(b)
d = nltk.ne_chunk(c, binary=True)
m = {}
for val in d:
if str(type(val)) == "<class 'nltk.tree.Tree'>":
for v in val:
m[v[0]] = val.label()
else:
m[val[0]] = "NNE"
for u, v in c:
if (v == 'NNP' and m[u] == 'NE' and u not in wrongNE) or (
v == 'CD' and u not in quantifier) or u == ',' or u == 'U.S':
stop_words.append(u.lower())
x = sent.lower().split()
elem = []
for i in range(len(x)):
w = x[i]
if (w not in stop_words):
elem.append(w)
if len(elem) >= 5 and missing[0] in elem:
elem = ' '.join(elem).replace(' .', '.')
elem = elem.replace(' ' + missing[0], '')
else:
elem = ' '.join(elem).replace(' .', '.')
for v in start:
if elem.startswith(v):
elem = elem.replace(v + ' ', '')
for v in replacements:
rep = v.split(',')
if rep[0] in elem:
elem = elem.replace(rep[0], rep[1])
return elem
def ifTwoNegation(utterance):
exception_vadarneg_words, missing_vadarneg_words= loadConfig('ROV')
utterance = utterance.replace(',','')
sid = SentimentIntensityAnalyzer()
arr = []
sent = word_tokenize(utterance)
for i in range(len(sent)):
w = sent[i]
if w == 'no':
continue
ss = sid.polarity_scores(w)
if (ss['neg']==1.0 or w in missing_vadarneg_words) and (w not in exception_vadarneg_words):
arr.append((w,i,abs(ss['compound'])))
if len(arr)==2:
if abs(arr[0][1]-arr[1][1])==2:
return [arr[0][0],arr[1][0]],True
else:
return [arr[1][0]],True
else:
return [],False
def isThereOnlyOneNegation(utterance):
exception_vadarneg_words, missing_vadarneg_words= loadConfig('ROV')
sid = SentimentIntensityAnalyzer()
count = 0
word = ''
arr = []
for w in word_tokenize(utterance):
if w=='no':
continue
ss = sid.polarity_scores(w)
if ss['neg']==1.0 and w not in exception_vadarneg_words:
count = count+1
if count<=1:
word = w
arr.append(word)
elif w in missing_vadarneg_words and count==0:
count = count+1
if count<=1:
word = w
if count==1:
return word,True
return 'cant_change',False
import torch
import numpy
from sentence_retriever import getSentences
from grammar import correct_grammar
from loadconfig import loadConfig
import random
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
correct_phrase = loadConfig('Rank')
def getRoberta():
roberta = torch.hub.load('pytorch/fairseq', 'roberta.large.mnli')
roberta.cuda()
roberta.eval()
return roberta
def getContradictionScores(roberta, sentences, rov):
scores = []
gender = ''
for sent in sentences:
sent, gender = correct_grammar(rov, sent, gender)
tokens = roberta.encode(rov, sent)
value = roberta.predict('mnli', tokens).cpu().detach().numpy()
value = round(value[0].tolist()[0], 3)
scores.append((value, sent.capitalize()))
return scores
import os
from urllib.parse import quote
import nltk
from loadconfig import loadConfig
words = []
nonoverlap_words = loadConfig('Sentences')
def islenPermissible(rov, retrieved):
c = 0
for w in nonoverlap_words:
if w in rov:
c = c + 1
if w in retrieved:
c = c + 1
tokens = nltk.word_tokenize(rov)
tokens1 = nltk.word_tokenize(retrieved)
if len(tokens1) > 2 * len(tokens) or len(tokens1) <= 3 or c >= 2:
return False
return True
def getAllConcepts():
for line in open('./data/concept.txt'):
words.append(line.strip())
return words
def updateConcept(concept):
from loadconfig import loadConfig
import os
import re
import random
__software__ = "FormatText"
__author__ = "MENG Yidong"
__version__ = "2.2"
__params__, __replacements__, __bullshits__ = loadConfig()
def control_illegal_characters(stringToControl):
# illegal characters 非法字符
__illegal_chars__ = __params__["illegal characters"]
for c in __illegal_chars__:
if c in stringToControl:
print("find '" + c + "'")
stringToControl = stringToControl.replace(c, " ")
print("replaced by space")
print("*"*10)
return stringToControl
def control_illegal_combinations(stringToControl):
for key in __replacements__.keys():
if key in stringToControl:
# print("+"*10, stringToControl, key)
stringToControl = stringToControl.replace(key, __replacements__[key])
# print(stringToControl, "-"*10)
return stringToControl