def simple_usage():
# Uncomment this line for debugging
# logging.basicConfig(level=logging.DEBUG)
vncorenlp_file = 'D:\study\PlagismDetector\PlagismDetector/VnCoreNLP/VnCoreNLP-1.1.1.jar'
sentences = 'VTV đồng ý chia sẻ bản quyền World Cup 2018 cho HTV để khai thác. ' \
'Nhưng cả hai nhà đài đều phải chờ sự đồng ý của FIFA mới thực hiện được điều này.'
# Use "with ... as" to close the server automatically
with VnCoreNLP(vncorenlp_file) as vncorenlp:
print('Tokenizing:', vncorenlp.tokenize(sentences))
print('POS Tagging:', vncorenlp.pos_tag(sentences))
print('Named-Entity Recognizing:', vncorenlp.ner(sentences))
print('Dependency Parsing:', vncorenlp.dep_parse(sentences))
print('Annotating:', vncorenlp.annotate(sentences))
print('Language:', vncorenlp.detect_language(sentences))
# In this way, you have to close the server manually by calling close function
vncorenlp = VnCoreNLP(vncorenlp_file)
print('Tokenizing:', vncorenlp.tokenize(sentences))
print('POS Tagging:', vncorenlp.pos_tag(sentences))
print('Named-Entity Recognizing:', vncorenlp.ner(sentences))
print('Dependency Parsing:', vncorenlp.dep_parse(sentences))
print('Annotating:', vncorenlp.annotate(sentences))
print('Language:', vncorenlp.detect_language(sentences))
# Do not forget to close the server
vncorenlp.close()
def vn_ner(self):
annotator = VnCoreNLP(address=DEFAULT_LOCAL_ADDRESS,
port=DEFAULT_VI_NER_PORT)
for line in self.textMap.keys():
taggedText = annotator.annotate(line)
try:
taggedText = taggedText['sentences'][0]
for value in taggedText:
if value['nerLabel'] in ['B-PER', 'I-PER']:
self.textMap[line][self.PER_KEY] += 1
if value['nerLabel'] in ['B-LOC', 'I-LOC']:
self.textMap[line][self.LOC_KEY] += 1
if value['nerLabel'] in ['B-ORG', 'I-ORG']:
self.textMap[line][self.ORG_KEY] += 1
except Exception as e:
print("Unable to anotate " + str(line))
print(e)
return e
"/hdd/Zalo_Team_HoaChan/resources/VnCoreNLP/VnCoreNLP-1.1.1.jar",
annotators="wseg,pos,ner,parse",
max_heap_size='-Xmx2g')
# To perform word segmentation, POS tagging and then NER
# annotator = VnCoreNLP("<FULL-PATH-to-VnCoreNLP-jar-file>", annotators="wseg,pos,ner", max_heap_size='-Xmx2g')
# To perform word segmentation and then POS tagging
# annotator = VnCoreNLP("<FULL-PATH-to-VnCoreNLP-jar-file>", annotators="wseg,pos", max_heap_size='-Xmx2g')
# To perform word segmentation only
# annotator = VnCoreNLP("<FULL-PATH-to-VnCoreNLP-jar-file>", annotators="wseg", max_heap_size='-Xmx500m')
# Input
text = "Tuy vậy, 1 phút sau, Chelsea đã cụ thể hóa sức ép liên tục bằng bàn thắng. Người lập công cho The Blues là đội trưởng Cahill với cú đánh đầu cận thành từ tình huống phạt góc."
# To perform word segmentation, POS tagging, NER and then dependency parsing
annotated_text = annotator.annotate(text)
# To perform word segmentation only
word_segmented_text = annotator.tokenize(text)
print('annotated_text: ', annotated_text)
print('word_segmented_text: ', word_segmented_text)
{
'index': 7,
'form': 'Cahill',
'posTag': 'Np',
'nerLabel': 'B-PER',
'head': 6,
'depLabel': 'nmod'
}
from vncorenlp import VnCoreNLP
import time
annotator = VnCoreNLP(
'/home/misa/PycharmProjects/lightning-projects/VnCoreNLP/VnCoreNLP-1.1.jar'
)
text = "Ông Nguyễn Khắc Chúc đang làm việc tại Đại học Quốc gia Hà Nội. Bà Lan, vợ ông Chúc, cũng làm việc tại đây."
annotated_text = annotator.annotate(text)
word_segmented_text = annotator.tokenize(text)
for i in range(10):
semi_text = text[:-i]
start_time = time.time()
annotated_text = annotator.annotate(semi_text)
word_segmented_text = annotator.tokenize(semi_text)
print(str(start_time - time.time()))
print()
class TextRank4KeywordVN():
"""Extract keywords from text"""
def __init__(self, stopwords, ngrams=1, window_size=3, candidate_pos=["N", "Np"], num_keywords=5, use_vncorenlp=True):
self.d = 0.85 # damping coefficient, usually is .85
self.min_diff = 1e-5 # convergence threshold
self.steps = 10 # iteration steps
self.node_weight = None # save keywords and its weight
self.ngrams = ngrams
self.window_size = window_size
self.candidate_pos = candidate_pos
self.num_keywords = num_keywords
self.stopwords = stopwords
self.use_vncorenlp = use_vncorenlp
if self.use_vncorenlp:
self.annotator = VnCoreNLP(VNCORENLP_JAR_PATH, annotators="wseg,pos", max_heap_size='-Xmx2g')
# tokenizing, filtering stopwords for sentence
def filtering_sentence(self, sent, stopwords, keyword, lower=False):
sent = re.sub(r'[^\w\s]','',sent)
filtered_words = []
if lower:
words = word_tokenize(sent, format="text").split(" ")
words = [word.lower() for word in words]
else:
words = word_tokenize(sent, format="text").split(" ")
for word in words:
if word not in stopwords and (keyword not in word) and (word not in keyword) and not (word.isnumeric()) and word not in punctuation:
try:
if detect(word) == "vi":
filtered_words.append(word)
except Exception:
continue
# filtered_words.append(word)
return ' '.join(filtered_words)
# pos-tagging text for segmentation step
def pos_tagging_sentence(self, sent):
if self.use_vncorenlp:
sent = sent.replace("_", " ")
temp = self.annotator.annotate(sent)
return [(element["form"], element["posTag"]) for sent in temp["sentences"] for element in sent]
else:
return pos_tag(sent)
def sentence_segment(self, doc, lower):
"""Store those words only in cadidate_pos"""
sentences = []
for sent in doc:
postag = self.pos_tagging_sentence(sent)
words = [x[0] for x in postag]
selected_words = []
res = []
for i in range(len(words)):
# Store words only with cadidate POS tag
if postag[i][1] in self.candidate_pos or words[i] in ["trình_duyệt"]:
if lower is True:
selected_words.append(words[i].lower())
else:
selected_words.append(words[i])
if self.ngrams == 1:
sentences.append(selected_words)
else:
for i in range(len(selected_words) - self.ngrams + 1):
word = ''
for j in range(self.ngrams):
word += selected_words[i+j]
if j != self.ngrams - 1:
word += ' '
res.append(word)
sentences.append(res)
return sentences
def analyze(self, text, keyword, lower=False):
"""Main function to analyze text"""
doc = sent_tokenize(text)
doc = [self.filtering_sentence(sent, self.stopwords, keyword, lower) for sent in doc]
# Filter sentences
sentences = self.sentence_segment(doc, lower) # list of list of words
# Build vocabulary
vocab = get_vocab(sentences)
# Get token_pairs from windows
token_pairs = get_token_pairs(sentences, self.window_size)
# Get normalized matrix
g = get_matrix(vocab, token_pairs)
# Initionlization for weight(pagerank value)
pr = np.array([1] * len(vocab))
# Iteration
previous_pr = 0
for epoch in range(self.steps):
pr = (1-self.d) + self.d * np.dot(g, pr)
if abs(previous_pr - sum(pr)) < self.min_diff:
break
else:
previous_pr = sum(pr)
# Get weight for each node
node_weight = dict()
for word, index in vocab.items():
node_weight[word] = pr[index]
self.node_weight = node_weight
return get_keywords(self.node_weight, self.num_keywords)
class Extractor:
def __init__(self, jarfile='VnCoreNLP-1.1.1.jar'):
print('Init VnCoreNLP Annotator...')
self.annotator = VnCoreNLP(jarfile,
annotators="wseg,pos,ner,parse",
max_heap_size='-Xmx2g')
def stop(self):
self.annotator.close()
def _pos_tagging(self, text):
pos_tagged_text = self.annotator.pos_tag(text)
return pos_tagged_text
def _ner(self, text):
ner_text = self.annotator.ner(text)
return ner_text
def _lemmatize(self, doc, allowed_postags=('N', 'Np', 'V')):
sentences = []
ignores = set()
for sent in doc:
new_sent = []
for word, tag in sent:
new_sent.append(word)
if tag not in allowed_postags:
ignores.add(word)
sentences.append(new_sent)
return sentences, ignores
def _get_named_entities(self, text):
endline = ('.', 'O')
old_tag = ''
entity_segments = []
entities = []
for sent in text:
sent.append(endline)
for word, tag in sent:
# not a segment of a named entity
if len(tag) < 3 or tag[-3:] not in NER_TAGS:
if entity_segments:
entity = ' '.join(entity_segments)
if (entity, old_tag) not in entities and not any(
p in entity.lower() for p in wrong_entity):
entities.append((entity, old_tag))
entity_segments = []
old_tag = ''
continue
# is a segment of a named entity
tag = tag[-3:]
if tag != old_tag:
if entity_segments:
entity = ' '.join(entity_segments)
if (entity, old_tag) not in entities and not any(
p in entity.lower() for p in wrong_entity):
entities.append((entity, old_tag))
entity_segments = []
old_tag = tag
entity_segments.append(word)
return entities
def extract(self, text):
annotated_text = self.annotator.annotate(text)
ner_text = [[(word['form'], word['nerLabel']) for word in sent]
for sent in annotated_text['sentences']]
pos_tagged_text = [[(word['form'], word['posTag']) for word in sent]
for sent in annotated_text['sentences']]
return self._get_named_entities(ner_text), self._lemmatize(
pos_tagged_text)
def annotate(self, doc):
annotated_doc = self.annotator.annotate(doc)
return [[
Token(word['form'], word['nerLabel'], word['posTag'])
for word in sent
] for sent in annotated_doc['sentences']]
def get_long_tokens(self,
annotated_doc,
pos_tags=('N', 'Ny', 'Np', 'Nc', 'Y', 'Z', 'A'),
min_word_number=2,
max_word_count=6):
eos = Token('.', '.', '.') # end of sentence
long_tokens = []
for sent in annotated_doc:
sent.append(eos)
for i, token in enumerate(sent):
if token.posTag in pos_tags:
tokens = [token.form]
for next_token in sent[i + 1:]:
if next_token.posTag in pos_tags:
tokens.append(next_token.form)
else:
new_long_token = ' '.join(tokens).lower()
if len(tokens) >= min_word_number and len(
tokens) <= max_word_count and not any(
p in new_long_token.replace('_', ' ')
for p in popular_phrase_part
) and not any(new_long_token in p
for p in long_tokens):
long_tokens.append(new_long_token)
break
return long_tokens
def merge_name_entities(self, annotated_doc):
remake_doc = [[(token.form, token.nerLabel) for token in sent]
for sent in annotated_doc]
ners = self._get_named_entities(remake_doc)
new_doc = []
for sent in annotated_doc:
raw_sent = ' '.join([token.form for token in sent]).lower()
pos_tags = [token.posTag for token in sent]
for ner, _ in ners:
ner = ner.lower()
i = raw_sent.find(ner)
while i > -1 and ner.count(' ') > 0:
raw_sent = raw_sent.replace(ner, ner.replace(' ', '_'), 1)
i = raw_sent.count(' ', 0, i)
pos_tags[i:i + ner.count(' ') + 1] = ['N']
i = raw_sent.find(ner)
new_sent = raw_sent.split(' ')
if len(new_sent) != len(pos_tags):
raise Exception('Wrong went merge NE')
new_doc.append([(new_sent[i], pos_tags[i])
for i in range(len(new_sent))])
return ners, new_doc
def merge_noun_phrases(self, tokenized_doc, noun_phrases=()):
new_doc = []
for sent in tokenized_doc:
raw_sent = ' '.join([word for word, tag in sent]).lower()
pos_tags = [tag for word, tag in sent]
for np in noun_phrases:
i = raw_sent.replace('_', ' ').find(np.replace('_', ' '))
while i > -1 and raw_sent[i:i + len(np)].count(' ') > 0:
j = raw_sent.count(' ', 0, i)
pos_tags[j:j + raw_sent[i:i + len(np)].count(' ') +
1] = ['N']
raw_sent = raw_sent[:i] + np.replace(
' ', '_') + raw_sent[i + len(np):]
i = raw_sent.replace('_',
' ').find(np.replace('_', ' '), i + 1)
new_sent = raw_sent.split()
if len(new_sent) != len(pos_tags):
raise Exception('Wrong went merge NE')
new_doc.append([(new_sent[i], pos_tags[i])
for i in range(len(new_sent))])
return new_doc
def get_most_noun_phrases(self, noun_phrases, threshold=2):
appearances = {}
for np in noun_phrases:
appearances[np] = appearances.get(np, 0) + 1
return [np for np, app in appearances.items() if app >= threshold]
def analyse_about(self, about):
annotated_doc = self.annotate(about)
noun_phrases = self.get_long_tokens(annotated_doc,
min_word_number=2,
max_word_count=4)
phrases = self.get_long_tokens(annotated_doc,
pos_tags=('N', 'Np', 'Nc', 'A', 'V'),
min_word_number=2,
max_word_count=5)
named_entities, _ = self.merge_name_entities(annotated_doc)
return noun_phrases, phrases, named_entities
def analyse_content(self, doc, noun_phrases_in_about):
annotated_doc = self.annotate(doc)
named_entities, new_doc = self.merge_name_entities(annotated_doc)
noun_phrases = self.get_long_tokens(annotated_doc,
min_word_number=2,
max_word_count=4)
popular_entity_noun_phrases = [
p for p in noun_phrases if any(
p.startswith(popular_prefix)
for popular_prefix in popular_prefix_named_entity)
]
most_noun_phrases = self.get_most_noun_phrases(noun_phrases)
merged_doc = self.merge_noun_phrases(
new_doc,
noun_phrases=popular_entity_noun_phrases + noun_phrases_in_about +
most_noun_phrases)
while len(merged_doc) > 0 and not merged_doc[0]:
del merged_doc[0]
return self._lemmatize(merged_doc), noun_phrases, named_entities